Latest stuff, rewriting lexer

.clang-format

@@ -0,0 +1,117 @@
+# AUTOGENERATED COPYRIGHT HEADER START
+# Copyright (C) NaN-NaN undefined
+# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+# AUTOGENERATED COPYRIGHT HEADER END
+
+# Basic Formatting
+TabWidth: 4
+UseTab: ForContinuationAndIndentation
+ColumnLimit: 65535
+#- 0 does not respect the original line breaks!
+
+# Language
+Language: Cpp
+Standard: c++17
+
+# Indentation
+AccessModifierOffset: 0
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+IndentCaseLabels: false
+#IndentPPDirectives: true
+IndentWidth: 4
+IndentWrappedFunctionNames: true
+NamespaceIndentation: All
+
+# Includes
+#IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex: '^"warning-disable.hpp"$'
+    Priority: 50
+  - Regex: '^(<|")(config.hpp|common.hpp|ui-common.hpp|strings.hpp|version.hpp|obs.h)("|>)'
+    Priority: 100
+  - Regex: '^<obs-'
+    Priority: 150
+  - Regex: '^<'
+    Priority: 200
+  - Regex: '^<Q'
+    Priority: 250
+  - Regex: '^"'
+    Priority: 300
+  - Regex: '.moc"$'
+    Priority: 300
+  - Regex: '^"warning-enable.hpp"$'
+    Priority: 500
+SortIncludes: true
+
+# Alignment
+AlignAfterOpenBracket: true
+AlignConsecutiveAssignments: true
+AlignConsecutiveDeclarations: true
+AlignEscapedNewlines: Left
+AlignOperands: true
+AlignTrailingComments: false
+DerivePointerAlignment: false
+PointerAlignment: Left
+
+# Wrapping and Breaking
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BraceWrapping:
+  AfterClass: false
+  AfterControlStatement: false
+  AfterEnum: false
+#  AfterExternBlock: false
+  AfterFunction: true
+  AfterNamespace: false
+  AfterStruct: false
+  AfterUnion: false
+  BeforeCatch: false
+  BeforeElse: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeBraces: Custom
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeColon
+#BreakInheritanceList: BeforeColon
+BreakStringLiterals: true
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+Cpp11BracedListStyle: true
+
+# Spaces
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: false
+SpaceBeforeAssignmentOperators: true
+#SpaceBeforeCpp11BracedList: false
+#SpaceBeforeCtorInitializerColon: true
+#SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+#SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInContainerLiterals: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+
+# Other
+CommentPragmas: '^(!FIXME!|!TODO!|ToDo:)'
+CompactNamespaces: false
+DisableFormat: false
+FixNamespaceComments: true
+#ForEachMacros: ''
+KeepEmptyLinesAtTheStartOfBlocks: false
+ReflowComments: false
+SortUsingDeclarations: true

.clang-tidy

@@ -0,0 +1,40 @@
+# AUTOGENERATED COPYRIGHT HEADER START
+# Copyright (C) NaN-NaN undefined
+# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+# AUTOGENERATED COPYRIGHT HEADER END
+
+Checks:          '-*,clang-diagnostic-*,clang-analyzer-*,readability-*,performance-*,portability-*,-portability-simd-intrinsics'
+WarningsAsErrors: false
+HeaderFilterRegex: ''
+AnalyzeTemporaryDtors: false
+FormatStyle:     file
+User:            Xaymar
+CheckOptions:
+  - key:             cert-dcl16-c.NewSuffixes
+    value:           'L;LL;LU;LLU'
+  - key:             cert-oop54-cpp.WarnOnlyIfThisHasSuspiciousField
+    value:           '0'
+  - key:             cppcoreguidelines-explicit-virtual-functions.IgnoreDestructors
+    value:           '1'
+  - key:             cppcoreguidelines-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic
+    value:           '1'
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '1'
+  - key:             google-readability-function-size.StatementThreshold
+    value:           '800'
+  - key:             google-readability-namespace-comments.ShortNamespaceLines
+    value:           '10'
+  - key:             google-readability-namespace-comments.SpacesBeforeComments
+    value:           '2'
+  - key:             modernize-loop-convert.MaxCopySize
+    value:           '16'
+  - key:             modernize-loop-convert.MinConfidence
+    value:           reasonable
+  - key:             modernize-loop-convert.NamingStyle
+    value:           CamelCase
+  - key:             modernize-pass-by-value.IncludeStyle
+    value:           llvm
+  - key:             modernize-replace-auto-ptr.IncludeStyle
+    value:           llvm
+  - key:             modernize-use-nullptr.NullMacros
+    value:           'NULL'

.editorconfig

@@ -0,0 +1,22 @@
+# AUTOGENERATED COPYRIGHT HEADER START
+# Copyright (C) NaN-NaN undefined
+# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+# AUTOGENERATED COPYRIGHT HEADER END
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file.
+[*]
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+indent_style = tab
+indent_size = 4
+
+[*.yml]
+indent_style = space
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false

CMakeLists.txt

@@ -1,3 +1,6 @@
+## AUTOGENERATED COPYRIGHT HEADER START
+# Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+# AUTOGENERATED COPYRIGHT HEADER END
 cmake_minimum_required(VERSION 3.26...3.29.2 FATAL_ERROR)
 
 ################################################################################
@@ -171,6 +174,61 @@ set(PROJECT_AUTHORS "See AUTHORS file")
 set(PROJECT_COPYRIGHT "All Rights Reserved. See LICENSE file for more information")
 set(PROJECT_TRADEMARKS "")
 
+function(init_project TARGET)
+	set_target_properties(${TARGET} PROPERTIES
+		# Always generate position independent code.
+		POSITION_INDEPENDENT_CODE ON
+
+		# Set C++ Standard and Extensions
+		C_STANDARD 17
+		C_STANDARD_REQUIRED ON
+		CXX_STANDARD 20
+		CXX_STANDARD_REQUIRED ON
+		CXX_EXTENSIONS OFF
+
+		# Remove prefix from generated files.
+		PREFIX ""
+		IMPORT_PREFIX ""
+
+		# Never treat warnings as errors.
+		COMPILE_WARNING_AS_ERROR OFF
+	)
+	target_compile_definitions(${TARGET} PRIVATE
+		__STDC_WANT_LIB_EXT1__=1
+	)
+
+	if(WIN32)
+		target_compile_definitions(${TARGET} PRIVATE
+			# windows.h
+			#- Disable MIN/MAX macro as this breaks a lot of code.
+			NOMINMAX
+			#- Disable IN/OUT macro as this breaks a lot of code.
+			NOINOUT
+		)
+	endif()
+
+	if(MSVC) # Microsoft Visual C/C++		
+		target_compile_options(${TARGET} PRIVATE
+			# Disable useless/terrible behavior from MSVC
+			_CRT_SECURE_NO_WARNINGS
+			_ENABLE_EXTENDED_ALIGNED_STORAGE
+
+			# Dynamically link Microsoft C/C++ Redistributable.
+			$<$<CONFIG:>:/MD>
+			$<$<CONFIG:Debug>:/MDd>
+			$<$<CONFIG:Release>:/MD>
+			$<$<CONFIG:RelWithDebInfo>:/MD>
+			$<$<CONFIG:MinSizeRel>:/MD>
+
+			# Always compile using multiple processors. Why is this defaulting off anyway?!
+			/MP
+
+			# We want full exception support all the time, not conditionally.
+			/EHa
+		)
+	endif()
+endfunction()
+
 ################################################################################
 # Sub-projects
 ################################################################################

code_compiler/CMakeLists.txt

@@ -1,10 +1,22 @@
-project(code_compiler)
+## AUTOGENERATED COPYRIGHT HEADER START
+# Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+# AUTOGENERATED COPYRIGHT HEADER END
+project(code_compiler
+	VERSION ${PROJECT_VERSION}
+)
 add_executable(${PROJECT_NAME})
+init_project(${PROJECT_NAME})
 
 target_sources(${PROJECT_NAME} PRIVATE
 	"source/main.cpp"
 	"source/lexer.hpp"
 	"source/lexer.cpp"
+	"source/error.hpp"
+	"source/error.cpp"
+	"source/parser.hpp"
+	"source/parser.cpp"
+	"source/compiler.hpp"
+	"source/compiler.cpp"
 	"source/ast/ast.hpp"
 	"source/ast/ast.cpp"
 	"source/ast/arithmetic.hpp"
@@ -13,10 +25,6 @@ target_sources(${PROJECT_NAME} PRIVATE
 	"source/ast/function.cpp"
 	"source/ast/value.hpp"
 	"source/ast/value.cpp"
-	"source/parser.hpp"
-	"source/parser.cpp"
-	"source/compiler.hpp"
-	"source/compiler.cpp"
 )
 target_include_directories(${PROJECT_NAME} PRIVATE
 	"${PROJECT_SOURCE_DIR}/source"
@@ -25,3 +33,4 @@ target_include_directories(${PROJECT_NAME} PRIVATE
 
 get_target_property(_SOURCES ${PROJECT_NAME} SOURCES)
 source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${_SOURCES})
+

code_compiler/source/ast/arithmetic.cpp

@@ -1,6 +1,9 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "arithmetic.hpp"
 
-blitz::AST::ArithmeticExpression::ArithmeticExpression(Operator op, std::unique_ptr<Expression> left, std::unique_ptr<Expression> right)
+blitz::ast::arithmetic_expression::arithmetic_expression(expression_operator op, std::unique_ptr<expression> left, std::unique_ptr<expression> right)
 	: m_operator(op), m_left(std::move(left)), m_right(std::move(right)) {}
 
-blitz::AST::ArithmeticExpression::~ArithmeticExpression() {}
+blitz::ast::arithmetic_expression::~arithmetic_expression() {}

code_compiler/source/ast/arithmetic.hpp

@@ -1,10 +1,13 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "ast.hpp"
 #include "value.hpp"
 
 namespace blitz {
-	namespace AST {
-		enum class Operator : int8_t {
+	namespace ast {
+		enum class expression_operator : int8_t {
 			Add, /*+*/
 			Subtract, /*-*/
 			Multiply, /***/
@@ -14,14 +17,14 @@ namespace blitz {
 			Equal, /*=*/
 		};
 
-		class ArithmeticExpression : public Expression {
+		class arithmetic_expression : public expression {
 			public:
-			ArithmeticExpression(Operator op, std::unique_ptr<Expression> left, std::unique_ptr<Expression> right);
-			virtual ~ArithmeticExpression();
+			arithmetic_expression(expression_operator op, std::unique_ptr<expression> left, std::unique_ptr<expression> right);
+			virtual ~arithmetic_expression();
 
 			private:
-			Operator m_operator;
-			std::unique_ptr<Expression> m_left, m_right;
+			expression_operator m_operator;
+			std::unique_ptr<expression> m_left, m_right;
 		};
 	}
 }

code_compiler/source/ast/ast.hpp

@@ -1,10 +1,13 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 
 namespace blitz {
-	namespace AST {
-		class Expression {
+	namespace ast {
+		class expression {
 			public:
-			virtual ~Expression() {};
+			virtual ~expression() {};
 		};
 	}
 }

code_compiler/source/ast/function.cpp

@@ -1,14 +1,17 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "function.hpp"
 
-blitz::AST::ScopeExpression::ScopeExpression() {}
+blitz::ast::ScopeExpression::ScopeExpression() {}
 
-blitz::AST::ScopeExpression::~ScopeExpression() {}
+blitz::ast::ScopeExpression::~ScopeExpression() {}
 
-void blitz::AST::ScopeExpression::AddExpression(std::unique_ptr<Expression> ex) {
+void blitz::ast::ScopeExpression::AddExpression(std::unique_ptr<expression> ex) {
 	m_expressions.push_back(std::move(ex));
 }
 
-blitz::AST::FunctionExpression::FunctionExpression(ValueType returnType,
+blitz::ast::FunctionExpression::FunctionExpression(ValueType returnType,
 	std::string& m_name,
 	std::list<std::unique_ptr<VariableExpression>> parameters, 
 	std::unique_ptr<ScopeExpression> scope)
@@ -16,11 +19,11 @@ blitz::AST::FunctionExpression::FunctionExpression(ValueType returnType,
 
 }
 
-blitz::AST::FunctionExpression::~FunctionExpression() {}
+blitz::ast::FunctionExpression::~FunctionExpression() {}
 
-blitz::AST::CallExpression::CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments)
+blitz::ast::CallExpression::CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments)
 	: m_name(name), m_arguments(std::move(arguments)) {
 
 }
 
-blitz::AST::CallExpression::~CallExpression() {}
+blitz::ast::CallExpression::~CallExpression() {}

code_compiler/source/ast/function.hpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "ast.hpp"
 #include "value.hpp"
@@ -6,16 +9,16 @@
 #include <string>
 
 namespace blitz {
-	namespace AST {
-		class ScopeExpression : public Expression {
+	namespace ast {
+		class ScopeExpression : public expression {
 			public:
 			ScopeExpression();
 			virtual ~ScopeExpression();
 
-			void AddExpression(std::unique_ptr<Expression> ex);
+			void AddExpression(std::unique_ptr<expression> ex);
 
 			private:
-			std::list<std::unique_ptr<Expression>> m_expressions;
+			std::list<std::unique_ptr<expression>> m_expressions;
 		};
 
 		class FunctionExpression : public ScopeExpression {
@@ -33,7 +36,7 @@ namespace blitz {
 			std::unique_ptr<ScopeExpression> m_content;
 		};
 
-		class CallExpression : public Expression {
+		class CallExpression : public expression {
 			public:
 			CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments);
 			virtual ~CallExpression();

code_compiler/source/ast/value.cpp

@@ -1,43 +1,46 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "value.hpp"
 
-blitz::AST::VariableExpression::VariableExpression(std::string& name, ValueType type /*= ValueType::Number*/)
+blitz::ast::VariableExpression::VariableExpression(std::string& name, ValueType type /*= ValueType::Number*/)
 	: m_name(name), m_type(type) {}
 
-blitz::AST::VariableExpression::~VariableExpression() {}
+blitz::ast::VariableExpression::~VariableExpression() {}
 
-blitz::AST::ValueType blitz::AST::VariableExpression::GetType() {
+blitz::ast::ValueType blitz::ast::VariableExpression::GetType() {
 	return m_type;
 }
 
-blitz::AST::NumberExpression::NumberExpression(int32_t value) : value(value) {}
+blitz::ast::NumberExpression::NumberExpression(int32_t value) : value(value) {}
 
-blitz::AST::NumberExpression::~NumberExpression() {}
+blitz::ast::NumberExpression::~NumberExpression() {}
 
-blitz::AST::ValueType blitz::AST::NumberExpression::GetType() {
+blitz::ast::ValueType blitz::ast::NumberExpression::GetType() {
 	return ValueType::Number;
 }
 
-blitz::AST::DecimalExpression::DecimalExpression(float_t value) : value(value) {}
+blitz::ast::DecimalExpression::DecimalExpression(float_t value) : value(value) {}
 
-blitz::AST::DecimalExpression::~DecimalExpression() {}
+blitz::ast::DecimalExpression::~DecimalExpression() {}
 
-blitz::AST::ValueType blitz::AST::DecimalExpression::GetType() {
+blitz::ast::ValueType blitz::ast::DecimalExpression::GetType() {
 	return ValueType::Decimal;
 }
 
-blitz::AST::StringExpression::StringExpression(std::string value) : value(value) {}
+blitz::ast::StringExpression::StringExpression(std::string value) : value(value) {}
 
-blitz::AST::StringExpression::~StringExpression() {}
+blitz::ast::StringExpression::~StringExpression() {}
 
-blitz::AST::ValueType blitz::AST::StringExpression::GetType() {
+blitz::ast::ValueType blitz::ast::StringExpression::GetType() {
 	return ValueType::String;
 }
 
-blitz::AST::ConstExpression::ConstExpression(std::string& name, std::unique_ptr<ValueExpression> value)
+blitz::ast::ConstExpression::ConstExpression(std::string& name, std::unique_ptr<ValueExpression> value)
 	: m_name(name), m_value(std::move(value)) {}
 
-blitz::AST::ConstExpression::~ConstExpression() {}
+blitz::ast::ConstExpression::~ConstExpression() {}
 
-blitz::AST::ValueType blitz::AST::ConstExpression::GetType() {
+blitz::ast::ValueType blitz::ast::ConstExpression::GetType() {
 	return m_value->GetType();
 }

code_compiler/source/ast/value.hpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "ast.hpp"
 #include "lexer.hpp"
@@ -8,7 +11,7 @@
 #include <string>
 
 namespace blitz {
-	namespace AST {
+	namespace ast {
 		enum class ValueType : int8_t {
 			Unknown,
 			Number,
@@ -17,7 +20,7 @@ namespace blitz {
 			Type,
 		};
 
-		class ValueExpression : public Expression {
+		class ValueExpression : public expression {
 			public:
 			virtual ValueType GetType() = 0;
 		};

code_compiler/source/compiler.cpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "compiler.hpp"
 #include "parser.hpp"
 #include "lexer.hpp"
@@ -7,19 +10,3 @@
 blitz::compiler::compiler() {}
 
 blitz::compiler::~compiler() {}
-
-bool blitz::compiler::compile(std::string in, std::string out) {
-	/*std::ifstream infile;
-	infile.open(in);
-	if (infile.bad() || !infile.good() || infile.eof()) {
-		std::cerr << "Failed to open file: " << in << std::endl;
-		return false;
-	}*/
-
-	parser psr = parser(in);
-	if (!psr.Parse()) {
-
-	}
-
-	return true;
-}

code_compiler/source/compiler.hpp

@@ -1,12 +1,22 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include <string>
+#include <filesystem>
+
+// Compiling requires several steps
+// 1. Lexing to known tokens
+//   - This part may require a unicode library/framework, like Qt or ICU
+// 2. Parsing the lexical tokens into an Abstract Syntax Tree (AST)
+//   - This is where we convert everything to logical steps, like math and function calls.
+// 3. Compiling from the AST to some kind of runnable binary.
+//   - In our case, we convert the AST to LLVM IR, and then compile it with LLVM.
 
 namespace blitz {
 	class compiler {
 		public:
 		compiler();
 		~compiler();
-
-		bool compile(std::string in, std::string out);
 	};
 }

code_compiler/source/error.cpp

@@ -0,0 +1,4 @@
+// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) NaN-NaN undefined
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END

code_compiler/source/error.hpp

@@ -0,0 +1,4 @@
+// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) NaN-NaN undefined
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END

code_compiler/source/lexer.cpp

@@ -1,49 +1,170 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "lexer.hpp"
 #include <codecvt>
+#include <sstream>
 
-std::pair<char, blitz::Lexer::Token> g_symbolCharacters[] = {
+/*std::pair<char, blitz::tokentype> g_symbolCharacters[] = {
 	//{ '\"', BlitzLLVM::Lexer::Token::TokenDoubleQuote }, // Has special meaning.
-	{ '+', blitz::Lexer::Token::TokenPlus },
-	{ '-', blitz::Lexer::Token::TokenMinus },
-	{ '/', blitz::Lexer::Token::TokenSlashForward },
-	{ '\\', blitz::Lexer::Token::TokenSlashBackward },
-	{ '*', blitz::Lexer::Token::TokenMultiply },
-	{ '=', blitz::Lexer::Token::TokenEqual },
-	{ '#', blitz::Lexer::Token::TokenOctothorp },
-	{ '%', blitz::Lexer::Token::TokenPercent },
-	{ '$', blitz::Lexer::Token::TokenDollar },
-	{ '(', blitz::Lexer::Token::TokenRoundBracketOpen },
-	{ ')', blitz::Lexer::Token::TokenRoundBracketClose },
-	{ '[', blitz::Lexer::Token::TokenSquareBracketOpen },
-	{ ']', blitz::Lexer::Token::TokenSquareBracketClose },
-	{ '<', blitz::Lexer::Token::TokenAngleBracketOpen },
-	{ '>', blitz::Lexer::Token::TokenAngleBracketClose },
-	//{ '.', BlitzLLVM::Lexer::Token::TokenDot }, // Special meaning.
-	{ ':', blitz::Lexer::Token::TokenColon },
-	{ ',', blitz::Lexer::Token::TokenComma },
-	//{ ';', BlitzLLVM::Lexer::Token::TokenSemicolon },
-	{ '^', blitz::Lexer::Token::TokenCaret },
-	{ '~', blitz::Lexer::Token::TokenBitNot },
-};
+	{ '+', blitz::tokentype::TokenPlus },
+	{ '-', blitz::tokentype::TokenMinus },
+	{ '/', blitz:::tokentype::TokenSlashForward },
+	{ '\\', blitz::tokentype::TokenSlashBackward },
+	{ '*', blitz::tokentype::TokenMultiply },
+	{ '=', blitz::tokentype::TokenEqual },
+	{ '#', blitz::tokentype::TokenOctothorp },
+	{ '%', blitz::tokentype::TokenPercent },
+	{ '$', blitz::tokentype::TokenDollar },
+	{ '(', blitz::tokentype::TokenRoundBracketOpen },
+	{ ')', blitz::tokentype::TokenRoundBracketClose },
+	{ '[', blitz::tokentype::TokenSquareBracketOpen },
+	{ ']', blitz::tokentype::TokenSquareBracketClose },
+	{ '<', blitz::tokentype::TokenAngleBracketOpen },
+	{ '>', blitz::tokentype::TokenAngleBracketClose },
+	//{ '.', BlitzLLVM::Token::TokenDot }, // Special meaning.
+	{ ':', blitz::tokentype::TokenColon },
+	{ ',', blitz::tokentype::TokenComma },
+	//{ ';', BlitzLLVM::Token::TokenSemicolon },
+	{ '^', blitz::tokentype::TokenCaret },
+	{ '~', blitz::tokentype::TokenBitNot },
+};*/
 
-blitz::Lexer::Lexer() {}
+blitz::lexer::~lexer() {}
 
-blitz::Lexer::~Lexer() {}
+blitz::lexer::lexer(std::filesystem::path file)
+{
+	// Usually files start at line and character 0, so we should start there too.
+	_line = _character = 0;
 
-std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetCurrentToken() {
-	return std::make_pair(m_currentToken, m_currentText);
+	// Try and open the file for reading.
+	_file   = file;
+	_stream = std::ifstream(_file, std::ios_base::binary); // We use binary so we can eventually support UTF-8.
+	if (!_stream.good() || _stream.eof() || _stream.bad() || _stream.fail()) {
+		char buffer[16384];
+		int  len = snprintf(buffer, sizeof(buffer), "Reading file '%s' failed.\0", file.generic_string().c_str());
+		throw std::runtime_error(std::string(buffer, buffer + len));
+	}
+
+	// Initialize token storage to a default token.
+	_override = _current = blitz::token{
+		.line      = 0,
+		.character = 0,
+		.text      = "",
+		.type      = token::variant::UNKNOWN,
+	};
 }
 
-std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shared_ptr<std::istream> fs) {
+blitz::token blitz::lexer::current()
+{
+	return _current;
+}
+
+blitz::token blitz::lexer::next()
+{
+	enum class stage {
+		DEFAULT,
+		TEXT,
+		NUMBER,
+		STRING,
+		COMMENT,
+	} state = stage::DEFAULT;
+	bool numberHasDot = false;
+
+	std::stringstream buffer;
+	blitz::token      token{
+			 .line      = _line,
+			 .character = _character,
+			 .text      = "",
+			 .type      = blitz::token::variant::UNKNOWN,
+    };
+
+	// Helper function to advance text.
+	auto advance = [this]() {
+		_character++;
+		return _stream.get();
+	};
+
+	while ((token.type == blitz::token::variant::UNKNOWN) && _stream.good() && !_stream.eof()) {
+		auto chr = advance();
+
+		if (state == stage::DEFAULT) {
+			if (chr == ';') { // We've encountered a comment, so we should change state and ignore this symbol.
+				state           = stage::COMMENT;
+				token.line      = _line;
+				token.character = _character;
+				token.type      = blitz::token::variant::COMMENT;
+			} else {
+				buffer << chr;
+			}
+		} else if (state == stage::NUMBER) {
+			if (isdigit(chr)) {
+				buffer << chr;
+			} else if (chr == '.')
+			{
+				if (numberHasDot) {
+					throw std::runtime_error("")
+				}
+				numberHasDot = true;
+			} else {
+
+			}
+
+		} else if (state == stage::TEXT) {
+		} else if (state == stage::STRING) {
+		} else if (state == stage::COMMENT) {
+			if (chr == '\r' && _stream.peek() == '\n') {
+				token.text = buffer.str();
+			} else {
+				buffer << chr;
+			}
+		}
+	}
+
+	_current = token;
+	return _current;
+}
+
+/*
+std::pair<blitz::tokentype, std::string> blitz::lexer::current() {
+	return _current;
+}
+
+std::pair<blitz::tokentype, std::string> blitz::lexer::next(std::istream& fs) {
+	std::stringstream buffer;
+	blitz::tokentype token;
+
+	enum class parserState {
+		DEFAULT,
+		TEXT,
+		NUMBER,
+		STRING,
+		COMMENT,
+	} state = parserState::DEFAULT;
+
+	while ((token == blitz::tokentype::TokenUnknown) && !fs.eof() && fs.good()) {
+		auto chr = fs.get();
+
+
+
+
+
+	}
+
+}
+
+
+/*
+std::pair<blitz::lexer::token, std::string> blitz::lexer::next(std::shared_ptr<std::istream> fs) {
 	std::string buf;
-	Token tkn = Token::TokenEOF;
+	token tkn = token::TokenUnknown;
 	bool haveResult = false;
 
 	// Allow "overriding" the next retrieved Token.
-	if (m_overrideToken != Token::TokenUnknown) {
+	if (m_overrideToken != token::TokenUnknown) {
 		buf = m_overrideText;
 		tkn = m_overrideToken;
-		m_overrideToken = Token::TokenUnknown;
+		m_overrideToken = token::TokenUnknown;
 		haveResult = true;
 	}
 
@@ -56,11 +177,11 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 		char chr = fs->get();
 
 		if (chr == '\r' || chr == '\n') {
-			if (tkn != Token::TokenEOF) {
-				m_overrideToken = Token::TokenNewLine;
+			if (tkn != token::TokenEOF) {
+				m_overrideToken = token::TokenNewLine;
 				m_overrideText = "";
 			} else {
-				tkn = Token::TokenNewLine;
+				tkn = token::TokenNewLine;
 				buf = "";
 			}
 
@@ -71,10 +192,10 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			break;
 		} else if (m_isStringMode) {
 			if (chr == '\"') {
-				m_overrideToken = Token::TokenDoubleQuote;
+				m_overrideToken = token::TokenDoubleQuote;
 				m_overrideText = chr;
 				m_isStringMode = false;
-				tkn = Token::TokenQuotedText;
+				tkn = token::TokenQuotedText;
 				break;
 			} else if (iscntrl(chr) || !isprint(chr)) {
 				fs->putback(chr);
@@ -97,7 +218,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			} else if (chr == '.') {
 				if (m_numberModeHasDecimal == false) {
 					m_numberModeHasDecimal = true;
-					tkn = Token::TokenDecimal;
+					tkn = token::TokenDecimal;
 					buf += chr;
 				} else {
 					fs->putback(chr);
@@ -111,7 +232,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			}
 		} else if (m_isCommentMode) {
 			buf += chr;
-			tkn = Token::TokenComment;
+			tkn = token::TokenComment;
 		} else {
 			// Whitespace
 			if (isspace(chr))
@@ -119,7 +240,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 
 			// Control Code
 			if (iscntrl(chr)) {
-				tkn = Token::TokenUnknown;
+				tkn = token::TokenUnknown;
 				buf = chr;
 			}
 
@@ -129,13 +250,13 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 				if (isdigit(chr2)) {
 					m_isNumberMode = true;
 					m_numberModeHasDecimal = false;
-					tkn = Token::TokenNumber;
+					tkn = token::TokenNumber;
 					buf = chr + chr2;
 					break;
 				} else if (chr2 == '.') {
 					m_isNumberMode = true;
 					m_numberModeHasDecimal = true;
-					tkn = Token::TokenDecimal;
+					tkn = token::TokenDecimal;
 					buf = chr + "0" + chr2;
 					break;
 				} else {
@@ -151,7 +272,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 					break;
 				}
 			}
-			if (tkn != Token::TokenEOF) {
+			if (tkn != token::TokenEOF) {
 				haveResult = true;
 				break;
 			}
@@ -159,30 +280,30 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			// Strings, Text, Numbers
 			if (chr == ';') {
 				m_isCommentMode = true;
-				tkn = Token::TokenSemicolon;
+				tkn = token::TokenSemicolon;
 				buf = chr;
 				break;
 			} else if (chr == '\"') {
 				m_isStringMode = true;
-				tkn = Token::TokenDoubleQuote;
+				tkn = token::TokenDoubleQuote;
 				buf = chr;
 				break;
 			} else if (isalpha(chr)) {
 				m_isTextMode = true;
-				tkn = Token::TokenText;
+				tkn = token::TokenText;
 				buf = chr;
 			} else if (isdigit(chr)) {
 				m_isNumberMode = true;
 				m_numberModeHasDecimal = false;
-				tkn = Token::TokenNumber;
+				tkn = token::TokenNumber;
 				buf = chr;
 			} else if (chr == '.') {
 				m_isNumberMode = true;
 				m_numberModeHasDecimal = true;
-				tkn = Token::TokenDecimal;
+				tkn = token::TokenDecimal;
 				buf = "0" + chr;
 			} else {
-				tkn = Token::TokenUnknown;
+				tkn = token::TokenUnknown;
 				buf = chr;
 				break;
 			}
@@ -190,90 +311,91 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 	}
 
 	// Convert from Text into native Token.
-	if (tkn == Token::TokenText)
-		tkn = ConvertTextToToken(tkn, buf);
+	if (tkn == token::TokenText)
+		tkn = to_token(tkn, buf);
 
 	return std::make_pair(tkn, buf);
 }
 
-blitz::Lexer::Token blitz::Lexer::ConvertTextToToken(Token in, std::string text) {
-	static std::pair<const char*, Token> l_textToTokenList[] = {
+blitz::lexer::token blitz::lexer::to_token(token in, std::string text) {
+	static std::pair<const char*, token> l_textToTokenList[] = {
 		// Binary
-		{ "not", Token::TokenNot },
-		{ "and", Token::TokenAnd },
-		{ "or", Token::TokenOr },
-		{ "xor", Token::TokenXor },
-		{ "shl", Token::TokenShl },
-		{ "shr", Token::TokenShr },
-		{ "sal", Token::TokenSal },
-		{ "sar", Token::TokenSar },
-		{ "false", Token::TokenFalse },
-		{ "true", Token::TokenTrue },
+		{ "not", token::TokenNot },
+		{ "and", token::TokenAnd },
+		{ "or", token::TokenOr },
+		{ "xor", token::TokenXor },
+		{ "shl", token::TokenShl },
+		{ "shr", token::TokenShr },
+		{ "sal", token::TokenSal },
+		{ "sar", token::TokenSar },
+		{ "false", token::TokenFalse },
+		{ "true", token::TokenTrue },
 
 		// Conversion
-		{ "float", Token::TokenFloat },
-		{ "string", Token::TokenString },
-		{ "hex", Token::TokenHex },
-		{ "int", Token::TokenInt },
+		{ "float", token::TokenFloat },
+		{ "string", token::TokenString },
+		{ "hex", token::TokenHex },
+		{ "int", token::TokenInt },
 
 		// Control
-		{ "if", Token::TokenIf },
-		{ "then", Token::TokenThen },
-		{ "elseIf", Token::TokenElseIf },
-		{ "else", Token::TokenElse },
-		{ "endIf", Token::TokenEndIf },
-		{ "select", Token::TokenSelect },
-		{ "case", Token::TokenCase },
-		{ "default", Token::TokenDefault },
-		{ "goto", Token::TokenGoto },
-		{ "gosub", Token::TokenGosub },
-		{ "return", Token::TokenReturn },
-		{ "function", Token::TokenFunction },
-		{ "end", Token::TokenEnd },
-		{ "stop", Token::TokenStop },
+		{ "if", token::TokenIf },
+		{ "then", token::TokenThen },
+		{ "elseif", token::TokenElseIf },
+		{ "else", token::TokenElse },
+		{ "endif", token::TokenEndIf },
+		{ "select", token::TokenSelect },
+		{ "case", token::TokenCase },
+		{ "default", token::TokenDefault },
+		{ "goto", token::TokenGoto },
+		{ "gosub", token::TokenGosub },
+		{ "return", token::TokenReturn },
+		{ "function", token::TokenFunction },
+		{ "end", token::TokenEnd },
+		{ "stop", token::TokenStop },
 
 		// Loop
-		{ "for", Token::TokenFor },
-		{ "to", Token::TokenTo },
-		{ "next", Token::TokenNext },
-		{ "while", Token::TokenWhile },
-		{ "wend", Token::TokenWend },
-		{ "repeat", Token::TokenRepeat },
-		{ "until", Token::TokenUntil },
-		{ "forever", Token::TokenForever },
-		{ "exit", Token::TokenExit },
+		{ "for", token::TokenFor },
+		{ "to", token::TokenTo },
+		{ "next", token::TokenNext },
+		{ "while", token::TokenWhile },
+		{ "wend", token::TokenWend },
+		{ "repeat", token::TokenRepeat },
+		{ "until", token::TokenUntil },
+		{ "forever", token::TokenForever },
+		{ "exit", token::TokenExit },
 
 		// Math
-		{ "abs", Token::TokenAbs },
-		{ "sign", Token::TokenSign },
-		{ "cos", Token::TokenCos },
-		{ "sin", Token::TokenSin },
-		{ "tan", Token::TokenTan },
-		{ "acos", Token::TokenACos },
-		{ "asin", Token::TokenASin },
-		{ "atan", Token::TokenATan },
-		{ "atan2", Token::TokenATan2 },
-		{ "log", Token::TokenLog },
-		{ "log10", Token::TokenLog10 },
-		{ "ceil", Token::TokenCeil },
-		{ "floor", Token::TokenFloor },
-		{ "mod", Token::TokenMod },
-		{ "pi", Token::TokenPi },
-		{ "exp", Token::TokenExp },
-		{ "sqr", Token::TokenSqr },
+		{ "abs", token::TokenAbs },
+		{ "sign", token::TokenSign },
+		{ "cos", token::TokenCos },
+		{ "sin", token::TokenSin },
+		{ "tan", token::TokenTan },
+		{ "acos", token::TokenACos },
+		{ "asin", token::TokenASin },
+		{ "atan", token::TokenATan },
+		{ "atan2", token::TokenATan2 },
+		{ "log", token::TokenLog },
+		{ "log10", token::TokenLog10 },
+		{ "ceil", token::TokenCeil },
+		{ "floor", token::TokenFloor },
+		{ "mod", token::TokenMod },
+		{ "pi", token::TokenPi },
+		{ "exp", token::TokenExp },
+		{ "sqr", token::TokenSqr },
 
 		// Variables
-		{ "const", Token::TokenConst },
-		{ "global", Token::TokenGlobal },
-		{ "local", Token::TokenLocal },
+		{ "const", token::TokenConst },
+		{ "global", token::TokenGlobal },
+		{ "local", token::TokenLocal },
 
 		// Includes
-		{ "include", Token::TokenInclude },
+		{ "include", token::TokenInclude },
 	};
 	for (auto v : l_textToTokenList) {
-		if (boost::iequals(text, v.first)) {
+		if (stricmp(text.c_str(), v.first)) {
 			return v.second;
 		}
 	}
 	return in;
 }
+*/

code_compiler/source/lexer.hpp

@@ -1,111 +1,60 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
+#include <filesystem>
+#include <fstream>
 #include <inttypes.h>
+#include <iostream>
 #include <istream>
 #include <memory>
 #include <string>
 #include <utility>
 
+// ToDo:
+// - Figure out a way to let the lexer output line and character information?
+
 namespace blitz {
-	class Lexer {
-		public:
-		enum class Token : uint64_t {
-			TokenUnknown,
-			TokenEOF,
-			TokenNewLine,
+	struct token {
+		uint64_t    line;
+		uint64_t    character;
 
-			// Symbols
-			TokenPlus,
-			TokenMinus,
-			TokenSlashForward,
-			TokenSlashBackward,
-			TokenMultiply,
-			TokenEqual,
-			TokenOctothorp,
-			TokenPercent,
-			TokenDollar,
-			TokenRoundBracketOpen,
-			TokenRoundBracketClose,
-			TokenSquareBracketOpen,
-			TokenSquareBracketClose,
-			TokenAngleBracketOpen,
-			TokenAngleBracketClose,
-			TokenDot,
-			TokenColon,
-			TokenComma,
-			TokenSemicolon,
-			TokenCaret,
-			TokenBitNot /*~*/,
-
-			// String Delimiter
-			TokenDoubleQuote,
-
-			// Types
-			TokenText,
-			TokenNumber,
-			TokenDecimal,
-			TokenQuotedText, // Text encapsulated by TokenDoubleQuote
-			TokenComment,
-
-			// Binary
-			TokenNot,
-			TokenAnd, TokenOr, TokenXor,
-			TokenShl, TokenShr,
-			TokenSal, TokenSar,
-			TokenFalse, TokenTrue,
-
-			// Conversion
-			TokenFloat,
-			TokenString, TokenHex,
-			TokenInt,
-
-			// Control
-			TokenIf, TokenThen, TokenElseIf, TokenElse, TokenEndIf,
-			TokenSelect, TokenCase, TokenDefault, // End Select = TokenEnd, TokenSelect.
-			TokenGoto, TokenGosub,
-			TokenReturn,
-			TokenFunction, // End Function = TokenEnd, TokenFunction.
-			TokenEnd,
-			TokenStop /* DEBUGGER! Ignore in Release mode. */,
-
-			// Loop
-			TokenFor, TokenTo, TokenNext,
-			TokenWhile, TokenWend,
-			TokenRepeat, TokenUntil, TokenForever,
-			TokenExit,
-
-			// Math
-			TokenAbs, TokenSign /*Sgn*/,
-			TokenCos, TokenSin, TokenTan,
-			TokenACos, TokenASin, TokenATan, TokenATan2,
-			TokenLog, TokenLog10,
-			TokenCeil, TokenFloor,
-			TokenMod,
-			TokenPi,
-			TokenExp, TokenSqr,
-
-			// Variables
-			TokenConst,
-			TokenGlobal,
-			TokenLocal,
-
-			// Including files.
-			TokenInclude,
-		};
-
-		public:
-		Lexer();
-		~Lexer();
-
-		std::pair<Token, std::string> GetCurrentToken();
-		std::pair<Token, std::string> GetNextToken(std::shared_ptr<std::istream> fs);
-		
-		private:
-		blitz::Lexer::Token ConvertTextToToken(Token in, std::string text);
-
-		private:
-		Token m_currentToken = Token::TokenUnknown;
-		std::string m_currentText = "";
-		Token m_overrideToken = Token::TokenUnknown;
-		std::string m_overrideText = "";
+		std::string text;
+		enum class variant : uint64_t {
+			UNKNOWN, // We have absolutely no fucking clue.
+			CONTROL, // All kinds of control signals, like NewLine, EndOfFile, ...
+			SYMBOL, // All kinds of symbols.
+			TEXT, // HelloWorld
+			NUMBER, // 1, 1%
+			DECIMAL, // 1.0, 1#
+			STRING, // "HelloWorld"
+			COMMENT, // ; Whatever
+		} type;
 	};
-}
+
+	class lexer {
+		std::filesystem::path _file;
+		std::ifstream         _stream;
+
+		// Current location in the file.
+		uint64_t _line;
+		uint64_t _character;
+
+		blitz::token _current;
+		blitz::token _override;
+
+		public:
+		~lexer();
+		lexer(std::filesystem::path file);
+
+		/** Retrieve the current token information.
+		 */
+		blitz::token current();
+
+		/** Retrieve the next token in the given stream.
+		 *
+		 * This will replace the current token.
+		 */
+		blitz::token next();
+	};
+} // namespace blitz

code_compiler/source/parser.cpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "parser.hpp"
 #include "ast/function.hpp"
 #include <iostream>
@@ -21,10 +24,10 @@ blitz::parser::~parser() {
 	}
 }
 
-std::unique_ptr<blitz::AST::Expression> blitz::parser::Parse() {
-	std::unique_ptr<AST::ScopeExpression> scope = std::make_unique<AST::ScopeExpression>();
+std::unique_ptr<blitz::ast::expression> blitz::parser::parse() {
+	std::unique_ptr<ast::ScopeExpression> scope = std::make_unique<ast::ScopeExpression>();
 
-	std::unique_ptr<AST::Expression> expr;
+	std::unique_ptr<ast::expression> expr;
 	while ((expr = std::move(parse_expression())) != nullptr) {
 		scope->AddExpression(std::move(expr));
 	}
@@ -32,7 +35,7 @@ std::unique_ptr<blitz::AST::Expression> blitz::parser::Parse() {
 	return std::move(scope);
 }
 
-void blitz::parser::LogMessage(const char* msg, ...) {
+void blitz::parser::log(const char* msg, ...) {
 	std::vector<char> buf(65535);
 	va_list val;
 	va_start(val, msg);
@@ -41,7 +44,7 @@ void blitz::parser::LogMessage(const char* msg, ...) {
 	std::cout << buf.data() << '\n';
 }
 
-void blitz::parser::LogError(const char* msg, ...) {
+void blitz::parser::log_error(const char* msg, ...) {
 	std::vector<char> buf(65535);
 	va_list val;
 	va_start(val, msg);
@@ -50,60 +53,61 @@ void blitz::parser::LogError(const char* msg, ...) {
 	std::cerr << buf.data() << '\n';
 }
 
-std::pair<blitz::Lexer::Token, std::string> blitz::parser::GetNextToken() {
-	return m_lexer.GetNextToken(m_files.top().second);
+std::pair<blitz::lexer::tokentype, std::string> blitz::parser::next() {
+	return m_lexer.next(m_files.top().second);
 }
 
-std::unique_ptr<blitz::AST::Expression> blitz::parser::parse_expression() {
+std::unique_ptr<blitz::ast::expression> blitz::parser::parse_expression() {
 	while (true) {
-		auto tkn = GetNextToken();
+		auto tkn = next();
+		log("%s", tkn.second.c_str());
 
 		switch (tkn.first) {
-			case blitz::Lexer::Token::TokenNewLine:
-			case blitz::Lexer::Token::TokenComment:
+			case blitz::lexer::tokentype::TokenNewLine:
+			case blitz::lexer::tokentype::TokenComment:
 				// Skip Comments, since we don't really need them for the AST.
 				continue;
-			case blitz::Lexer::Token::TokenPlus:
-			case blitz::Lexer::Token::TokenMinus:
+			case blitz::lexer::tokentype::TokenPlus:
+			case blitz::lexer::tokentype::TokenMinus:
 
 
 			default: // End Of File / Unknown
-			case blitz::Lexer::Token::TokenUnknown:
-			case blitz::Lexer::Token::TokenEOF:
+			case blitz::lexer::tokentype::TokenUnknown:
+			case blitz::lexer::tokentype::TokenEOF:
 				return nullptr;
 				break;
 		}
 	}
 }
 
-std::unique_ptr<blitz::AST::NumberExpression> blitz::parser::parse_number(blitz::Lexer::Token token, std::string value) {
-	if (token != Lexer::Token::TokenNumber) {
-		LogError("Unexpected Token during parsing, expected number.");
+std::unique_ptr<blitz::ast::NumberExpression> blitz::parser::parse_number(blitz::lexer::tokentype token, std::string value) {
+	if (token != lexer::tokentype::TokenNumber) {
+		log_error("Unexpected Token during parsing, expected number.");
 		return nullptr;
 	}
 
 	char* endptr = const_cast<char*>(value.c_str() + value.size());
 	int32_t parsed = strtol(value.c_str(), &endptr, 10);
 	if (errno == ERANGE) {
-		LogError("Number out of range.");
+		log_error("Number out of range.");
 		return nullptr;
 	}
 
-	return std::make_unique<blitz::AST::NumberExpression>(parsed);
+	return std::make_unique<blitz::ast::NumberExpression>(parsed);
 }
 
-std::unique_ptr<blitz::AST::DecimalExpression> blitz::parser::parse_decimal(blitz::Lexer::Token token, std::string value) {
-	if (token != Lexer::Token::TokenNumber) {
-		LogError("Unexpected Token during parsing, expected number.");
+std::unique_ptr<blitz::ast::DecimalExpression> blitz::parser::parse_decimal(blitz::lexer::tokentype token, std::string value) {
+	if (token != lexer::tokentype::TokenNumber) {
+		log_error("Unexpected Token during parsing, expected number.");
 		return nullptr;
 	}
 
 	char* endptr = const_cast<char*>(value.c_str() + value.size());
 	float_t parsed = strtof(value.c_str(), &endptr);
 	if (errno == ERANGE) {
-		LogError("Number out of range.");
+		log_error("Number out of range.");
 		return nullptr;
 	}
 
-	return std::make_unique<blitz::AST::DecimalExpression>(parsed);
+	return std::make_unique<blitz::ast::DecimalExpression>(parsed);
 }

code_compiler/source/parser.hpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "lexer.hpp"
 #include "ast/ast.hpp"
@@ -14,22 +17,22 @@ namespace blitz {
 		parser(std::string file);
 		~parser();
 
-		std::unique_ptr<AST::Expression> Parse();
+		std::unique_ptr<ast::expression> parse();
 
 		protected:
-		void LogMessage(const char* msg, ...);
-		void LogError(const char* msg, ...);
+		void log(const char* msg, ...);
+		void log_error(const char* msg, ...);
 
 		private:
-		std::pair<blitz::Lexer::Token, std::string> GetNextToken();
+		std::pair<blitz::lexer::tokentype, std::string> next();
 
 		private:
-		std::unique_ptr<AST::Expression> parse_expression();
-		std::unique_ptr<AST::NumberExpression> parse_number(blitz::Lexer::Token token, std::string value);
-		std::unique_ptr<AST::DecimalExpression> parse_decimal(blitz::Lexer::Token token, std::string value);
+		std::unique_ptr<ast::expression> parse_expression();
+		std::unique_ptr<ast::NumberExpression> parse_number(blitz::lexer::tokentype token, std::string value);
+		std::unique_ptr<ast::DecimalExpression> parse_decimal(blitz::lexer::tokentype token, std::string value);
 
 		private:
-		Lexer m_lexer;
+		lexer m_lexer;
 		std::stack<std::pair<std::string, std::shared_ptr<std::istream>>> m_files;
 
 	};

pre-commit.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# AUTOGENERATED COPYRIGHT HEADER START
+# Copyright (C) NaN-NaN undefined
+# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+# AUTOGENERATED COPYRIGHT HEADER END
+
+_self=$(realpath "$0")
+_self_dir=$(dirname "$_self")
+
+if [ "$1" == "install" ]; then
+	cp "${_self}" "${_self_dir}/.git/hooks/pre-commit"
+	echo "Installed pre-commit hook to '${_self_dir}/.git/hooks/pre-commit'."
+else
+	declare -a FILES; readarray -t FILES < <(git diff --name-only --cached)
+	node "$PWD/tools/copyright.js" "${FILES[@]}"
+fi

tests/.gitignore

@@ -0,0 +1,6 @@
+# AUTOGENERATED COPYRIGHT HEADER START
+# Copyright (C) NaN-NaN undefined
+# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+# AUTOGENERATED COPYRIGHT HEADER END
+/stranded2
+/beyond-frontier

tools/copyright.js

@@ -0,0 +1,581 @@
+// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) NaN-NaN undefined
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
+
+const ignoreList = [
+	/^\.git$/gi,
+	/^cmake\/clang$/gi,
+	/^cmake\/version$/gi,
+	/^third-party$/gi,
+]
+const sectionStart = "AUTOGENERATED COPYRIGHT HEADER START";
+const sectionEnd = "AUTOGENERATED COPYRIGHT HEADER END";
+const formatStyleList = {
+	"#": {
+		files: [
+			"cmakelists.txt"
+		], exts: [
+			".clang-tidy",
+			".clang-format",
+			".cmake",
+			".editorconfig",
+			".gitignore",
+			".gitmodules",
+			".yml",
+			".sh"
+		],
+		prepend: [
+			`# ${sectionStart}`,
+		],
+		append: [
+			`# ${sectionEnd}`,
+		],
+		prefix: "# ",
+		suffix: "",
+	},
+	";": {
+		files: [
+		], exts: [
+			".iss",
+			".iss.in",
+		],
+		prepend: [
+			`; ${sectionStart}`,
+		],
+		append: [
+			`; ${sectionEnd}`,
+		],
+		prefix: "; ",
+		suffix: "",
+	},
+	"//": {
+		files: [
+		], exts: [
+			".c",
+			".c.in",
+			".cpp",
+			".cpp.in",
+			".h",
+			".h.in",
+			".hpp",
+			".hpp.in",
+			".js",
+			".rc",
+			".rc.in",
+			".effect"
+		],
+		prepend: [
+			`// ${sectionStart}`,
+		],
+		append: [
+			`// ${sectionEnd}`,
+		],
+		prefix: "// ",
+		suffix: "",
+	},
+	"<!---->": {
+		files: [
+		], exts: [
+			".htm",
+			".htm.in",
+			".html",
+			".html.in",
+			".xml",
+			".xml.in",
+			".plist",
+			".plist.in",
+			".pkgproj",
+			".pkgproj.in",
+			".md"
+		],
+		prepend: [
+			`<!-- ${sectionStart} -->`,
+		],
+		append: [
+			`<!-- ${sectionEnd} -->`,
+		],
+		prefix: "<!-- ",
+		suffix: " -->",
+	}
+};
+const commitFormat="%aI|%aN <%aE>"
+const debug = false;
+
+// --------------------------------------------------------------------------------
+// End of easily modified area, best not touch things beyond here.
+// --------------------------------------------------------------------------------
+const CHILD_PROCESS = require("node:child_process");
+const PROCESS = require("node:process");
+const PATH = require("node:path");
+const FS = require("node:fs");
+const FSPROMISES = require("node:fs/promises");
+const OS = require("os");
+
+if (!debug)
+	console.debug = function() {}
+
+class RateLimiter {
+    constructor(limit) {
+        Object.defineProperty(this, "_maximum", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "_available", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "_instances", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        if (!limit) {
+            this._maximum = Math.ceil(Math.max(1, OS.cpus().length / 3 * 2));
+        }
+        else {
+            this._maximum = limit;
+        }
+        this._available = this._maximum;
+        this._instances = [];
+    }
+    async queue(executor, ...args) {
+        while (this._available == 0) {
+            await Promise.race(this._instances);
+        }
+        --this._available;
+        const data = {};
+        data.task = new Promise((resolve, reject) => {
+            try {
+				if (executor.constructor.name == "Promise") {
+					executor.then((res) => {
+						resolve(res);
+					}, (err) => {
+						reject(err);
+					});
+                } else if (executor.constructor.name == "AsyncFunction") {
+                    executor(...args).then((res) => {
+                        resolve(res);
+                    }, (err) => {
+                        reject(err);
+                    });
+                }
+                else {
+                    resolve(executor(...args));
+                }
+            }
+            catch (ex) {
+				console.error(executor, executor.toString())
+                reject(ex);
+            }
+        });
+        data.solver = data.task.finally(() => {
+            const taskIndex = this._instances.indexOf(data.task);
+            if (taskIndex >= 0) {
+                this._instances.splice(taskIndex, 1);
+            }
+            const solverIndex = this._instances.indexOf(data.solver);
+            if (solverIndex >= 0) {
+                this._instances.splice(solverIndex, 1);
+            }
+            ++this._available;
+        });
+        this._instances.push(data.solver);
+        return await data.solver;
+    }
+}
+
+let abortAllWork = false;
+let gitRL = new RateLimiter(1);
+let workRL = new RateLimiter();
+let gitCurrentFiles;
+let gitUserName;
+let gitUserMail;
+let gitDate = (new Date()).toISOString();
+
+/** Run a process asynchronously, returning an array of messages.
+ * 
+ * @param {*} path 
+ * @param {*} args 
+ * @param {*} options 
+ * @returns [ErrorCode, [{type, data, text}, ...]]
+ */
+async function runProcessAsync(path, args, options) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	return await new Promise((resolve, reject) => {
+		let messages = [];
+
+		try {
+			let proc = CHILD_PROCESS.spawn(path, args, Object.assign({
+				"cwd": PROCESS.cwd(),
+				"encoding": "utf8",
+			}, options));
+
+			let pushOutput = function(type, chunk) {
+				if (!chunk) return;
+
+				let lastMessage = undefined;
+				if (messages.length == 0 || messages[messages.length - 1].type !== type) {
+					lastMessage = {
+						type: type,
+						chunks: [],
+						length: 0,
+					};
+					messages.push(lastMessage);
+				} else {
+					lastMessage = messages[messages.length - 1];
+				}
+				lastMessage.length += chunk.length;
+				lastMessage.chunks.push(chunk);
+			};
+
+			proc.stderr.on('data', (chunk) => { pushOutput("error", chunk); });
+			proc.stderr.on('close', () => { pushOutput("output", proc.stderr.read()); });
+			proc.stdout.on('data', (chunk) => { pushOutput("output", chunk); });
+			proc.stdout.on('close', () => { pushOutput("output", proc.stdout.read()); });
+			proc.on('error', (err) => {
+				reject(err);
+			})
+			proc.on('exit', (code) => {
+				// Merge chunks of messages into a single message.
+				for (let message of messages) {
+					message.data = Buffer.alloc(message.length);
+					let offset = 0;
+					for (let chunk of message.chunks) {
+						Buffer.from(chunk).copy(message.data, offset, 0);
+						offset += chunk.byteLength;
+					}
+					message.chunks = undefined;
+					message.length = undefined;
+					message.text = message.data.toString();
+				}
+				resolve([code, messages]);
+			});
+		} catch (ex) {
+			reject(ex);
+		}
+	});
+}
+
+async function git_isIgnored(path) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	let rpath = PATH.relative(process.cwd(), path).replaceAll(PATH.sep, PATH.posix.sep);
+	for (let ignore of ignoreList) {
+		if (ignore instanceof RegExp) {
+			if (ignore.global) {
+				let matches = rpath.matchAll(ignore);
+				for (let match of matches) {
+					return true;
+				}
+			} else {
+				if (rpath.match(ignore) !== null) {
+					return true;
+				}
+			}
+		} else if (rpath.startsWith(ignore)) {
+			return true;
+		}
+	}
+
+	let result = await gitRL.queue(runProcessAsync, "git", ["check-ignore", path], {});
+	return (result[0] == 0)
+}
+
+async function git_getCurrentAuthor() {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	if (!gitUserName) {
+		let result = await gitRL.queue(runProcessAsync, "git", ["config", "user.name"], {});
+		gitUserName = result[1][0].text.split('\n')[0];
+	}
+	if (!gitUserMail) {
+		let result = await gitRL.queue(runProcessAsync, "git", ["config", "user.email"], {});
+		gitUserMail = result[1][0].text.split('\n')[0];
+	}
+	return commitFormat.replace("%aI", gitDate).replace("%aN", gitUserName).replace("%aE", gitUserMail);
+}
+
+async function git_isInCurrentCommit(file) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	if (!gitCurrentFiles) {
+		let files = [];
+		let result = await gitRL.queue(runProcessAsync, "git", ["diff", "--name-only", "--cached"], {});
+		if (result[0] != 0) {
+			let output = "";
+			for (let message of result[1]) {
+				output += message.text;
+			}
+			throw new Error(output);
+		} else {
+			lines = "";
+			for (let message of result[1]) {
+				if (message.type == "output")
+					lines += message.text;
+			}
+			files = lines.split(lines.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
+		}
+
+		gitCurrentFiles = new Set(files);
+		for (let file of files) {
+			if (file.length == 0) continue;
+			file = PATH.relative(PROCESS.cwd(), PATH.resolve(file));
+			gitCurrentFiles.add(file);
+		}
+	}
+
+	// Normalize file given to us.
+	file = PATH.relative(PROCESS.cwd(), PATH.resolve(file));
+
+	// Check and return if it's in here.
+	return gitCurrentFiles.has(file);
+}
+
+async function git_retrieveAuthors(file) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	// git --no-pager log --date-order --reverse "--format=format:%aI|%aN <%aE>" -- file
+	let lines = [];
+	let result = await gitRL.queue(runProcessAsync, "git", ["--no-pager", "log", "--follow", `--format=format:${commitFormat}`, "--",	file], {});
+	if (result[0] != 0) {
+		let output = "";
+		for (let message of result[1]) {
+			output += message.text;
+		}
+		throw new Error(output);
+	} else {
+		let buffered = "";
+		for (let message of result[1]) {
+			if (message.type == "output")
+				buffered += message.text;
+		}
+		lines = buffered.split(buffered.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
+	}
+
+	if (await git_isInCurrentCommit(file)) {
+		lines.push(await git_getCurrentAuthor());
+	}
+
+	let authors = new Map();
+	for (let line of lines) {
+		let [date, name] = line.split("|");
+
+		let author = authors.get(name);
+		if (author) {
+			let dt = new Date(date)
+			if (author.from > dt)
+				author.from = dt;
+			if (author.to < dt)
+				author.to = dt;
+		} else {
+			authors.set(name, {
+				from: new Date(date),
+				to: new Date(date),
+			})
+		}
+	}
+
+	return authors;
+}
+
+async function generateCopyright(file) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	let authors = await git_retrieveAuthors(file)
+	authors = new Map([...authors].sort((a, b) => {
+		if (a[1].from < b[1].from) {
+			return -1;
+		} else if (a[1].from > b[1].from) {
+			return 1;
+		}
+		return 0;
+	}));
+	let lines = [];
+	for (let entry of authors) {
+		let from = entry[1].from.getUTCFullYear();
+		let to = entry[1].to.getUTCFullYear();
+		lines.push(`Copyright (C) ${from != to ? `${from}-${to}` : to} ${entry[0]}`);
+	}
+	return lines;
+}
+
+function makeHeader(file, copyright) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	let file_name = PATH.basename(file).toLocaleLowerCase();
+	let file_exts = file_name.substring(file_name.indexOf("."));
+
+	let styles = formatStyleList;
+
+	for (let key in styles) {
+		let style = [key, styles[key]];
+		if (style[1].files.includes(file_name)
+			|| style[1].files.includes(file)
+			|| style[1].exts.includes(file_exts)) {
+			let header = [];
+			header.push(...style[1].prepend);
+			for (let line of copyright) {
+				header.push(`${style[1].prefix}${line}${style[1].suffix}`);
+			}
+			header.push(...style[1].append);
+			return header;
+		}
+	}
+
+	throw new Error("Unrecognized file format.")
+}
+
+async function updateFile(file) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	await workRL.queue(async () => {
+		try {
+			if (abortAllWork) {
+				return;
+			}
+
+			// Copyright information.
+			let copyright = await generateCopyright(file);
+			let header = undefined;
+			try {
+				header = makeHeader(file, copyright);
+			} catch (ex) {
+				console.log(`Skipping file '${file}': ${ex.message}`);
+				return;
+			}
+			console.log(`Updating file '${file}'...`);
+
+			// File contents.
+			let content = await FSPROMISES.readFile(file);
+			let eol = (content.indexOf("\r\n") != -1 ? OS.EOL : "\n");
+			let insert = Buffer.from(header.join(eol) + eol);
+
+			// Find the starting point.
+			let startHeader = content.indexOf(sectionStart);
+			startHeader = content.lastIndexOf(eol, startHeader);
+			startHeader += Buffer.from(eol).byteLength;
+
+			// Find the ending point.
+			let endHeader = content.indexOf(sectionEnd);
+			endHeader = content.indexOf(eol, endHeader);
+			endHeader += Buffer.from(eol).byteLength;
+
+			if (abortAllWork) {
+				return;
+			}
+
+			let fd = await FSPROMISES.open(file, "w");
+			let fp = [];
+			if ((startHeader >= 0) && (endHeader > startHeader)) {
+				let pos = 0;
+				if (startHeader > 0) {
+					fd.write(content, 0, startHeader, 0);
+					pos += startHeader;
+				}
+				fd.write(insert, 0, undefined, pos);
+				pos += insert.byteLength;
+				fd.write(content, endHeader, undefined, pos);
+			} else {
+				fd.write(insert, 0, undefined, 0);
+				fd.write(content, 0, undefined, insert.byteLength);
+			}
+			await fd.close();
+		} catch (ex) {
+			console.error(`Error processing '${file}'!: ${ex}`);
+			abortAllWork = true;
+			PROCESS.exitCode = 1;
+			return;
+		}
+	});
+}
+
+async function scanPath(path) {
+	console.debug(arguments.callee.name, Array.from(arguments));
+	// Abort here if the user aborted the process, or if the path is ignored.
+	if (abortAllWork) {
+		return;
+	}
+
+	let promises = [];
+
+	await workRL.queue(async () => {
+		let files = await FSPROMISES.readdir(path, { "withFileTypes": true });
+		for (let file of files) {
+			if (abortAllWork) {
+				break;
+			}
+
+			let fullname = PATH.join(path, file.name);
+			if (await git_isIgnored(fullname)) {
+				console.log(`Ignoring path '${fullname}'...`);
+				continue;
+			}
+
+			if (file.isDirectory()) {
+				console.log(`Scanning path '${fullname}'...`);
+				promises.push(scanPath(fullname));
+			} else {
+				promises.push(updateFile(fullname));
+			}
+		}
+	});
+
+	await Promise.all(promises);
+}
+
+(async function () {
+	PROCESS.on("SIGINT", () => {
+		abortAllWork = true;
+		PROCESS.exitCode = 1;
+		console.log("Sanely aborting all pending work...");
+	})
+
+	const root_path = PROCESS.cwd();
+	let is_git_directory = false;
+	
+	console.debug(root_path, PROCESS.argv, PROCESS.execArgv);
+
+	var args = PROCESS.argv.slice(2);
+	while (args.length > 0) {
+		// Try to place ourselves where git actually is.
+		while (!is_git_directory) {
+			if (abortAllWork) {
+				return;
+			}
+
+			let entries = await FSPROMISES.readdir(PROCESS.cwd());
+			if (entries.includes(".git")) {
+				console.log(`Found .git at '${process.cwd()}'.`);
+				is_git_directory = true;
+			} else {
+				PROCESS.chdir(PATH.resolve(PATH.join(PROCESS.cwd(), "..")));
+			}
+		}
+
+		// If that failed, we just exit now.
+		if (!is_git_directory) {
+			console.error("Failed to figure out where git is, is this even a repository?");
+			PROCESS.exitCode = 1;
+			return;
+		}
+
+		// Then proceed with normal work.
+		let path = PATH.normalize(PATH.relative(process.cwd(), PATH.resolve(root_path, args[0])));
+
+		// If we're handed a path, scan said path, otherwise update the file itself.
+		let pathStat = await FSPROMISES.stat(path);
+		if (pathStat) {
+			if (await git_isIgnored(path)) {
+				console.log(`Ignoring path '${path}'...`);
+			} else if(pathStat.isDirectory()) {
+				console.log(`Scanning path '${path}'...`);
+				await scanPath(path);
+			} else {
+				await updateFile(path);
+			}
+		}
+
+		// Slice off the first argument and continue.
+		args = args.slice(1);
+	}
+	console.log("Done");
+})();