Latest stuff, rewriting lexer

code_compiler/source/ast/arithmetic.cpp

@@ -1,6 +1,9 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "arithmetic.hpp"
 
-blitz::AST::ArithmeticExpression::ArithmeticExpression(Operator op, std::unique_ptr<Expression> left, std::unique_ptr<Expression> right)
+blitz::ast::arithmetic_expression::arithmetic_expression(expression_operator op, std::unique_ptr<expression> left, std::unique_ptr<expression> right)
 	: m_operator(op), m_left(std::move(left)), m_right(std::move(right)) {}
 
-blitz::AST::ArithmeticExpression::~ArithmeticExpression() {}
+blitz::ast::arithmetic_expression::~arithmetic_expression() {}

code_compiler/source/ast/arithmetic.hpp

@@ -1,10 +1,13 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "ast.hpp"
 #include "value.hpp"
 
 namespace blitz {
-	namespace AST {
-		enum class Operator : int8_t {
+	namespace ast {
+		enum class expression_operator : int8_t {
 			Add, /*+*/
 			Subtract, /*-*/
 			Multiply, /***/
@@ -14,14 +17,14 @@ namespace blitz {
 			Equal, /*=*/
 		};
 
-		class ArithmeticExpression : public Expression {
+		class arithmetic_expression : public expression {
 			public:
-			ArithmeticExpression(Operator op, std::unique_ptr<Expression> left, std::unique_ptr<Expression> right);
-			virtual ~ArithmeticExpression();
+			arithmetic_expression(expression_operator op, std::unique_ptr<expression> left, std::unique_ptr<expression> right);
+			virtual ~arithmetic_expression();
 
 			private:
-			Operator m_operator;
-			std::unique_ptr<Expression> m_left, m_right;
+			expression_operator m_operator;
+			std::unique_ptr<expression> m_left, m_right;
 		};
 	}
 }

code_compiler/source/ast/ast.hpp

@@ -1,10 +1,13 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 
 namespace blitz {
-	namespace AST {
-		class Expression {
+	namespace ast {
+		class expression {
 			public:
-			virtual ~Expression() {};
+			virtual ~expression() {};
 		};
 	}
 }

code_compiler/source/ast/function.cpp

@@ -1,14 +1,17 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "function.hpp"
 
-blitz::AST::ScopeExpression::ScopeExpression() {}
+blitz::ast::ScopeExpression::ScopeExpression() {}
 
-blitz::AST::ScopeExpression::~ScopeExpression() {}
+blitz::ast::ScopeExpression::~ScopeExpression() {}
 
-void blitz::AST::ScopeExpression::AddExpression(std::unique_ptr<Expression> ex) {
+void blitz::ast::ScopeExpression::AddExpression(std::unique_ptr<expression> ex) {
 	m_expressions.push_back(std::move(ex));
 }
 
-blitz::AST::FunctionExpression::FunctionExpression(ValueType returnType,
+blitz::ast::FunctionExpression::FunctionExpression(ValueType returnType,
 	std::string& m_name,
 	std::list<std::unique_ptr<VariableExpression>> parameters, 
 	std::unique_ptr<ScopeExpression> scope)
@@ -16,11 +19,11 @@ blitz::AST::FunctionExpression::FunctionExpression(ValueType returnType,
 
 }
 
-blitz::AST::FunctionExpression::~FunctionExpression() {}
+blitz::ast::FunctionExpression::~FunctionExpression() {}
 
-blitz::AST::CallExpression::CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments)
+blitz::ast::CallExpression::CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments)
 	: m_name(name), m_arguments(std::move(arguments)) {
 
 }
 
-blitz::AST::CallExpression::~CallExpression() {}
+blitz::ast::CallExpression::~CallExpression() {}

code_compiler/source/ast/function.hpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "ast.hpp"
 #include "value.hpp"
@@ -6,16 +9,16 @@
 #include <string>
 
 namespace blitz {
-	namespace AST {
-		class ScopeExpression : public Expression {
+	namespace ast {
+		class ScopeExpression : public expression {
 			public:
 			ScopeExpression();
 			virtual ~ScopeExpression();
 
-			void AddExpression(std::unique_ptr<Expression> ex);
+			void AddExpression(std::unique_ptr<expression> ex);
 
 			private:
-			std::list<std::unique_ptr<Expression>> m_expressions;
+			std::list<std::unique_ptr<expression>> m_expressions;
 		};
 
 		class FunctionExpression : public ScopeExpression {
@@ -33,7 +36,7 @@ namespace blitz {
 			std::unique_ptr<ScopeExpression> m_content;
 		};
 
-		class CallExpression : public Expression {
+		class CallExpression : public expression {
 			public:
 			CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments);
 			virtual ~CallExpression();

code_compiler/source/ast/value.cpp

@@ -1,43 +1,46 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "value.hpp"
 
-blitz::AST::VariableExpression::VariableExpression(std::string& name, ValueType type /*= ValueType::Number*/)
+blitz::ast::VariableExpression::VariableExpression(std::string& name, ValueType type /*= ValueType::Number*/)
 	: m_name(name), m_type(type) {}
 
-blitz::AST::VariableExpression::~VariableExpression() {}
+blitz::ast::VariableExpression::~VariableExpression() {}
 
-blitz::AST::ValueType blitz::AST::VariableExpression::GetType() {
+blitz::ast::ValueType blitz::ast::VariableExpression::GetType() {
 	return m_type;
 }
 
-blitz::AST::NumberExpression::NumberExpression(int32_t value) : value(value) {}
+blitz::ast::NumberExpression::NumberExpression(int32_t value) : value(value) {}
 
-blitz::AST::NumberExpression::~NumberExpression() {}
+blitz::ast::NumberExpression::~NumberExpression() {}
 
-blitz::AST::ValueType blitz::AST::NumberExpression::GetType() {
+blitz::ast::ValueType blitz::ast::NumberExpression::GetType() {
 	return ValueType::Number;
 }
 
-blitz::AST::DecimalExpression::DecimalExpression(float_t value) : value(value) {}
+blitz::ast::DecimalExpression::DecimalExpression(float_t value) : value(value) {}
 
-blitz::AST::DecimalExpression::~DecimalExpression() {}
+blitz::ast::DecimalExpression::~DecimalExpression() {}
 
-blitz::AST::ValueType blitz::AST::DecimalExpression::GetType() {
+blitz::ast::ValueType blitz::ast::DecimalExpression::GetType() {
 	return ValueType::Decimal;
 }
 
-blitz::AST::StringExpression::StringExpression(std::string value) : value(value) {}
+blitz::ast::StringExpression::StringExpression(std::string value) : value(value) {}
 
-blitz::AST::StringExpression::~StringExpression() {}
+blitz::ast::StringExpression::~StringExpression() {}
 
-blitz::AST::ValueType blitz::AST::StringExpression::GetType() {
+blitz::ast::ValueType blitz::ast::StringExpression::GetType() {
 	return ValueType::String;
 }
 
-blitz::AST::ConstExpression::ConstExpression(std::string& name, std::unique_ptr<ValueExpression> value)
+blitz::ast::ConstExpression::ConstExpression(std::string& name, std::unique_ptr<ValueExpression> value)
 	: m_name(name), m_value(std::move(value)) {}
 
-blitz::AST::ConstExpression::~ConstExpression() {}
+blitz::ast::ConstExpression::~ConstExpression() {}
 
-blitz::AST::ValueType blitz::AST::ConstExpression::GetType() {
+blitz::ast::ValueType blitz::ast::ConstExpression::GetType() {
 	return m_value->GetType();
 }

code_compiler/source/ast/value.hpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "ast.hpp"
 #include "lexer.hpp"
@@ -8,7 +11,7 @@
 #include <string>
 
 namespace blitz {
-	namespace AST {
+	namespace ast {
 		enum class ValueType : int8_t {
 			Unknown,
 			Number,
@@ -17,7 +20,7 @@ namespace blitz {
 			Type,
 		};
 
-		class ValueExpression : public Expression {
+		class ValueExpression : public expression {
 			public:
 			virtual ValueType GetType() = 0;
 		};

code_compiler/source/compiler.cpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "compiler.hpp"
 #include "parser.hpp"
 #include "lexer.hpp"
@@ -7,19 +10,3 @@
 blitz::compiler::compiler() {}
 
 blitz::compiler::~compiler() {}
-
-bool blitz::compiler::compile(std::string in, std::string out) {
-	/*std::ifstream infile;
-	infile.open(in);
-	if (infile.bad() || !infile.good() || infile.eof()) {
-		std::cerr << "Failed to open file: " << in << std::endl;
-		return false;
-	}*/
-
-	parser psr = parser(in);
-	if (!psr.Parse()) {
-
-	}
-
-	return true;
-}

code_compiler/source/compiler.hpp

@@ -1,12 +1,22 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include <string>
+#include <filesystem>
+
+// Compiling requires several steps
+// 1. Lexing to known tokens
+//   - This part may require a unicode library/framework, like Qt or ICU
+// 2. Parsing the lexical tokens into an Abstract Syntax Tree (AST)
+//   - This is where we convert everything to logical steps, like math and function calls.
+// 3. Compiling from the AST to some kind of runnable binary.
+//   - In our case, we convert the AST to LLVM IR, and then compile it with LLVM.
 
 namespace blitz {
 	class compiler {
 		public:
 		compiler();
 		~compiler();
-
-		bool compile(std::string in, std::string out);
 	};
 }

code_compiler/source/error.cpp

@@ -0,0 +1,4 @@
+// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) NaN-NaN undefined
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END

code_compiler/source/error.hpp

@@ -0,0 +1,4 @@
+// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) NaN-NaN undefined
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END

code_compiler/source/lexer.cpp

@@ -1,49 +1,170 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "lexer.hpp"
 #include <codecvt>
+#include <sstream>
 
-std::pair<char, blitz::Lexer::Token> g_symbolCharacters[] = {
+/*std::pair<char, blitz::tokentype> g_symbolCharacters[] = {
 	//{ '\"', BlitzLLVM::Lexer::Token::TokenDoubleQuote }, // Has special meaning.
-	{ '+', blitz::Lexer::Token::TokenPlus },
-	{ '-', blitz::Lexer::Token::TokenMinus },
-	{ '/', blitz::Lexer::Token::TokenSlashForward },
-	{ '\\', blitz::Lexer::Token::TokenSlashBackward },
-	{ '*', blitz::Lexer::Token::TokenMultiply },
-	{ '=', blitz::Lexer::Token::TokenEqual },
-	{ '#', blitz::Lexer::Token::TokenOctothorp },
-	{ '%', blitz::Lexer::Token::TokenPercent },
-	{ '$', blitz::Lexer::Token::TokenDollar },
-	{ '(', blitz::Lexer::Token::TokenRoundBracketOpen },
-	{ ')', blitz::Lexer::Token::TokenRoundBracketClose },
-	{ '[', blitz::Lexer::Token::TokenSquareBracketOpen },
-	{ ']', blitz::Lexer::Token::TokenSquareBracketClose },
-	{ '<', blitz::Lexer::Token::TokenAngleBracketOpen },
-	{ '>', blitz::Lexer::Token::TokenAngleBracketClose },
-	//{ '.', BlitzLLVM::Lexer::Token::TokenDot }, // Special meaning.
-	{ ':', blitz::Lexer::Token::TokenColon },
-	{ ',', blitz::Lexer::Token::TokenComma },
-	//{ ';', BlitzLLVM::Lexer::Token::TokenSemicolon },
-	{ '^', blitz::Lexer::Token::TokenCaret },
-	{ '~', blitz::Lexer::Token::TokenBitNot },
-};
+	{ '+', blitz::tokentype::TokenPlus },
+	{ '-', blitz::tokentype::TokenMinus },
+	{ '/', blitz:::tokentype::TokenSlashForward },
+	{ '\\', blitz::tokentype::TokenSlashBackward },
+	{ '*', blitz::tokentype::TokenMultiply },
+	{ '=', blitz::tokentype::TokenEqual },
+	{ '#', blitz::tokentype::TokenOctothorp },
+	{ '%', blitz::tokentype::TokenPercent },
+	{ '$', blitz::tokentype::TokenDollar },
+	{ '(', blitz::tokentype::TokenRoundBracketOpen },
+	{ ')', blitz::tokentype::TokenRoundBracketClose },
+	{ '[', blitz::tokentype::TokenSquareBracketOpen },
+	{ ']', blitz::tokentype::TokenSquareBracketClose },
+	{ '<', blitz::tokentype::TokenAngleBracketOpen },
+	{ '>', blitz::tokentype::TokenAngleBracketClose },
+	//{ '.', BlitzLLVM::Token::TokenDot }, // Special meaning.
+	{ ':', blitz::tokentype::TokenColon },
+	{ ',', blitz::tokentype::TokenComma },
+	//{ ';', BlitzLLVM::Token::TokenSemicolon },
+	{ '^', blitz::tokentype::TokenCaret },
+	{ '~', blitz::tokentype::TokenBitNot },
+};*/
 
-blitz::Lexer::Lexer() {}
+blitz::lexer::~lexer() {}
 
-blitz::Lexer::~Lexer() {}
+blitz::lexer::lexer(std::filesystem::path file)
+{
+	// Usually files start at line and character 0, so we should start there too.
+	_line = _character = 0;
 
-std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetCurrentToken() {
-	return std::make_pair(m_currentToken, m_currentText);
+	// Try and open the file for reading.
+	_file   = file;
+	_stream = std::ifstream(_file, std::ios_base::binary); // We use binary so we can eventually support UTF-8.
+	if (!_stream.good() || _stream.eof() || _stream.bad() || _stream.fail()) {
+		char buffer[16384];
+		int  len = snprintf(buffer, sizeof(buffer), "Reading file '%s' failed.\0", file.generic_string().c_str());
+		throw std::runtime_error(std::string(buffer, buffer + len));
+	}
+
+	// Initialize token storage to a default token.
+	_override = _current = blitz::token{
+		.line      = 0,
+		.character = 0,
+		.text      = "",
+		.type      = token::variant::UNKNOWN,
+	};
 }
 
-std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shared_ptr<std::istream> fs) {
+blitz::token blitz::lexer::current()
+{
+	return _current;
+}
+
+blitz::token blitz::lexer::next()
+{
+	enum class stage {
+		DEFAULT,
+		TEXT,
+		NUMBER,
+		STRING,
+		COMMENT,
+	} state = stage::DEFAULT;
+	bool numberHasDot = false;
+
+	std::stringstream buffer;
+	blitz::token      token{
+			 .line      = _line,
+			 .character = _character,
+			 .text      = "",
+			 .type      = blitz::token::variant::UNKNOWN,
+    };
+
+	// Helper function to advance text.
+	auto advance = [this]() {
+		_character++;
+		return _stream.get();
+	};
+
+	while ((token.type == blitz::token::variant::UNKNOWN) && _stream.good() && !_stream.eof()) {
+		auto chr = advance();
+
+		if (state == stage::DEFAULT) {
+			if (chr == ';') { // We've encountered a comment, so we should change state and ignore this symbol.
+				state           = stage::COMMENT;
+				token.line      = _line;
+				token.character = _character;
+				token.type      = blitz::token::variant::COMMENT;
+			} else {
+				buffer << chr;
+			}
+		} else if (state == stage::NUMBER) {
+			if (isdigit(chr)) {
+				buffer << chr;
+			} else if (chr == '.')
+			{
+				if (numberHasDot) {
+					throw std::runtime_error("")
+				}
+				numberHasDot = true;
+			} else {
+
+			}
+
+		} else if (state == stage::TEXT) {
+		} else if (state == stage::STRING) {
+		} else if (state == stage::COMMENT) {
+			if (chr == '\r' && _stream.peek() == '\n') {
+				token.text = buffer.str();
+			} else {
+				buffer << chr;
+			}
+		}
+	}
+
+	_current = token;
+	return _current;
+}
+
+/*
+std::pair<blitz::tokentype, std::string> blitz::lexer::current() {
+	return _current;
+}
+
+std::pair<blitz::tokentype, std::string> blitz::lexer::next(std::istream& fs) {
+	std::stringstream buffer;
+	blitz::tokentype token;
+
+	enum class parserState {
+		DEFAULT,
+		TEXT,
+		NUMBER,
+		STRING,
+		COMMENT,
+	} state = parserState::DEFAULT;
+
+	while ((token == blitz::tokentype::TokenUnknown) && !fs.eof() && fs.good()) {
+		auto chr = fs.get();
+
+
+
+
+
+	}
+
+}
+
+
+/*
+std::pair<blitz::lexer::token, std::string> blitz::lexer::next(std::shared_ptr<std::istream> fs) {
 	std::string buf;
-	Token tkn = Token::TokenEOF;
+	token tkn = token::TokenUnknown;
 	bool haveResult = false;
 
 	// Allow "overriding" the next retrieved Token.
-	if (m_overrideToken != Token::TokenUnknown) {
+	if (m_overrideToken != token::TokenUnknown) {
 		buf = m_overrideText;
 		tkn = m_overrideToken;
-		m_overrideToken = Token::TokenUnknown;
+		m_overrideToken = token::TokenUnknown;
 		haveResult = true;
 	}
 
@@ -56,11 +177,11 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 		char chr = fs->get();
 
 		if (chr == '\r' || chr == '\n') {
-			if (tkn != Token::TokenEOF) {
-				m_overrideToken = Token::TokenNewLine;
+			if (tkn != token::TokenEOF) {
+				m_overrideToken = token::TokenNewLine;
 				m_overrideText = "";
 			} else {
-				tkn = Token::TokenNewLine;
+				tkn = token::TokenNewLine;
 				buf = "";
 			}
 
@@ -71,10 +192,10 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			break;
 		} else if (m_isStringMode) {
 			if (chr == '\"') {
-				m_overrideToken = Token::TokenDoubleQuote;
+				m_overrideToken = token::TokenDoubleQuote;
 				m_overrideText = chr;
 				m_isStringMode = false;
-				tkn = Token::TokenQuotedText;
+				tkn = token::TokenQuotedText;
 				break;
 			} else if (iscntrl(chr) || !isprint(chr)) {
 				fs->putback(chr);
@@ -97,7 +218,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			} else if (chr == '.') {
 				if (m_numberModeHasDecimal == false) {
 					m_numberModeHasDecimal = true;
-					tkn = Token::TokenDecimal;
+					tkn = token::TokenDecimal;
 					buf += chr;
 				} else {
 					fs->putback(chr);
@@ -111,7 +232,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			}
 		} else if (m_isCommentMode) {
 			buf += chr;
-			tkn = Token::TokenComment;
+			tkn = token::TokenComment;
 		} else {
 			// Whitespace
 			if (isspace(chr))
@@ -119,7 +240,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 
 			// Control Code
 			if (iscntrl(chr)) {
-				tkn = Token::TokenUnknown;
+				tkn = token::TokenUnknown;
 				buf = chr;
 			}
 
@@ -129,13 +250,13 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 				if (isdigit(chr2)) {
 					m_isNumberMode = true;
 					m_numberModeHasDecimal = false;
-					tkn = Token::TokenNumber;
+					tkn = token::TokenNumber;
 					buf = chr + chr2;
 					break;
 				} else if (chr2 == '.') {
 					m_isNumberMode = true;
 					m_numberModeHasDecimal = true;
-					tkn = Token::TokenDecimal;
+					tkn = token::TokenDecimal;
 					buf = chr + "0" + chr2;
 					break;
 				} else {
@@ -151,7 +272,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 					break;
 				}
 			}
-			if (tkn != Token::TokenEOF) {
+			if (tkn != token::TokenEOF) {
 				haveResult = true;
 				break;
 			}
@@ -159,30 +280,30 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 			// Strings, Text, Numbers
 			if (chr == ';') {
 				m_isCommentMode = true;
-				tkn = Token::TokenSemicolon;
+				tkn = token::TokenSemicolon;
 				buf = chr;
 				break;
 			} else if (chr == '\"') {
 				m_isStringMode = true;
-				tkn = Token::TokenDoubleQuote;
+				tkn = token::TokenDoubleQuote;
 				buf = chr;
 				break;
 			} else if (isalpha(chr)) {
 				m_isTextMode = true;
-				tkn = Token::TokenText;
+				tkn = token::TokenText;
 				buf = chr;
 			} else if (isdigit(chr)) {
 				m_isNumberMode = true;
 				m_numberModeHasDecimal = false;
-				tkn = Token::TokenNumber;
+				tkn = token::TokenNumber;
 				buf = chr;
 			} else if (chr == '.') {
 				m_isNumberMode = true;
 				m_numberModeHasDecimal = true;
-				tkn = Token::TokenDecimal;
+				tkn = token::TokenDecimal;
 				buf = "0" + chr;
 			} else {
-				tkn = Token::TokenUnknown;
+				tkn = token::TokenUnknown;
 				buf = chr;
 				break;
 			}
@@ -190,90 +311,91 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
 	}
 
 	// Convert from Text into native Token.
-	if (tkn == Token::TokenText)
-		tkn = ConvertTextToToken(tkn, buf);
+	if (tkn == token::TokenText)
+		tkn = to_token(tkn, buf);
 
 	return std::make_pair(tkn, buf);
 }
 
-blitz::Lexer::Token blitz::Lexer::ConvertTextToToken(Token in, std::string text) {
-	static std::pair<const char*, Token> l_textToTokenList[] = {
+blitz::lexer::token blitz::lexer::to_token(token in, std::string text) {
+	static std::pair<const char*, token> l_textToTokenList[] = {
 		// Binary
-		{ "not", Token::TokenNot },
-		{ "and", Token::TokenAnd },
-		{ "or", Token::TokenOr },
-		{ "xor", Token::TokenXor },
-		{ "shl", Token::TokenShl },
-		{ "shr", Token::TokenShr },
-		{ "sal", Token::TokenSal },
-		{ "sar", Token::TokenSar },
-		{ "false", Token::TokenFalse },
-		{ "true", Token::TokenTrue },
+		{ "not", token::TokenNot },
+		{ "and", token::TokenAnd },
+		{ "or", token::TokenOr },
+		{ "xor", token::TokenXor },
+		{ "shl", token::TokenShl },
+		{ "shr", token::TokenShr },
+		{ "sal", token::TokenSal },
+		{ "sar", token::TokenSar },
+		{ "false", token::TokenFalse },
+		{ "true", token::TokenTrue },
 
 		// Conversion
-		{ "float", Token::TokenFloat },
-		{ "string", Token::TokenString },
-		{ "hex", Token::TokenHex },
-		{ "int", Token::TokenInt },
+		{ "float", token::TokenFloat },
+		{ "string", token::TokenString },
+		{ "hex", token::TokenHex },
+		{ "int", token::TokenInt },
 
 		// Control
-		{ "if", Token::TokenIf },
-		{ "then", Token::TokenThen },
-		{ "elseIf", Token::TokenElseIf },
-		{ "else", Token::TokenElse },
-		{ "endIf", Token::TokenEndIf },
-		{ "select", Token::TokenSelect },
-		{ "case", Token::TokenCase },
-		{ "default", Token::TokenDefault },
-		{ "goto", Token::TokenGoto },
-		{ "gosub", Token::TokenGosub },
-		{ "return", Token::TokenReturn },
-		{ "function", Token::TokenFunction },
-		{ "end", Token::TokenEnd },
-		{ "stop", Token::TokenStop },
-		
+		{ "if", token::TokenIf },
+		{ "then", token::TokenThen },
+		{ "elseif", token::TokenElseIf },
+		{ "else", token::TokenElse },
+		{ "endif", token::TokenEndIf },
+		{ "select", token::TokenSelect },
+		{ "case", token::TokenCase },
+		{ "default", token::TokenDefault },
+		{ "goto", token::TokenGoto },
+		{ "gosub", token::TokenGosub },
+		{ "return", token::TokenReturn },
+		{ "function", token::TokenFunction },
+		{ "end", token::TokenEnd },
+		{ "stop", token::TokenStop },
+
 		// Loop
-		{ "for", Token::TokenFor },
-		{ "to", Token::TokenTo },
-		{ "next", Token::TokenNext },
-		{ "while", Token::TokenWhile },
-		{ "wend", Token::TokenWend },
-		{ "repeat", Token::TokenRepeat },
-		{ "until", Token::TokenUntil },
-		{ "forever", Token::TokenForever },
-		{ "exit", Token::TokenExit },
+		{ "for", token::TokenFor },
+		{ "to", token::TokenTo },
+		{ "next", token::TokenNext },
+		{ "while", token::TokenWhile },
+		{ "wend", token::TokenWend },
+		{ "repeat", token::TokenRepeat },
+		{ "until", token::TokenUntil },
+		{ "forever", token::TokenForever },
+		{ "exit", token::TokenExit },
 
 		// Math
-		{ "abs", Token::TokenAbs },
-		{ "sign", Token::TokenSign },
-		{ "cos", Token::TokenCos },
-		{ "sin", Token::TokenSin },
-		{ "tan", Token::TokenTan },
-		{ "acos", Token::TokenACos },
-		{ "asin", Token::TokenASin },
-		{ "atan", Token::TokenATan },
-		{ "atan2", Token::TokenATan2 },
-		{ "log", Token::TokenLog },
-		{ "log10", Token::TokenLog10 },
-		{ "ceil", Token::TokenCeil },
-		{ "floor", Token::TokenFloor },
-		{ "mod", Token::TokenMod },
-		{ "pi", Token::TokenPi },
-		{ "exp", Token::TokenExp },
-		{ "sqr", Token::TokenSqr },
+		{ "abs", token::TokenAbs },
+		{ "sign", token::TokenSign },
+		{ "cos", token::TokenCos },
+		{ "sin", token::TokenSin },
+		{ "tan", token::TokenTan },
+		{ "acos", token::TokenACos },
+		{ "asin", token::TokenASin },
+		{ "atan", token::TokenATan },
+		{ "atan2", token::TokenATan2 },
+		{ "log", token::TokenLog },
+		{ "log10", token::TokenLog10 },
+		{ "ceil", token::TokenCeil },
+		{ "floor", token::TokenFloor },
+		{ "mod", token::TokenMod },
+		{ "pi", token::TokenPi },
+		{ "exp", token::TokenExp },
+		{ "sqr", token::TokenSqr },
 
 		// Variables
-		{ "const", Token::TokenConst },
-		{ "global", Token::TokenGlobal },
-		{ "local", Token::TokenLocal },
+		{ "const", token::TokenConst },
+		{ "global", token::TokenGlobal },
+		{ "local", token::TokenLocal },
 
 		// Includes
-		{ "include", Token::TokenInclude },
+		{ "include", token::TokenInclude },
 	};
 	for (auto v : l_textToTokenList) {
-		if (boost::iequals(text, v.first)) {
+		if (stricmp(text.c_str(), v.first)) {
 			return v.second;
 		}
 	}
 	return in;
 }
+*/

code_compiler/source/lexer.hpp

@@ -1,111 +1,60 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
+#include <filesystem>
+#include <fstream>
 #include <inttypes.h>
+#include <iostream>
 #include <istream>
 #include <memory>
 #include <string>
 #include <utility>
 
-namespace blitz {	
-	class Lexer {
-		public:
-		enum class Token : uint64_t {
-			TokenUnknown,
-			TokenEOF,
-			TokenNewLine,
+// ToDo:
+// - Figure out a way to let the lexer output line and character information?
 
-			// Symbols
-			TokenPlus,
-			TokenMinus,
-			TokenSlashForward,
-			TokenSlashBackward,
-			TokenMultiply,
-			TokenEqual,
-			TokenOctothorp,
-			TokenPercent,
-			TokenDollar,
-			TokenRoundBracketOpen,
-			TokenRoundBracketClose,
-			TokenSquareBracketOpen,
-			TokenSquareBracketClose,
-			TokenAngleBracketOpen,
-			TokenAngleBracketClose,
-			TokenDot,
-			TokenColon,
-			TokenComma,
-			TokenSemicolon,
-			TokenCaret,
-			TokenBitNot /*~*/,
+namespace blitz {
+	struct token {
+		uint64_t    line;
+		uint64_t    character;
 
-			// String Delimiter
-			TokenDoubleQuote,
-
-			// Types
-			TokenText,
-			TokenNumber,
-			TokenDecimal,
-			TokenQuotedText, // Text encapsulated by TokenDoubleQuote
-			TokenComment,
-
-			// Binary
-			TokenNot,
-			TokenAnd, TokenOr, TokenXor,
-			TokenShl, TokenShr,
-			TokenSal, TokenSar,
-			TokenFalse, TokenTrue,
-
-			// Conversion
-			TokenFloat,
-			TokenString, TokenHex,
-			TokenInt,
-
-			// Control
-			TokenIf, TokenThen, TokenElseIf, TokenElse, TokenEndIf,
-			TokenSelect, TokenCase, TokenDefault, // End Select = TokenEnd, TokenSelect.
-			TokenGoto, TokenGosub,
-			TokenReturn,
-			TokenFunction, // End Function = TokenEnd, TokenFunction.
-			TokenEnd,
-			TokenStop /* DEBUGGER! Ignore in Release mode. */,
-
-			// Loop
-			TokenFor, TokenTo, TokenNext,
-			TokenWhile, TokenWend,
-			TokenRepeat, TokenUntil, TokenForever,
-			TokenExit,
-
-			// Math
-			TokenAbs, TokenSign /*Sgn*/,
-			TokenCos, TokenSin, TokenTan,
-			TokenACos, TokenASin, TokenATan, TokenATan2,
-			TokenLog, TokenLog10,
-			TokenCeil, TokenFloor,
-			TokenMod,
-			TokenPi,
-			TokenExp, TokenSqr,
-
-			// Variables
-			TokenConst,
-			TokenGlobal,
-			TokenLocal,
-
-			// Including files.
-			TokenInclude,
-		};
-
-		public:
-		Lexer();
-		~Lexer();
-
-		std::pair<Token, std::string> GetCurrentToken();
-		std::pair<Token, std::string> GetNextToken(std::shared_ptr<std::istream> fs);
-		
-		private:
-		blitz::Lexer::Token ConvertTextToToken(Token in, std::string text);
-
-		private:
-		Token m_currentToken = Token::TokenUnknown;
-		std::string m_currentText = "";
-		Token m_overrideToken = Token::TokenUnknown;
-		std::string m_overrideText = "";
+		std::string text;
+		enum class variant : uint64_t {
+			UNKNOWN, // We have absolutely no fucking clue.
+			CONTROL, // All kinds of control signals, like NewLine, EndOfFile, ...
+			SYMBOL, // All kinds of symbols.
+			TEXT, // HelloWorld
+			NUMBER, // 1, 1%
+			DECIMAL, // 1.0, 1#
+			STRING, // "HelloWorld"
+			COMMENT, // ; Whatever
+		} type;
 	};
-}
+
+	class lexer {
+		std::filesystem::path _file;
+		std::ifstream         _stream;
+
+		// Current location in the file.
+		uint64_t _line;
+		uint64_t _character;
+
+		blitz::token _current;
+		blitz::token _override;
+
+		public:
+		~lexer();
+		lexer(std::filesystem::path file);
+
+		/** Retrieve the current token information.
+		 */
+		blitz::token current();
+
+		/** Retrieve the next token in the given stream.
+		 *
+		 * This will replace the current token.
+		 */
+		blitz::token next();
+	};
+} // namespace blitz

code_compiler/source/parser.cpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #include "parser.hpp"
 #include "ast/function.hpp"
 #include <iostream>
@@ -21,10 +24,10 @@ blitz::parser::~parser() {
 	}
 }
 
-std::unique_ptr<blitz::AST::Expression> blitz::parser::Parse() {
-	std::unique_ptr<AST::ScopeExpression> scope = std::make_unique<AST::ScopeExpression>();
+std::unique_ptr<blitz::ast::expression> blitz::parser::parse() {
+	std::unique_ptr<ast::ScopeExpression> scope = std::make_unique<ast::ScopeExpression>();
 
-	std::unique_ptr<AST::Expression> expr;
+	std::unique_ptr<ast::expression> expr;
 	while ((expr = std::move(parse_expression())) != nullptr) {
 		scope->AddExpression(std::move(expr));
 	}
@@ -32,7 +35,7 @@ std::unique_ptr<blitz::AST::Expression> blitz::parser::Parse() {
 	return std::move(scope);
 }
 
-void blitz::parser::LogMessage(const char* msg, ...) {
+void blitz::parser::log(const char* msg, ...) {
 	std::vector<char> buf(65535);
 	va_list val;
 	va_start(val, msg);
@@ -41,7 +44,7 @@ void blitz::parser::LogMessage(const char* msg, ...) {
 	std::cout << buf.data() << '\n';
 }
 
-void blitz::parser::LogError(const char* msg, ...) {
+void blitz::parser::log_error(const char* msg, ...) {
 	std::vector<char> buf(65535);
 	va_list val;
 	va_start(val, msg);
@@ -50,60 +53,61 @@ void blitz::parser::LogError(const char* msg, ...) {
 	std::cerr << buf.data() << '\n';
 }
 
-std::pair<blitz::Lexer::Token, std::string> blitz::parser::GetNextToken() {
-	return m_lexer.GetNextToken(m_files.top().second);
+std::pair<blitz::lexer::tokentype, std::string> blitz::parser::next() {
+	return m_lexer.next(m_files.top().second);
 }
 
-std::unique_ptr<blitz::AST::Expression> blitz::parser::parse_expression() {
+std::unique_ptr<blitz::ast::expression> blitz::parser::parse_expression() {
 	while (true) {
-		auto tkn = GetNextToken();
+		auto tkn = next();
+		log("%s", tkn.second.c_str());
 
 		switch (tkn.first) {
-			case blitz::Lexer::Token::TokenNewLine:
-			case blitz::Lexer::Token::TokenComment:
+			case blitz::lexer::tokentype::TokenNewLine:
+			case blitz::lexer::tokentype::TokenComment:
 				// Skip Comments, since we don't really need them for the AST.
 				continue;
-			case blitz::Lexer::Token::TokenPlus:
-			case blitz::Lexer::Token::TokenMinus:
+			case blitz::lexer::tokentype::TokenPlus:
+			case blitz::lexer::tokentype::TokenMinus:
 
 
 			default: // End Of File / Unknown
-			case blitz::Lexer::Token::TokenUnknown:
-			case blitz::Lexer::Token::TokenEOF:
+			case blitz::lexer::tokentype::TokenUnknown:
+			case blitz::lexer::tokentype::TokenEOF:
 				return nullptr;
 				break;
 		}
 	}
 }
 
-std::unique_ptr<blitz::AST::NumberExpression> blitz::parser::parse_number(blitz::Lexer::Token token, std::string value) {
-	if (token != Lexer::Token::TokenNumber) {
-		LogError("Unexpected Token during parsing, expected number.");
+std::unique_ptr<blitz::ast::NumberExpression> blitz::parser::parse_number(blitz::lexer::tokentype token, std::string value) {
+	if (token != lexer::tokentype::TokenNumber) {
+		log_error("Unexpected Token during parsing, expected number.");
 		return nullptr;
 	}
 
 	char* endptr = const_cast<char*>(value.c_str() + value.size());
 	int32_t parsed = strtol(value.c_str(), &endptr, 10);
 	if (errno == ERANGE) {
-		LogError("Number out of range.");
+		log_error("Number out of range.");
 		return nullptr;
 	}
 
-	return std::make_unique<blitz::AST::NumberExpression>(parsed);
+	return std::make_unique<blitz::ast::NumberExpression>(parsed);
 }
 
-std::unique_ptr<blitz::AST::DecimalExpression> blitz::parser::parse_decimal(blitz::Lexer::Token token, std::string value) {
-	if (token != Lexer::Token::TokenNumber) {
-		LogError("Unexpected Token during parsing, expected number.");
+std::unique_ptr<blitz::ast::DecimalExpression> blitz::parser::parse_decimal(blitz::lexer::tokentype token, std::string value) {
+	if (token != lexer::tokentype::TokenNumber) {
+		log_error("Unexpected Token during parsing, expected number.");
 		return nullptr;
 	}
 
 	char* endptr = const_cast<char*>(value.c_str() + value.size());
 	float_t parsed = strtof(value.c_str(), &endptr);
 	if (errno == ERANGE) {
-		LogError("Number out of range.");
+		log_error("Number out of range.");
 		return nullptr;
 	}
 
-	return std::make_unique<blitz::AST::DecimalExpression>(parsed);
+	return std::make_unique<blitz::ast::DecimalExpression>(parsed);
 }

code_compiler/source/parser.hpp

@@ -1,3 +1,6 @@
+/// AUTOGENERATED COPYRIGHT HEADER START
+// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
+// AUTOGENERATED COPYRIGHT HEADER END
 #pragma once
 #include "lexer.hpp"
 #include "ast/ast.hpp"
@@ -14,22 +17,22 @@ namespace blitz {
 		parser(std::string file);
 		~parser();
 
-		std::unique_ptr<AST::Expression> Parse();
+		std::unique_ptr<ast::expression> parse();
 
 		protected:
-		void LogMessage(const char* msg, ...);
-		void LogError(const char* msg, ...);
+		void log(const char* msg, ...);
+		void log_error(const char* msg, ...);
 
 		private:
-		std::pair<blitz::Lexer::Token, std::string> GetNextToken();
+		std::pair<blitz::lexer::tokentype, std::string> next();
 
 		private:
-		std::unique_ptr<AST::Expression> parse_expression();
-		std::unique_ptr<AST::NumberExpression> parse_number(blitz::Lexer::Token token, std::string value);
-		std::unique_ptr<AST::DecimalExpression> parse_decimal(blitz::Lexer::Token token, std::string value);
+		std::unique_ptr<ast::expression> parse_expression();
+		std::unique_ptr<ast::NumberExpression> parse_number(blitz::lexer::tokentype token, std::string value);
+		std::unique_ptr<ast::DecimalExpression> parse_decimal(blitz::lexer::tokentype token, std::string value);
 
 		private:
-		Lexer m_lexer;
+		lexer m_lexer;
 		std::stack<std::pair<std::string, std::shared_ptr<std::istream>>> m_files;
 
 	};