xaymar/BlitzLLVM
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Latest stuff, rewriting lexer

Browse Source
This commit is contained in:
Michael Fabian 'Xaymar' Dirks
2024-06-25 18:59:15 +02:00
parent 7f669f55e2
commit fa81c2a7fa
23 changed files with 1263 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.clang-format
+117
View File
@@ -0,0 +1,117 @@
# AUTOGENERATED COPYRIGHT HEADER START
# Copyright (C) NaN-NaN undefined
# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
# AUTOGENERATED COPYRIGHT HEADER END
# Basic Formatting
TabWidth: 4
UseTab: ForContinuationAndIndentation
ColumnLimit: 65535
#- 0 does not respect the original line breaks!
# Language
Language: Cpp
Standard: c++17
# Indentation
AccessModifierOffset: 0
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
IndentCaseLabels: false
#IndentPPDirectives: true
IndentWidth: 4
IndentWrappedFunctionNames: true
NamespaceIndentation: All
# Includes
#IncludeBlocks: Regroup
IncludeCategories:
- Regex: '^"warning-disable.hpp"$'
Priority: 50
- Regex: '^(<|")(config.hpp|common.hpp|ui-common.hpp|strings.hpp|version.hpp|obs.h)("|>)'
Priority: 100
- Regex: '^<obs-'
Priority: 150
- Regex: '^<'
Priority: 200
- Regex: '^<Q'
Priority: 250
- Regex: '^"'
Priority: 300
- Regex: '.moc"$'
Priority: 300
- Regex: '^"warning-enable.hpp"$'
Priority: 500
SortIncludes: true
# Alignment
AlignAfterOpenBracket: true
AlignConsecutiveAssignments: true
AlignConsecutiveDeclarations: true
AlignEscapedNewlines: Left
AlignOperands: true
AlignTrailingComments: false
DerivePointerAlignment: false
PointerAlignment: Left
# Wrapping and Breaking
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: true
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
# AfterExternBlock: false
AfterFunction: true
AfterNamespace: false
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
BinPackArguments: true
BinPackParameters: true
BreakBeforeBinaryOperators: NonAssignment
BreakBeforeBraces: Custom
BreakBeforeTernaryOperators: true
BreakConstructorInitializers: BeforeColon
#BreakInheritanceList: BeforeColon
BreakStringLiterals: true
ConstructorInitializerAllOnOneLineOrOnePerLine: false
Cpp11BracedListStyle: true
# Spaces
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
#SpaceBeforeCpp11BracedList: false
#SpaceBeforeCtorInitializerColon: true
#SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
#SpaceBeforeRangeBasedForLoopColon: true
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInCStyleCastParentheses: false
SpacesInContainerLiterals: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
# Other
CommentPragmas: '^(!FIXME!|!TODO!|ToDo:)'
CompactNamespaces: false
DisableFormat: false
FixNamespaceComments: true
#ForEachMacros: ''
KeepEmptyLinesAtTheStartOfBlocks: false
ReflowComments: false
SortUsingDeclarations: true
.clang-tidy
+40
View File
@@ -0,0 +1,40 @@
# AUTOGENERATED COPYRIGHT HEADER START
# Copyright (C) NaN-NaN undefined
# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
# AUTOGENERATED COPYRIGHT HEADER END
Checks: '-*,clang-diagnostic-*,clang-analyzer-*,readability-*,performance-*,portability-*,-portability-simd-intrinsics'
WarningsAsErrors: false
HeaderFilterRegex: ''
AnalyzeTemporaryDtors: false
FormatStyle: file
User: Xaymar
CheckOptions:
- key: cert-dcl16-c.NewSuffixes
value: 'L;LL;LU;LLU'
- key: cert-oop54-cpp.WarnOnlyIfThisHasSuspiciousField
value: '0'
- key: cppcoreguidelines-explicit-virtual-functions.IgnoreDestructors
value: '1'
- key: cppcoreguidelines-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic
value: '1'
- key: google-readability-braces-around-statements.ShortStatementLines
value: '1'
- key: google-readability-function-size.StatementThreshold
value: '800'
- key: google-readability-namespace-comments.ShortNamespaceLines
value: '10'
- key: google-readability-namespace-comments.SpacesBeforeComments
value: '2'
- key: modernize-loop-convert.MaxCopySize
value: '16'
- key: modernize-loop-convert.MinConfidence
value: reasonable
- key: modernize-loop-convert.NamingStyle
value: CamelCase
- key: modernize-pass-by-value.IncludeStyle
value: llvm
- key: modernize-replace-auto-ptr.IncludeStyle
value: llvm
- key: modernize-use-nullptr.NullMacros
value: 'NULL'
.editorconfig
+22
View File
@@ -0,0 +1,22 @@
# AUTOGENERATED COPYRIGHT HEADER START
# Copyright (C) NaN-NaN undefined
# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
# AUTOGENERATED COPYRIGHT HEADER END
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file.
[*]
insert_final_newline = true
trim_trailing_whitespace = true
charset = utf-8
indent_style = tab
indent_size = 4
[*.yml]
indent_style = space
indent_size = 2
[*.md]
trim_trailing_whitespace = false
CMakeLists.txt
+58
View File
@@ -1,3 +1,6 @@
## AUTOGENERATED COPYRIGHT HEADER START
# Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
# AUTOGENERATED COPYRIGHT HEADER END
cmake_minimum_required(VERSION 3.26...3.29.2 FATAL_ERROR) cmake_minimum_required(VERSION 3.26...3.29.2 FATAL_ERROR)
################################################################################ ################################################################################
@@ -171,6 +174,61 @@ set(PROJECT_AUTHORS "See AUTHORS file")
set(PROJECT_COPYRIGHT "All Rights Reserved. See LICENSE file for more information") set(PROJECT_COPYRIGHT "All Rights Reserved. See LICENSE file for more information")
set(PROJECT_TRADEMARKS "") set(PROJECT_TRADEMARKS "")
function(init_project TARGET)
set_target_properties(${TARGET} PROPERTIES
# Always generate position independent code.
POSITION_INDEPENDENT_CODE ON
# Set C++ Standard and Extensions
C_STANDARD 17
C_STANDARD_REQUIRED ON
CXX_STANDARD 20
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF
# Remove prefix from generated files.
PREFIX ""
IMPORT_PREFIX ""
# Never treat warnings as errors.
COMPILE_WARNING_AS_ERROR OFF
)
target_compile_definitions(${TARGET} PRIVATE
__STDC_WANT_LIB_EXT1__=1
)
if(WIN32)
target_compile_definitions(${TARGET} PRIVATE
# windows.h
#- Disable MIN/MAX macro as this breaks a lot of code.
NOMINMAX
#- Disable IN/OUT macro as this breaks a lot of code.
NOINOUT
)
endif()
if(MSVC) # Microsoft Visual C/C++
target_compile_options(${TARGET} PRIVATE
# Disable useless/terrible behavior from MSVC
_CRT_SECURE_NO_WARNINGS
_ENABLE_EXTENDED_ALIGNED_STORAGE
# Dynamically link Microsoft C/C++ Redistributable.
$<$<CONFIG:>:/MD>
$<$<CONFIG:Debug>:/MDd>
$<$<CONFIG:Release>:/MD>
$<$<CONFIG:RelWithDebInfo>:/MD>
$<$<CONFIG:MinSizeRel>:/MD>
# Always compile using multiple processors. Why is this defaulting off anyway?!
/MP
# We want full exception support all the time, not conditionally.
/EHa
)
endif()
endfunction()
################################################################################ ################################################################################
# Sub-projects # Sub-projects
################################################################################ ################################################################################
code_compiler/CMakeLists.txt
+14 -5
View File
@@ -1,10 +1,22 @@
project(code_compiler) ## AUTOGENERATED COPYRIGHT HEADER START
# Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
# AUTOGENERATED COPYRIGHT HEADER END
project(code_compiler
VERSION ${PROJECT_VERSION}
)
add_executable(${PROJECT_NAME}) add_executable(${PROJECT_NAME})
init_project(${PROJECT_NAME})
target_sources(${PROJECT_NAME} PRIVATE target_sources(${PROJECT_NAME} PRIVATE
"source/main.cpp" "source/main.cpp"
"source/lexer.hpp" "source/lexer.hpp"
"source/lexer.cpp" "source/lexer.cpp"
"source/error.hpp"
"source/error.cpp"
"source/parser.hpp"
"source/parser.cpp"
"source/compiler.hpp"
"source/compiler.cpp"
"source/ast/ast.hpp" "source/ast/ast.hpp"
"source/ast/ast.cpp" "source/ast/ast.cpp"
"source/ast/arithmetic.hpp" "source/ast/arithmetic.hpp"
@@ -13,10 +25,6 @@ target_sources(${PROJECT_NAME} PRIVATE
"source/ast/function.cpp" "source/ast/function.cpp"
"source/ast/value.hpp" "source/ast/value.hpp"
"source/ast/value.cpp" "source/ast/value.cpp"
"source/parser.hpp"
"source/parser.cpp"
"source/compiler.hpp"
"source/compiler.cpp"
) )
target_include_directories(${PROJECT_NAME} PRIVATE target_include_directories(${PROJECT_NAME} PRIVATE
"${PROJECT_SOURCE_DIR}/source" "${PROJECT_SOURCE_DIR}/source"
@@ -25,3 +33,4 @@ target_include_directories(${PROJECT_NAME} PRIVATE
get_target_property(_SOURCES ${PROJECT_NAME} SOURCES) get_target_property(_SOURCES ${PROJECT_NAME} SOURCES)
source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${_SOURCES}) source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${_SOURCES})
code_compiler/source/ast/arithmetic.cpp
+5 -2
View File
@@ -1,6 +1,9 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#include "arithmetic.hpp" #include "arithmetic.hpp"
blitz::AST::ArithmeticExpression::ArithmeticExpression(Operator op, std::unique_ptr<Expression> left, std::unique_ptr<Expression> right) blitz::ast::arithmetic_expression::arithmetic_expression(expression_operator op, std::unique_ptr<expression> left, std::unique_ptr<expression> right)
: m_operator(op), m_left(std::move(left)), m_right(std::move(right)) {} : m_operator(op), m_left(std::move(left)), m_right(std::move(right)) {}
blitz::AST::ArithmeticExpression::~ArithmeticExpression() {} blitz::ast::arithmetic_expression::~arithmetic_expression() {}
code_compiler/source/ast/arithmetic.hpp
+10 -7
View File
@@ -1,10 +1,13 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#pragma once #pragma once
#include "ast.hpp" #include "ast.hpp"
#include "value.hpp" #include "value.hpp"
namespace blitz { namespace blitz {
namespace AST { namespace ast {
enum class Operator : int8_t { enum class expression_operator : int8_t {
Add, /*+*/ Add, /*+*/
Subtract, /*-*/ Subtract, /*-*/
Multiply, /***/ Multiply, /***/
@@ -14,14 +17,14 @@ namespace blitz {
Equal, /*=*/ Equal, /*=*/
}; };
class ArithmeticExpression : public Expression { class arithmetic_expression : public expression {
public: public:
ArithmeticExpression(Operator op, std::unique_ptr<Expression> left, std::unique_ptr<Expression> right); arithmetic_expression(expression_operator op, std::unique_ptr<expression> left, std::unique_ptr<expression> right);
virtual ~ArithmeticExpression(); virtual ~arithmetic_expression();
private: private:
Operator m_operator; expression_operator m_operator;
std::unique_ptr<Expression> m_left, m_right; std::unique_ptr<expression> m_left, m_right;
}; };
} }
} }
code_compiler/source/ast/ast.hpp
+6 -3
View File
@@ -1,10 +1,13 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#pragma once #pragma once
namespace blitz { namespace blitz {
namespace AST { namespace ast {
class Expression { class expression {
public: public:
virtual ~Expression() {}; virtual ~expression() {};
}; };
} }
} }
code_compiler/source/ast/function.cpp
+10 -7
View File
@@ -1,14 +1,17 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#include "function.hpp" #include "function.hpp"
blitz::AST::ScopeExpression::ScopeExpression() {} blitz::ast::ScopeExpression::ScopeExpression() {}
blitz::AST::ScopeExpression::~ScopeExpression() {} blitz::ast::ScopeExpression::~ScopeExpression() {}
void blitz::AST::ScopeExpression::AddExpression(std::unique_ptr<Expression> ex) { void blitz::ast::ScopeExpression::AddExpression(std::unique_ptr<expression> ex) {
m_expressions.push_back(std::move(ex)); m_expressions.push_back(std::move(ex));
} }
blitz::AST::FunctionExpression::FunctionExpression(ValueType returnType, blitz::ast::FunctionExpression::FunctionExpression(ValueType returnType,
std::string& m_name, std::string& m_name,
std::list<std::unique_ptr<VariableExpression>> parameters, std::list<std::unique_ptr<VariableExpression>> parameters,
std::unique_ptr<ScopeExpression> scope) std::unique_ptr<ScopeExpression> scope)
@@ -16,11 +19,11 @@ blitz::AST::FunctionExpression::FunctionExpression(ValueType returnType,
} }
blitz::AST::FunctionExpression::~FunctionExpression() {} blitz::ast::FunctionExpression::~FunctionExpression() {}
blitz::AST::CallExpression::CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments) blitz::ast::CallExpression::CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments)
: m_name(name), m_arguments(std::move(arguments)) { : m_name(name), m_arguments(std::move(arguments)) {
} }
blitz::AST::CallExpression::~CallExpression() {} blitz::ast::CallExpression::~CallExpression() {}
code_compiler/source/ast/function.hpp
+8 -5
View File
@@ -1,3 +1,6 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#pragma once #pragma once
#include "ast.hpp" #include "ast.hpp"
#include "value.hpp" #include "value.hpp"
@@ -6,16 +9,16 @@
#include <string> #include <string>
namespace blitz { namespace blitz {
namespace AST { namespace ast {
class ScopeExpression : public Expression { class ScopeExpression : public expression {
public: public:
ScopeExpression(); ScopeExpression();
virtual ~ScopeExpression(); virtual ~ScopeExpression();
void AddExpression(std::unique_ptr<Expression> ex); void AddExpression(std::unique_ptr<expression> ex);
private: private:
std::list<std::unique_ptr<Expression>> m_expressions; std::list<std::unique_ptr<expression>> m_expressions;
}; };
class FunctionExpression : public ScopeExpression { class FunctionExpression : public ScopeExpression {
@@ -33,7 +36,7 @@ namespace blitz {
std::unique_ptr<ScopeExpression> m_content; std::unique_ptr<ScopeExpression> m_content;
}; };
class CallExpression : public Expression { class CallExpression : public expression {
public: public:
CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments); CallExpression(std::string& name, std::list<std::unique_ptr<VariableExpression>> arguments);
virtual ~CallExpression(); virtual ~CallExpression();
code_compiler/source/ast/value.cpp
+18 -15
View File
@@ -1,43 +1,46 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#include "value.hpp" #include "value.hpp"
blitz::AST::VariableExpression::VariableExpression(std::string& name, ValueType type /*= ValueType::Number*/) blitz::ast::VariableExpression::VariableExpression(std::string& name, ValueType type /*= ValueType::Number*/)
: m_name(name), m_type(type) {} : m_name(name), m_type(type) {}
blitz::AST::VariableExpression::~VariableExpression() {} blitz::ast::VariableExpression::~VariableExpression() {}
blitz::AST::ValueType blitz::AST::VariableExpression::GetType() { blitz::ast::ValueType blitz::ast::VariableExpression::GetType() {
return m_type; return m_type;
} }
blitz::AST::NumberExpression::NumberExpression(int32_t value) : value(value) {} blitz::ast::NumberExpression::NumberExpression(int32_t value) : value(value) {}
blitz::AST::NumberExpression::~NumberExpression() {} blitz::ast::NumberExpression::~NumberExpression() {}
blitz::AST::ValueType blitz::AST::NumberExpression::GetType() { blitz::ast::ValueType blitz::ast::NumberExpression::GetType() {
return ValueType::Number; return ValueType::Number;
} }
blitz::AST::DecimalExpression::DecimalExpression(float_t value) : value(value) {} blitz::ast::DecimalExpression::DecimalExpression(float_t value) : value(value) {}
blitz::AST::DecimalExpression::~DecimalExpression() {} blitz::ast::DecimalExpression::~DecimalExpression() {}
blitz::AST::ValueType blitz::AST::DecimalExpression::GetType() { blitz::ast::ValueType blitz::ast::DecimalExpression::GetType() {
return ValueType::Decimal; return ValueType::Decimal;
} }
blitz::AST::StringExpression::StringExpression(std::string value) : value(value) {} blitz::ast::StringExpression::StringExpression(std::string value) : value(value) {}
blitz::AST::StringExpression::~StringExpression() {} blitz::ast::StringExpression::~StringExpression() {}
blitz::AST::ValueType blitz::AST::StringExpression::GetType() { blitz::ast::ValueType blitz::ast::StringExpression::GetType() {
return ValueType::String; return ValueType::String;
} }
blitz::AST::ConstExpression::ConstExpression(std::string& name, std::unique_ptr<ValueExpression> value) blitz::ast::ConstExpression::ConstExpression(std::string& name, std::unique_ptr<ValueExpression> value)
: m_name(name), m_value(std::move(value)) {} : m_name(name), m_value(std::move(value)) {}
blitz::AST::ConstExpression::~ConstExpression() {} blitz::ast::ConstExpression::~ConstExpression() {}
blitz::AST::ValueType blitz::AST::ConstExpression::GetType() { blitz::ast::ValueType blitz::ast::ConstExpression::GetType() {
return m_value->GetType(); return m_value->GetType();
} }
code_compiler/source/ast/value.hpp
+5 -2
View File
@@ -1,3 +1,6 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#pragma once #pragma once
#include "ast.hpp" #include "ast.hpp"
#include "lexer.hpp" #include "lexer.hpp"
@@ -8,7 +11,7 @@
#include <string> #include <string>
namespace blitz { namespace blitz {
namespace AST { namespace ast {
enum class ValueType : int8_t { enum class ValueType : int8_t {
Unknown, Unknown,
Number, Number,
@@ -17,7 +20,7 @@ namespace blitz {
Type, Type,
}; };
class ValueExpression : public Expression { class ValueExpression : public expression {
public: public:
virtual ValueType GetType() = 0; virtual ValueType GetType() = 0;
}; };
code_compiler/source/compiler.cpp
+3 -16
View File
@@ -1,3 +1,6 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#include "compiler.hpp" #include "compiler.hpp"
#include "parser.hpp" #include "parser.hpp"
#include "lexer.hpp" #include "lexer.hpp"
@@ -7,19 +10,3 @@
blitz::compiler::compiler() {} blitz::compiler::compiler() {}
blitz::compiler::~compiler() {} blitz::compiler::~compiler() {}
bool blitz::compiler::compile(std::string in, std::string out) {
/*std::ifstream infile;
infile.open(in);
if (infile.bad() || !infile.good() || infile.eof()) {
std::cerr << "Failed to open file: " << in << std::endl;
return false;
}*/
parser psr = parser(in);
if (!psr.Parse()) {
}
return true;
}
code_compiler/source/compiler.hpp
+12 -2
View File
@@ -1,12 +1,22 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#pragma once #pragma once
#include <string> #include <string>
#include <filesystem>
// Compiling requires several steps
// 1. Lexing to known tokens
// - This part may require a unicode library/framework, like Qt or ICU
// 2. Parsing the lexical tokens into an Abstract Syntax Tree (AST)
// - This is where we convert everything to logical steps, like math and function calls.
// 3. Compiling from the AST to some kind of runnable binary.
// - In our case, we convert the AST to LLVM IR, and then compile it with LLVM.
namespace blitz { namespace blitz {
class compiler { class compiler {
public: public:
compiler(); compiler();
~compiler(); ~compiler();
bool compile(std::string in, std::string out);
}; };
} }
code_compiler/source/error.cpp
+4
View File
@@ -0,0 +1,4 @@
// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) NaN-NaN undefined
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
code_compiler/source/error.hpp
+4
View File
@@ -0,0 +1,4 @@
// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) NaN-NaN undefined
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
code_compiler/source/lexer.cpp
+234 -112
View File
@@ -1,49 +1,170 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#include "lexer.hpp" #include "lexer.hpp"
#include <codecvt> #include <codecvt>
#include <sstream>
std::pair<char, blitz::Lexer::Token> g_symbolCharacters[] = { /*std::pair<char, blitz::tokentype> g_symbolCharacters[] = {
//{ '\"', BlitzLLVM::Lexer::Token::TokenDoubleQuote }, // Has special meaning. //{ '\"', BlitzLLVM::Lexer::Token::TokenDoubleQuote }, // Has special meaning.
{ '+', blitz::Lexer::Token::TokenPlus }, { '+', blitz::tokentype::TokenPlus },
{ '-', blitz::Lexer::Token::TokenMinus }, { '-', blitz::tokentype::TokenMinus },
{ '/', blitz::Lexer::Token::TokenSlashForward }, { '/', blitz:::tokentype::TokenSlashForward },
{ '\\', blitz::Lexer::Token::TokenSlashBackward }, { '\\', blitz::tokentype::TokenSlashBackward },
{ '*', blitz::Lexer::Token::TokenMultiply }, { '*', blitz::tokentype::TokenMultiply },
{ '=', blitz::Lexer::Token::TokenEqual }, { '=', blitz::tokentype::TokenEqual },
{ '#', blitz::Lexer::Token::TokenOctothorp }, { '#', blitz::tokentype::TokenOctothorp },
{ '%', blitz::Lexer::Token::TokenPercent }, { '%', blitz::tokentype::TokenPercent },
{ '$', blitz::Lexer::Token::TokenDollar }, { '$', blitz::tokentype::TokenDollar },
{ '(', blitz::Lexer::Token::TokenRoundBracketOpen }, { '(', blitz::tokentype::TokenRoundBracketOpen },
{ ')', blitz::Lexer::Token::TokenRoundBracketClose }, { ')', blitz::tokentype::TokenRoundBracketClose },
{ '[', blitz::Lexer::Token::TokenSquareBracketOpen }, { '[', blitz::tokentype::TokenSquareBracketOpen },
{ ']', blitz::Lexer::Token::TokenSquareBracketClose }, { ']', blitz::tokentype::TokenSquareBracketClose },
{ '<', blitz::Lexer::Token::TokenAngleBracketOpen }, { '<', blitz::tokentype::TokenAngleBracketOpen },
{ '>', blitz::Lexer::Token::TokenAngleBracketClose }, { '>', blitz::tokentype::TokenAngleBracketClose },
//{ '.', BlitzLLVM::Lexer::Token::TokenDot }, // Special meaning. //{ '.', BlitzLLVM::Token::TokenDot }, // Special meaning.
{ ':', blitz::Lexer::Token::TokenColon }, { ':', blitz::tokentype::TokenColon },
{ ',', blitz::Lexer::Token::TokenComma }, { ',', blitz::tokentype::TokenComma },
//{ ';', BlitzLLVM::Lexer::Token::TokenSemicolon }, //{ ';', BlitzLLVM::Token::TokenSemicolon },
{ '^', blitz::Lexer::Token::TokenCaret }, { '^', blitz::tokentype::TokenCaret },
{ '~', blitz::Lexer::Token::TokenBitNot }, { '~', blitz::tokentype::TokenBitNot },
}; };*/
blitz::Lexer::Lexer() {} blitz::lexer::~lexer() {}
blitz::Lexer::~Lexer() {} blitz::lexer::lexer(std::filesystem::path file)
{
// Usually files start at line and character 0, so we should start there too.
_line = _character = 0;
std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetCurrentToken() { // Try and open the file for reading.
return std::make_pair(m_currentToken, m_currentText); _file = file;
_stream = std::ifstream(_file, std::ios_base::binary); // We use binary so we can eventually support UTF-8.
if (!_stream.good() || _stream.eof() || _stream.bad() || _stream.fail()) {
char buffer[16384];
int len = snprintf(buffer, sizeof(buffer), "Reading file '%s' failed.\0", file.generic_string().c_str());
throw std::runtime_error(std::string(buffer, buffer + len));
}
// Initialize token storage to a default token.
_override = _current = blitz::token{
.line = 0,
.character = 0,
.text = "",
.type = token::variant::UNKNOWN,
};
} }
std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shared_ptr<std::istream> fs) { blitz::token blitz::lexer::current()
{
return _current;
}
blitz::token blitz::lexer::next()
{
enum class stage {
DEFAULT,
TEXT,
NUMBER,
STRING,
COMMENT,
} state = stage::DEFAULT;
bool numberHasDot = false;
std::stringstream buffer;
blitz::token token{
.line = _line,
.character = _character,
.text = "",
.type = blitz::token::variant::UNKNOWN,
};
// Helper function to advance text.
auto advance = [this]() {
_character++;
return _stream.get();
};
while ((token.type == blitz::token::variant::UNKNOWN) && _stream.good() && !_stream.eof()) {
auto chr = advance();
if (state == stage::DEFAULT) {
if (chr == ';') { // We've encountered a comment, so we should change state and ignore this symbol.
state = stage::COMMENT;
token.line = _line;
token.character = _character;
token.type = blitz::token::variant::COMMENT;
} else {
buffer << chr;
}
} else if (state == stage::NUMBER) {
if (isdigit(chr)) {
buffer << chr;
} else if (chr == '.')
{
if (numberHasDot) {
throw std::runtime_error("")
}
numberHasDot = true;
} else {
}
} else if (state == stage::TEXT) {
} else if (state == stage::STRING) {
} else if (state == stage::COMMENT) {
if (chr == '\r' && _stream.peek() == '\n') {
token.text = buffer.str();
} else {
buffer << chr;
}
}
}
_current = token;
return _current;
}
/*
std::pair<blitz::tokentype, std::string> blitz::lexer::current() {
return _current;
}
std::pair<blitz::tokentype, std::string> blitz::lexer::next(std::istream& fs) {
std::stringstream buffer;
blitz::tokentype token;
enum class parserState {
DEFAULT,
TEXT,
NUMBER,
STRING,
COMMENT,
} state = parserState::DEFAULT;
while ((token == blitz::tokentype::TokenUnknown) && !fs.eof() && fs.good()) {
auto chr = fs.get();
}
}
/*
std::pair<blitz::lexer::token, std::string> blitz::lexer::next(std::shared_ptr<std::istream> fs) {
std::string buf; std::string buf;
Token tkn = Token::TokenEOF; token tkn = token::TokenUnknown;
bool haveResult = false; bool haveResult = false;
// Allow "overriding" the next retrieved Token. // Allow "overriding" the next retrieved Token.
if (m_overrideToken != Token::TokenUnknown) { if (m_overrideToken != token::TokenUnknown) {
buf = m_overrideText; buf = m_overrideText;
tkn = m_overrideToken; tkn = m_overrideToken;
m_overrideToken = Token::TokenUnknown; m_overrideToken = token::TokenUnknown;
haveResult = true; haveResult = true;
} }
@@ -56,11 +177,11 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
char chr = fs->get(); char chr = fs->get();
if (chr == '\r' || chr == '\n') { if (chr == '\r' || chr == '\n') {
if (tkn != Token::TokenEOF) { if (tkn != token::TokenEOF) {
m_overrideToken = Token::TokenNewLine; m_overrideToken = token::TokenNewLine;
m_overrideText = ""; m_overrideText = "";
} else { } else {
tkn = Token::TokenNewLine; tkn = token::TokenNewLine;
buf = ""; buf = "";
} }
@@ -71,10 +192,10 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
break; break;
} else if (m_isStringMode) { } else if (m_isStringMode) {
if (chr == '\"') { if (chr == '\"') {
m_overrideToken = Token::TokenDoubleQuote; m_overrideToken = token::TokenDoubleQuote;
m_overrideText = chr; m_overrideText = chr;
m_isStringMode = false; m_isStringMode = false;
tkn = Token::TokenQuotedText; tkn = token::TokenQuotedText;
break; break;
} else if (iscntrl(chr) || !isprint(chr)) { } else if (iscntrl(chr) || !isprint(chr)) {
fs->putback(chr); fs->putback(chr);
@@ -97,7 +218,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
} else if (chr == '.') { } else if (chr == '.') {
if (m_numberModeHasDecimal == false) { if (m_numberModeHasDecimal == false) {
m_numberModeHasDecimal = true; m_numberModeHasDecimal = true;
tkn = Token::TokenDecimal; tkn = token::TokenDecimal;
buf += chr; buf += chr;
} else { } else {
fs->putback(chr); fs->putback(chr);
@@ -111,7 +232,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
} }
} else if (m_isCommentMode) { } else if (m_isCommentMode) {
buf += chr; buf += chr;
tkn = Token::TokenComment; tkn = token::TokenComment;
} else { } else {
// Whitespace // Whitespace
if (isspace(chr)) if (isspace(chr))
@@ -119,7 +240,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
// Control Code // Control Code
if (iscntrl(chr)) { if (iscntrl(chr)) {
tkn = Token::TokenUnknown; tkn = token::TokenUnknown;
buf = chr; buf = chr;
} }
@@ -129,13 +250,13 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
if (isdigit(chr2)) { if (isdigit(chr2)) {
m_isNumberMode = true; m_isNumberMode = true;
m_numberModeHasDecimal = false; m_numberModeHasDecimal = false;
tkn = Token::TokenNumber; tkn = token::TokenNumber;
buf = chr + chr2; buf = chr + chr2;
break; break;
} else if (chr2 == '.') { } else if (chr2 == '.') {
m_isNumberMode = true; m_isNumberMode = true;
m_numberModeHasDecimal = true; m_numberModeHasDecimal = true;
tkn = Token::TokenDecimal; tkn = token::TokenDecimal;
buf = chr + "0" + chr2; buf = chr + "0" + chr2;
break; break;
} else { } else {
@@ -151,7 +272,7 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
break; break;
} }
} }
if (tkn != Token::TokenEOF) { if (tkn != token::TokenEOF) {
haveResult = true; haveResult = true;
break; break;
} }
@@ -159,30 +280,30 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
// Strings, Text, Numbers // Strings, Text, Numbers
if (chr == ';') { if (chr == ';') {
m_isCommentMode = true; m_isCommentMode = true;
tkn = Token::TokenSemicolon; tkn = token::TokenSemicolon;
buf = chr; buf = chr;
break; break;
} else if (chr == '\"') { } else if (chr == '\"') {
m_isStringMode = true; m_isStringMode = true;
tkn = Token::TokenDoubleQuote; tkn = token::TokenDoubleQuote;
buf = chr; buf = chr;
break; break;
} else if (isalpha(chr)) { } else if (isalpha(chr)) {
m_isTextMode = true; m_isTextMode = true;
tkn = Token::TokenText; tkn = token::TokenText;
buf = chr; buf = chr;
} else if (isdigit(chr)) { } else if (isdigit(chr)) {
m_isNumberMode = true; m_isNumberMode = true;
m_numberModeHasDecimal = false; m_numberModeHasDecimal = false;
tkn = Token::TokenNumber; tkn = token::TokenNumber;
buf = chr; buf = chr;
} else if (chr == '.') { } else if (chr == '.') {
m_isNumberMode = true; m_isNumberMode = true;
m_numberModeHasDecimal = true; m_numberModeHasDecimal = true;
tkn = Token::TokenDecimal; tkn = token::TokenDecimal;
buf = "0" + chr; buf = "0" + chr;
} else { } else {
tkn = Token::TokenUnknown; tkn = token::TokenUnknown;
buf = chr; buf = chr;
break; break;
} }
@@ -190,90 +311,91 @@ std::pair<blitz::Lexer::Token, std::string> blitz::Lexer::GetNextToken(std::shar
} }
// Convert from Text into native Token. // Convert from Text into native Token.
if (tkn == Token::TokenText) if (tkn == token::TokenText)
tkn = ConvertTextToToken(tkn, buf); tkn = to_token(tkn, buf);
return std::make_pair(tkn, buf); return std::make_pair(tkn, buf);
} }
blitz::Lexer::Token blitz::Lexer::ConvertTextToToken(Token in, std::string text) { blitz::lexer::token blitz::lexer::to_token(token in, std::string text) {
static std::pair<const char*, Token> l_textToTokenList[] = { static std::pair<const char*, token> l_textToTokenList[] = {
// Binary // Binary
{ "not", Token::TokenNot }, { "not", token::TokenNot },
{ "and", Token::TokenAnd }, { "and", token::TokenAnd },
{ "or", Token::TokenOr }, { "or", token::TokenOr },
{ "xor", Token::TokenXor }, { "xor", token::TokenXor },
{ "shl", Token::TokenShl }, { "shl", token::TokenShl },
{ "shr", Token::TokenShr }, { "shr", token::TokenShr },
{ "sal", Token::TokenSal }, { "sal", token::TokenSal },
{ "sar", Token::TokenSar }, { "sar", token::TokenSar },
{ "false", Token::TokenFalse }, { "false", token::TokenFalse },
{ "true", Token::TokenTrue }, { "true", token::TokenTrue },
// Conversion // Conversion
{ "float", Token::TokenFloat }, { "float", token::TokenFloat },
{ "string", Token::TokenString }, { "string", token::TokenString },
{ "hex", Token::TokenHex }, { "hex", token::TokenHex },
{ "int", Token::TokenInt }, { "int", token::TokenInt },
// Control // Control
{ "if", Token::TokenIf }, { "if", token::TokenIf },
{ "then", Token::TokenThen }, { "then", token::TokenThen },
{ "elseIf", Token::TokenElseIf }, { "elseif", token::TokenElseIf },
{ "else", Token::TokenElse }, { "else", token::TokenElse },
{ "endIf", Token::TokenEndIf }, { "endif", token::TokenEndIf },
{ "select", Token::TokenSelect }, { "select", token::TokenSelect },
{ "case", Token::TokenCase }, { "case", token::TokenCase },
{ "default", Token::TokenDefault }, { "default", token::TokenDefault },
{ "goto", Token::TokenGoto }, { "goto", token::TokenGoto },
{ "gosub", Token::TokenGosub }, { "gosub", token::TokenGosub },
{ "return", Token::TokenReturn }, { "return", token::TokenReturn },
{ "function", Token::TokenFunction }, { "function", token::TokenFunction },
{ "end", Token::TokenEnd }, { "end", token::TokenEnd },
{ "stop", Token::TokenStop }, { "stop", token::TokenStop },
// Loop // Loop
{ "for", Token::TokenFor }, { "for", token::TokenFor },
{ "to", Token::TokenTo }, { "to", token::TokenTo },
{ "next", Token::TokenNext }, { "next", token::TokenNext },
{ "while", Token::TokenWhile }, { "while", token::TokenWhile },
{ "wend", Token::TokenWend }, { "wend", token::TokenWend },
{ "repeat", Token::TokenRepeat }, { "repeat", token::TokenRepeat },
{ "until", Token::TokenUntil }, { "until", token::TokenUntil },
{ "forever", Token::TokenForever }, { "forever", token::TokenForever },
{ "exit", Token::TokenExit }, { "exit", token::TokenExit },
// Math // Math
{ "abs", Token::TokenAbs }, { "abs", token::TokenAbs },
{ "sign", Token::TokenSign }, { "sign", token::TokenSign },
{ "cos", Token::TokenCos }, { "cos", token::TokenCos },
{ "sin", Token::TokenSin }, { "sin", token::TokenSin },
{ "tan", Token::TokenTan }, { "tan", token::TokenTan },
{ "acos", Token::TokenACos }, { "acos", token::TokenACos },
{ "asin", Token::TokenASin }, { "asin", token::TokenASin },
{ "atan", Token::TokenATan }, { "atan", token::TokenATan },
{ "atan2", Token::TokenATan2 }, { "atan2", token::TokenATan2 },
{ "log", Token::TokenLog }, { "log", token::TokenLog },
{ "log10", Token::TokenLog10 }, { "log10", token::TokenLog10 },
{ "ceil", Token::TokenCeil }, { "ceil", token::TokenCeil },
{ "floor", Token::TokenFloor }, { "floor", token::TokenFloor },
{ "mod", Token::TokenMod }, { "mod", token::TokenMod },
{ "pi", Token::TokenPi }, { "pi", token::TokenPi },
{ "exp", Token::TokenExp }, { "exp", token::TokenExp },
{ "sqr", Token::TokenSqr }, { "sqr", token::TokenSqr },
// Variables // Variables
{ "const", Token::TokenConst }, { "const", token::TokenConst },
{ "global", Token::TokenGlobal }, { "global", token::TokenGlobal },
{ "local", Token::TokenLocal }, { "local", token::TokenLocal },
// Includes // Includes
{ "include", Token::TokenInclude }, { "include", token::TokenInclude },
}; };
for (auto v : l_textToTokenList) { for (auto v : l_textToTokenList) {
if (boost::iequals(text, v.first)) { if (stricmp(text.c_str(), v.first)) {
return v.second; return v.second;
} }
} }
return in; return in;
} }
*/
code_compiler/source/lexer.hpp
+50 -101
View File
@@ -1,111 +1,60 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#pragma once #pragma once
#include <filesystem>
#include <fstream>
#include <inttypes.h> #include <inttypes.h>
#include <iostream>
#include <istream> #include <istream>
#include <memory> #include <memory>
#include <string> #include <string>
#include <utility> #include <utility>
namespace blitz { // ToDo:
class Lexer { // - Figure out a way to let the lexer output line and character information?
public:
enum class Token : uint64_t {
TokenUnknown,
TokenEOF,
TokenNewLine,
// Symbols namespace blitz {
TokenPlus, struct token {
TokenMinus, uint64_t line;
TokenSlashForward, uint64_t character;
TokenSlashBackward,
TokenMultiply,
TokenEqual,
TokenOctothorp,
TokenPercent,
TokenDollar,
TokenRoundBracketOpen,
TokenRoundBracketClose,
TokenSquareBracketOpen,
TokenSquareBracketClose,
TokenAngleBracketOpen,
TokenAngleBracketClose,
TokenDot,
TokenColon,
TokenComma,
TokenSemicolon,
TokenCaret,
TokenBitNot /*~*/,
// String Delimiter std::string text;
TokenDoubleQuote, enum class variant : uint64_t {
UNKNOWN, // We have absolutely no fucking clue.
// Types CONTROL, // All kinds of control signals, like NewLine, EndOfFile, ...
TokenText, SYMBOL, // All kinds of symbols.
TokenNumber, TEXT, // HelloWorld
TokenDecimal, NUMBER, // 1, 1%
TokenQuotedText, // Text encapsulated by TokenDoubleQuote DECIMAL, // 1.0, 1#
TokenComment, STRING, // "HelloWorld"
COMMENT, // ; Whatever
// Binary } type;
TokenNot,
TokenAnd, TokenOr, TokenXor,
TokenShl, TokenShr,
TokenSal, TokenSar,
TokenFalse, TokenTrue,
// Conversion
TokenFloat,
TokenString, TokenHex,
TokenInt,
// Control
TokenIf, TokenThen, TokenElseIf, TokenElse, TokenEndIf,
TokenSelect, TokenCase, TokenDefault, // End Select = TokenEnd, TokenSelect.
TokenGoto, TokenGosub,
TokenReturn,
TokenFunction, // End Function = TokenEnd, TokenFunction.
TokenEnd,
TokenStop /* DEBUGGER! Ignore in Release mode. */,
// Loop
TokenFor, TokenTo, TokenNext,
TokenWhile, TokenWend,
TokenRepeat, TokenUntil, TokenForever,
TokenExit,
// Math
TokenAbs, TokenSign /*Sgn*/,
TokenCos, TokenSin, TokenTan,
TokenACos, TokenASin, TokenATan, TokenATan2,
TokenLog, TokenLog10,
TokenCeil, TokenFloor,
TokenMod,
TokenPi,
TokenExp, TokenSqr,
// Variables
TokenConst,
TokenGlobal,
TokenLocal,
// Including files.
TokenInclude,
};
public:
Lexer();
~Lexer();
std::pair<Token, std::string> GetCurrentToken();
std::pair<Token, std::string> GetNextToken(std::shared_ptr<std::istream> fs);
private:
blitz::Lexer::Token ConvertTextToToken(Token in, std::string text);
private:
Token m_currentToken = Token::TokenUnknown;
std::string m_currentText = "";
Token m_overrideToken = Token::TokenUnknown;
std::string m_overrideText = "";
}; };
}
class lexer {
std::filesystem::path _file;
std::ifstream _stream;
// Current location in the file.
uint64_t _line;
uint64_t _character;
blitz::token _current;
blitz::token _override;
public:
~lexer();
lexer(std::filesystem::path file);
/** Retrieve the current token information.
*/
blitz::token current();
/** Retrieve the next token in the given stream.
*
* This will replace the current token.
*/
blitz::token next();
};
} // namespace blitz
code_compiler/source/parser.cpp
+29 -25
View File
@@ -1,3 +1,6 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#include "parser.hpp" #include "parser.hpp"
#include "ast/function.hpp" #include "ast/function.hpp"
#include <iostream> #include <iostream>
@@ -21,10 +24,10 @@ blitz::parser::~parser() {
} }
} }
std::unique_ptr<blitz::AST::Expression> blitz::parser::Parse() { std::unique_ptr<blitz::ast::expression> blitz::parser::parse() {
std::unique_ptr<AST::ScopeExpression> scope = std::make_unique<AST::ScopeExpression>(); std::unique_ptr<ast::ScopeExpression> scope = std::make_unique<ast::ScopeExpression>();
std::unique_ptr<AST::Expression> expr; std::unique_ptr<ast::expression> expr;
while ((expr = std::move(parse_expression())) != nullptr) { while ((expr = std::move(parse_expression())) != nullptr) {
scope->AddExpression(std::move(expr)); scope->AddExpression(std::move(expr));
} }
@@ -32,7 +35,7 @@ std::unique_ptr<blitz::AST::Expression> blitz::parser::Parse() {
return std::move(scope); return std::move(scope);
} }
void blitz::parser::LogMessage(const char* msg, ...) { void blitz::parser::log(const char* msg, ...) {
std::vector<char> buf(65535); std::vector<char> buf(65535);
va_list val; va_list val;
va_start(val, msg); va_start(val, msg);
@@ -41,7 +44,7 @@ void blitz::parser::LogMessage(const char* msg, ...) {
std::cout << buf.data() << '\n'; std::cout << buf.data() << '\n';
} }
void blitz::parser::LogError(const char* msg, ...) { void blitz::parser::log_error(const char* msg, ...) {
std::vector<char> buf(65535); std::vector<char> buf(65535);
va_list val; va_list val;
va_start(val, msg); va_start(val, msg);
@@ -50,60 +53,61 @@ void blitz::parser::LogError(const char* msg, ...) {
std::cerr << buf.data() << '\n'; std::cerr << buf.data() << '\n';
} }
std::pair<blitz::Lexer::Token, std::string> blitz::parser::GetNextToken() { std::pair<blitz::lexer::tokentype, std::string> blitz::parser::next() {
return m_lexer.GetNextToken(m_files.top().second); return m_lexer.next(m_files.top().second);
} }
std::unique_ptr<blitz::AST::Expression> blitz::parser::parse_expression() { std::unique_ptr<blitz::ast::expression> blitz::parser::parse_expression() {
while (true) { while (true) {
auto tkn = GetNextToken(); auto tkn = next();
log("%s", tkn.second.c_str());
switch (tkn.first) { switch (tkn.first) {
case blitz::Lexer::Token::TokenNewLine: case blitz::lexer::tokentype::TokenNewLine:
case blitz::Lexer::Token::TokenComment: case blitz::lexer::tokentype::TokenComment:
// Skip Comments, since we don't really need them for the AST. // Skip Comments, since we don't really need them for the AST.
continue; continue;
case blitz::Lexer::Token::TokenPlus: case blitz::lexer::tokentype::TokenPlus:
case blitz::Lexer::Token::TokenMinus: case blitz::lexer::tokentype::TokenMinus:
default: // End Of File / Unknown default: // End Of File / Unknown
case blitz::Lexer::Token::TokenUnknown: case blitz::lexer::tokentype::TokenUnknown:
case blitz::Lexer::Token::TokenEOF: case blitz::lexer::tokentype::TokenEOF:
return nullptr; return nullptr;
break; break;
} }
} }
} }
std::unique_ptr<blitz::AST::NumberExpression> blitz::parser::parse_number(blitz::Lexer::Token token, std::string value) { std::unique_ptr<blitz::ast::NumberExpression> blitz::parser::parse_number(blitz::lexer::tokentype token, std::string value) {
if (token != Lexer::Token::TokenNumber) { if (token != lexer::tokentype::TokenNumber) {
LogError("Unexpected Token during parsing, expected number."); log_error("Unexpected Token during parsing, expected number.");
return nullptr; return nullptr;
} }
char* endptr = const_cast<char*>(value.c_str() + value.size()); char* endptr = const_cast<char*>(value.c_str() + value.size());
int32_t parsed = strtol(value.c_str(), &endptr, 10); int32_t parsed = strtol(value.c_str(), &endptr, 10);
if (errno == ERANGE) { if (errno == ERANGE) {
LogError("Number out of range."); log_error("Number out of range.");
return nullptr; return nullptr;
} }
return std::make_unique<blitz::AST::NumberExpression>(parsed); return std::make_unique<blitz::ast::NumberExpression>(parsed);
} }
std::unique_ptr<blitz::AST::DecimalExpression> blitz::parser::parse_decimal(blitz::Lexer::Token token, std::string value) { std::unique_ptr<blitz::ast::DecimalExpression> blitz::parser::parse_decimal(blitz::lexer::tokentype token, std::string value) {
if (token != Lexer::Token::TokenNumber) { if (token != lexer::tokentype::TokenNumber) {
LogError("Unexpected Token during parsing, expected number."); log_error("Unexpected Token during parsing, expected number.");
return nullptr; return nullptr;
} }
char* endptr = const_cast<char*>(value.c_str() + value.size()); char* endptr = const_cast<char*>(value.c_str() + value.size());
float_t parsed = strtof(value.c_str(), &endptr); float_t parsed = strtof(value.c_str(), &endptr);
if (errno == ERANGE) { if (errno == ERANGE) {
LogError("Number out of range."); log_error("Number out of range.");
return nullptr; return nullptr;
} }
return std::make_unique<blitz::AST::DecimalExpression>(parsed); return std::make_unique<blitz::ast::DecimalExpression>(parsed);
} }
code_compiler/source/parser.hpp
+11 -8
View File
@@ -1,3 +1,6 @@
/// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) 2017-2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
#pragma once #pragma once
#include "lexer.hpp" #include "lexer.hpp"
#include "ast/ast.hpp" #include "ast/ast.hpp"
@@ -14,22 +17,22 @@ namespace blitz {
parser(std::string file); parser(std::string file);
~parser(); ~parser();
std::unique_ptr<AST::Expression> Parse(); std::unique_ptr<ast::expression> parse();
protected: protected:
void LogMessage(const char* msg, ...); void log(const char* msg, ...);
void LogError(const char* msg, ...); void log_error(const char* msg, ...);
private: private:
std::pair<blitz::Lexer::Token, std::string> GetNextToken(); std::pair<blitz::lexer::tokentype, std::string> next();
private: private:
std::unique_ptr<AST::Expression> parse_expression(); std::unique_ptr<ast::expression> parse_expression();
std::unique_ptr<AST::NumberExpression> parse_number(blitz::Lexer::Token token, std::string value); std::unique_ptr<ast::NumberExpression> parse_number(blitz::lexer::tokentype token, std::string value);
std::unique_ptr<AST::DecimalExpression> parse_decimal(blitz::Lexer::Token token, std::string value); std::unique_ptr<ast::DecimalExpression> parse_decimal(blitz::lexer::tokentype token, std::string value);
private: private:
Lexer m_lexer; lexer m_lexer;
std::stack<std::pair<std::string, std::shared_ptr<std::istream>>> m_files; std::stack<std::pair<std::string, std::shared_ptr<std::istream>>> m_files;
}; };
pre-commit.sh
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
# AUTOGENERATED COPYRIGHT HEADER START
# Copyright (C) NaN-NaN undefined
# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
# AUTOGENERATED COPYRIGHT HEADER END
_self=$(realpath "$0")
_self_dir=$(dirname "$_self")
if [ "$1" == "install" ]; then
cp "${_self}" "${_self_dir}/.git/hooks/pre-commit"
echo "Installed pre-commit hook to '${_self_dir}/.git/hooks/pre-commit'."
else
declare -a FILES; readarray -t FILES < <(git diff --name-only --cached)
node "$PWD/tools/copyright.js" "${FILES[@]}"
fi
tests/.gitignore
+6
View File
@@ -0,0 +1,6 @@
# AUTOGENERATED COPYRIGHT HEADER START
# Copyright (C) NaN-NaN undefined
# Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
# AUTOGENERATED COPYRIGHT HEADER END
/stranded2
/beyond-frontier
tools/copyright.js
+581
View File
@@ -0,0 +1,581 @@
// AUTOGENERATED COPYRIGHT HEADER START
// Copyright (C) NaN-NaN undefined
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
// AUTOGENERATED COPYRIGHT HEADER END
const ignoreList = [
/^\.git$/gi,
/^cmake\/clang$/gi,
/^cmake\/version$/gi,
/^third-party$/gi,
]
const sectionStart = "AUTOGENERATED COPYRIGHT HEADER START";
const sectionEnd = "AUTOGENERATED COPYRIGHT HEADER END";
const formatStyleList = {
"#": {
files: [
"cmakelists.txt"
], exts: [
".clang-tidy",
".clang-format",
".cmake",
".editorconfig",
".gitignore",
".gitmodules",
".yml",
".sh"
],
prepend: [
`# ${sectionStart}`,
],
append: [
`# ${sectionEnd}`,
],
prefix: "# ",
suffix: "",
},
";": {
files: [
], exts: [
".iss",
".iss.in",
],
prepend: [
`; ${sectionStart}`,
],
append: [
`; ${sectionEnd}`,
],
prefix: "; ",
suffix: "",
},
"//": {
files: [
], exts: [
".c",
".c.in",
".cpp",
".cpp.in",
".h",
".h.in",
".hpp",
".hpp.in",
".js",
".rc",
".rc.in",
".effect"
],
prepend: [
`// ${sectionStart}`,
],
append: [
`// ${sectionEnd}`,
],
prefix: "// ",
suffix: "",
},
"<!---->": {
files: [
], exts: [
".htm",
".htm.in",
".html",
".html.in",
".xml",
".xml.in",
".plist",
".plist.in",
".pkgproj",
".pkgproj.in",
".md"
],
prepend: [
`<!-- ${sectionStart} -->`,
],
append: [
`<!-- ${sectionEnd} -->`,
],
prefix: "<!-- ",
suffix: " -->",
}
};
const commitFormat="%aI|%aN <%aE>"
const debug = false;
// --------------------------------------------------------------------------------
// End of easily modified area, best not touch things beyond here.
// --------------------------------------------------------------------------------
const CHILD_PROCESS = require("node:child_process");
const PROCESS = require("node:process");
const PATH = require("node:path");
const FS = require("node:fs");
const FSPROMISES = require("node:fs/promises");
const OS = require("os");
if (!debug)
console.debug = function() {}
class RateLimiter {
constructor(limit) {
Object.defineProperty(this, "_maximum", {
enumerable: true,
configurable: true,
writable: true,
value: 0
});
Object.defineProperty(this, "_available", {
enumerable: true,
configurable: true,
writable: true,
value: 0
});
Object.defineProperty(this, "_instances", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
if (!limit) {
this._maximum = Math.ceil(Math.max(1, OS.cpus().length / 3 * 2));
}
else {
this._maximum = limit;
}
this._available = this._maximum;
this._instances = [];
}
async queue(executor, ...args) {
while (this._available == 0) {
await Promise.race(this._instances);
}
--this._available;
const data = {};
data.task = new Promise((resolve, reject) => {
try {
if (executor.constructor.name == "Promise") {
executor.then((res) => {
resolve(res);
}, (err) => {
reject(err);
});
} else if (executor.constructor.name == "AsyncFunction") {
executor(...args).then((res) => {
resolve(res);
}, (err) => {
reject(err);
});
}
else {
resolve(executor(...args));
}
}
catch (ex) {
console.error(executor, executor.toString())
reject(ex);
}
});
data.solver = data.task.finally(() => {
const taskIndex = this._instances.indexOf(data.task);
if (taskIndex >= 0) {
this._instances.splice(taskIndex, 1);
}
const solverIndex = this._instances.indexOf(data.solver);
if (solverIndex >= 0) {
this._instances.splice(solverIndex, 1);
}
++this._available;
});
this._instances.push(data.solver);
return await data.solver;
}
}
let abortAllWork = false;
let gitRL = new RateLimiter(1);
let workRL = new RateLimiter();
let gitCurrentFiles;
let gitUserName;
let gitUserMail;
let gitDate = (new Date()).toISOString();
/** Run a process asynchronously, returning an array of messages.
*
* @param {*} path
* @param {*} args
* @param {*} options
* @returns [ErrorCode, [{type, data, text}, ...]]
*/
async function runProcessAsync(path, args, options) {
console.debug(arguments.callee.name, Array.from(arguments));
return await new Promise((resolve, reject) => {
let messages = [];
try {
let proc = CHILD_PROCESS.spawn(path, args, Object.assign({
"cwd": PROCESS.cwd(),
"encoding": "utf8",
}, options));
let pushOutput = function(type, chunk) {
if (!chunk) return;
let lastMessage = undefined;
if (messages.length == 0 || messages[messages.length - 1].type !== type) {
lastMessage = {
type: type,
chunks: [],
length: 0,
};
messages.push(lastMessage);
} else {
lastMessage = messages[messages.length - 1];
}
lastMessage.length += chunk.length;
lastMessage.chunks.push(chunk);
};
proc.stderr.on('data', (chunk) => { pushOutput("error", chunk); });
proc.stderr.on('close', () => { pushOutput("output", proc.stderr.read()); });
proc.stdout.on('data', (chunk) => { pushOutput("output", chunk); });
proc.stdout.on('close', () => { pushOutput("output", proc.stdout.read()); });
proc.on('error', (err) => {
reject(err);
})
proc.on('exit', (code) => {
// Merge chunks of messages into a single message.
for (let message of messages) {
message.data = Buffer.alloc(message.length);
let offset = 0;
for (let chunk of message.chunks) {
Buffer.from(chunk).copy(message.data, offset, 0);
offset += chunk.byteLength;
}
message.chunks = undefined;
message.length = undefined;
message.text = message.data.toString();
}
resolve([code, messages]);
});
} catch (ex) {
reject(ex);
}
});
}
async function git_isIgnored(path) {
console.debug(arguments.callee.name, Array.from(arguments));
let rpath = PATH.relative(process.cwd(), path).replaceAll(PATH.sep, PATH.posix.sep);
for (let ignore of ignoreList) {
if (ignore instanceof RegExp) {
if (ignore.global) {
let matches = rpath.matchAll(ignore);
for (let match of matches) {
return true;
}
} else {
if (rpath.match(ignore) !== null) {
return true;
}
}
} else if (rpath.startsWith(ignore)) {
return true;
}
}
let result = await gitRL.queue(runProcessAsync, "git", ["check-ignore", path], {});
return (result[0] == 0)
}
async function git_getCurrentAuthor() {
console.debug(arguments.callee.name, Array.from(arguments));
if (!gitUserName) {
let result = await gitRL.queue(runProcessAsync, "git", ["config", "user.name"], {});
gitUserName = result[1][0].text.split('\n')[0];
}
if (!gitUserMail) {
let result = await gitRL.queue(runProcessAsync, "git", ["config", "user.email"], {});
gitUserMail = result[1][0].text.split('\n')[0];
}
return commitFormat.replace("%aI", gitDate).replace("%aN", gitUserName).replace("%aE", gitUserMail);
}
async function git_isInCurrentCommit(file) {
console.debug(arguments.callee.name, Array.from(arguments));
if (!gitCurrentFiles) {
let files = [];
let result = await gitRL.queue(runProcessAsync, "git", ["diff", "--name-only", "--cached"], {});
if (result[0] != 0) {
let output = "";
for (let message of result[1]) {
output += message.text;
}
throw new Error(output);
} else {
lines = "";
for (let message of result[1]) {
if (message.type == "output")
lines += message.text;
}
files = lines.split(lines.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
}
gitCurrentFiles = new Set(files);
for (let file of files) {
if (file.length == 0) continue;
file = PATH.relative(PROCESS.cwd(), PATH.resolve(file));
gitCurrentFiles.add(file);
}
}
// Normalize file given to us.
file = PATH.relative(PROCESS.cwd(), PATH.resolve(file));
// Check and return if it's in here.
return gitCurrentFiles.has(file);
}
async function git_retrieveAuthors(file) {
console.debug(arguments.callee.name, Array.from(arguments));
// git --no-pager log --date-order --reverse "--format=format:%aI|%aN <%aE>" -- file
let lines = [];
let result = await gitRL.queue(runProcessAsync, "git", ["--no-pager", "log", "--follow", `--format=format:${commitFormat}`, "--", file], {});
if (result[0] != 0) {
let output = "";
for (let message of result[1]) {
output += message.text;
}
throw new Error(output);
} else {
let buffered = "";
for (let message of result[1]) {
if (message.type == "output")
buffered += message.text;
}
lines = buffered.split(buffered.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
}
if (await git_isInCurrentCommit(file)) {
lines.push(await git_getCurrentAuthor());
}
let authors = new Map();
for (let line of lines) {
let [date, name] = line.split("|");
let author = authors.get(name);
if (author) {
let dt = new Date(date)
if (author.from > dt)
author.from = dt;
if (author.to < dt)
author.to = dt;
} else {
authors.set(name, {
from: new Date(date),
to: new Date(date),
})
}
}
return authors;
}
async function generateCopyright(file) {
console.debug(arguments.callee.name, Array.from(arguments));
let authors = await git_retrieveAuthors(file)
authors = new Map([...authors].sort((a, b) => {
if (a[1].from < b[1].from) {
return -1;
} else if (a[1].from > b[1].from) {
return 1;
}
return 0;
}));
let lines = [];
for (let entry of authors) {
let from = entry[1].from.getUTCFullYear();
let to = entry[1].to.getUTCFullYear();
lines.push(`Copyright (C) ${from != to ? `${from}-${to}` : to} ${entry[0]}`);
}
return lines;
}
function makeHeader(file, copyright) {
console.debug(arguments.callee.name, Array.from(arguments));
let file_name = PATH.basename(file).toLocaleLowerCase();
let file_exts = file_name.substring(file_name.indexOf("."));
let styles = formatStyleList;
for (let key in styles) {
let style = [key, styles[key]];
if (style[1].files.includes(file_name)
|| style[1].files.includes(file)
|| style[1].exts.includes(file_exts)) {
let header = [];
header.push(...style[1].prepend);
for (let line of copyright) {
header.push(`${style[1].prefix}${line}${style[1].suffix}`);
}
header.push(...style[1].append);
return header;
}
}
throw new Error("Unrecognized file format.")
}
async function updateFile(file) {
console.debug(arguments.callee.name, Array.from(arguments));
await workRL.queue(async () => {
try {
if (abortAllWork) {
return;
}
// Copyright information.
let copyright = await generateCopyright(file);
let header = undefined;
try {
header = makeHeader(file, copyright);
} catch (ex) {
console.log(`Skipping file '${file}': ${ex.message}`);
return;
}
console.log(`Updating file '${file}'...`);
// File contents.
let content = await FSPROMISES.readFile(file);
let eol = (content.indexOf("\r\n") != -1 ? OS.EOL : "\n");
let insert = Buffer.from(header.join(eol) + eol);
// Find the starting point.
let startHeader = content.indexOf(sectionStart);
startHeader = content.lastIndexOf(eol, startHeader);
startHeader += Buffer.from(eol).byteLength;
// Find the ending point.
let endHeader = content.indexOf(sectionEnd);
endHeader = content.indexOf(eol, endHeader);
endHeader += Buffer.from(eol).byteLength;
if (abortAllWork) {
return;
}
let fd = await FSPROMISES.open(file, "w");
let fp = [];
if ((startHeader >= 0) && (endHeader > startHeader)) {
let pos = 0;
if (startHeader > 0) {
fd.write(content, 0, startHeader, 0);
pos += startHeader;
}
fd.write(insert, 0, undefined, pos);
pos += insert.byteLength;
fd.write(content, endHeader, undefined, pos);
} else {
fd.write(insert, 0, undefined, 0);
fd.write(content, 0, undefined, insert.byteLength);
}
await fd.close();
} catch (ex) {
console.error(`Error processing '${file}'!: ${ex}`);
abortAllWork = true;
PROCESS.exitCode = 1;
return;
}
});
}
async function scanPath(path) {
console.debug(arguments.callee.name, Array.from(arguments));
// Abort here if the user aborted the process, or if the path is ignored.
if (abortAllWork) {
return;
}
let promises = [];
await workRL.queue(async () => {
let files = await FSPROMISES.readdir(path, { "withFileTypes": true });
for (let file of files) {
if (abortAllWork) {
break;
}
let fullname = PATH.join(path, file.name);
if (await git_isIgnored(fullname)) {
console.log(`Ignoring path '${fullname}'...`);
continue;
}
if (file.isDirectory()) {
console.log(`Scanning path '${fullname}'...`);
promises.push(scanPath(fullname));
} else {
promises.push(updateFile(fullname));
}
}
});
await Promise.all(promises);
}
(async function () {
PROCESS.on("SIGINT", () => {
abortAllWork = true;
PROCESS.exitCode = 1;
console.log("Sanely aborting all pending work...");
})
const root_path = PROCESS.cwd();
let is_git_directory = false;
console.debug(root_path, PROCESS.argv, PROCESS.execArgv);
var args = PROCESS.argv.slice(2);
while (args.length > 0) {
// Try to place ourselves where git actually is.
while (!is_git_directory) {
if (abortAllWork) {
return;
}
let entries = await FSPROMISES.readdir(PROCESS.cwd());
if (entries.includes(".git")) {
console.log(`Found .git at '${process.cwd()}'.`);
is_git_directory = true;
} else {
PROCESS.chdir(PATH.resolve(PATH.join(PROCESS.cwd(), "..")));
}
}
// If that failed, we just exit now.
if (!is_git_directory) {
console.error("Failed to figure out where git is, is this even a repository?");
PROCESS.exitCode = 1;
return;
}
// Then proceed with normal work.
let path = PATH.normalize(PATH.relative(process.cwd(), PATH.resolve(root_path, args[0])));
// If we're handed a path, scan said path, otherwise update the file itself.
let pathStat = await FSPROMISES.stat(path);
if (pathStat) {
if (await git_isIgnored(path)) {
console.log(`Ignoring path '${path}'...`);
} else if(pathStat.isDirectory()) {
console.log(`Scanning path '${path}'...`);
await scanPath(path);
} else {
await updateFile(path);
}
}
// Slice off the first argument and continue.
args = args.slice(1);
}
console.log("Done");
})();