diff --git a/projects/code_compiler/source/lexer.cpp b/projects/code_compiler/source/lexer.cpp index 1b24958..e8313c6 100644 --- a/projects/code_compiler/source/lexer.cpp +++ b/projects/code_compiler/source/lexer.cpp @@ -17,11 +17,6 @@ #include "lexer.hpp" #include -char g_whitespaceCharacters[] = { - ' ', - '\t', -}; - std::pair g_symbolCharacters[] = { //{ '\"', BlitzLLVM::Lexer::Token::TokenDoubleQuote }, // Has special meaning. { '+', BlitzLLVM::Lexer::Token::TokenPlus }, @@ -82,16 +77,15 @@ std::pair BlitzLLVM::Lexer::GetNextToken() break; } else if (!m_isStringMode && !m_isTextMode && !m_isNumberMode) { // Whitespace - bool isWhitespace = false; - for (char v : g_whitespaceCharacters) { - if (v == chr) { - isWhitespace = true; - break; - } - } - if (isWhitespace) + if (isspace(chr)) continue; + // Control Code + if (iscntrl(chr)) { + tkn = Token::TokenUnknown; + buf = chr; + } + // Symbol for (auto v : g_symbolCharacters) { if (v.first == chr) { @@ -137,6 +131,10 @@ std::pair BlitzLLVM::Lexer::GetNextToken() m_isStringMode = false; tkn = Token::TokenQuotedText; break; + } else if (iscntrl(chr) || !isprint(chr)) { + m_fileStream.putback(chr); + m_isStringMode = false; + break; } else { buf += chr; }