code_compiler: Use isspace, iscntrl and isprint for tests

projects/code_compiler/source/lexer.cpp

@@ -17,11 +17,6 @@
 #include "lexer.hpp"
 #include <codecvt>
 
-char g_whitespaceCharacters[] = {
-	' ',
-	'\t',
-};
-
 std::pair<char, BlitzLLVM::Lexer::Token> g_symbolCharacters[] = {
 	//{ '\"', BlitzLLVM::Lexer::Token::TokenDoubleQuote }, // Has special meaning.
 	{ '+', BlitzLLVM::Lexer::Token::TokenPlus },
@@ -82,16 +77,15 @@ std::pair<BlitzLLVM::Lexer::Token, std::string> BlitzLLVM::Lexer::GetNextToken()
 			break;
 		} else if (!m_isStringMode && !m_isTextMode && !m_isNumberMode) {
 			// Whitespace
-			bool isWhitespace = false;
+			if (isspace(chr))
-			for (char v : g_whitespaceCharacters) {
-				if (v == chr) {
-					isWhitespace = true;
-					break;
-				}
-			}
-			if (isWhitespace)
 				continue;
 
+			// Control Code
+			if (iscntrl(chr)) {
+				tkn = Token::TokenUnknown;
+				buf = chr;
+			}
+
 			// Symbol
 			for (auto v : g_symbolCharacters) {
 				if (v.first == chr) {
@@ -137,6 +131,10 @@ std::pair<BlitzLLVM::Lexer::Token, std::string> BlitzLLVM::Lexer::GetNextToken()
 				m_isStringMode = false;
 				tkn = Token::TokenQuotedText;
 				break;
+			} else if (iscntrl(chr) || !isprint(chr)) {
+				m_fileStream.putback(chr);
+				m_isStringMode = false;
+				break;
 			} else {
 				buf += chr;
 			}