BlitzNext/compiler/lib/toker.cpp

#include "toker.hpp"
#include <cctype>
#include "ex.hpp"

#include <stdutil.hpp>

int Toker::chars_toked;

static std::map<std::string, int> alphaTokes, lowerTokes;

static void makeKeywords()
{
	static bool made;
	if (made)
		return;

	alphaTokes["Dim"]          = DIM;
	alphaTokes["Goto"]         = GOTO;
	alphaTokes["Gosub"]        = GOSUB;
	alphaTokes["Return"]       = RETURN;
	alphaTokes["Exit"]         = EXIT;
	alphaTokes["If"]           = IF;
	alphaTokes["Then"]         = THEN;
	alphaTokes["Else"]         = ELSE;
	alphaTokes["EndIf"]        = ENDIF;
	alphaTokes["End If"]       = ENDIF;
	alphaTokes["ElseIf"]       = ELSEIF;
	alphaTokes["Else If"]      = ELSEIF;
	alphaTokes["While"]        = WHILE;
	alphaTokes["Wend"]         = WEND;
	alphaTokes["For"]          = FOR;
	alphaTokes["To"]           = TO;
	alphaTokes["Step"]         = STEP;
	alphaTokes["Next"]         = NEXT;
	alphaTokes["Function"]     = FUNCTION;
	alphaTokes["End Function"] = ENDFUNCTION;
	alphaTokes["Type"]         = TYPE;
	alphaTokes["End Type"]     = ENDTYPE;
	alphaTokes["Each"]         = EACH;
	alphaTokes["Local"]        = LOCAL;
	alphaTokes["Global"]       = GLOBAL;
	alphaTokes["Field"]        = FIELD;
	alphaTokes["Const"]        = BBCONST;
	alphaTokes["Select"]       = SELECT;
	alphaTokes["Case"]         = CASE;
	alphaTokes["Default"]      = DEFAULT;
	alphaTokes["End Select"]   = ENDSELECT;
	alphaTokes["Repeat"]       = REPEAT;
	alphaTokes["Until"]        = UNTIL;
	alphaTokes["Forever"]      = FOREVER;
	alphaTokes["Data"]         = DATA;
	alphaTokes["Read"]         = READ;
	alphaTokes["Restore"]      = RESTORE;
	alphaTokes["Abs"]          = ABS;
	alphaTokes["Sgn"]          = SGN;
	alphaTokes["Mod"]          = MOD;
	alphaTokes["Pi"]           = PI;
	alphaTokes["True"]         = BBTRUE;
	alphaTokes["False"]        = BBFALSE;
	alphaTokes["Int"]          = BBINT;
	alphaTokes["Float"]        = BBFLOAT;
	alphaTokes["Str"]          = BBSTR;
	alphaTokes["Include"]      = INCLUDE;

	alphaTokes["New"]    = BBNEW;
	alphaTokes["Delete"] = BBDELETE;
	alphaTokes["First"]  = FIRST;
	alphaTokes["Last"]   = LAST;
	alphaTokes["Insert"] = INSERT;
	alphaTokes["Before"] = BEFORE;
	alphaTokes["After"]  = AFTER;
	alphaTokes["Null"]   = BBNULL;
	alphaTokes["Object"] = OBJECT;
	alphaTokes["Handle"] = BBHANDLE;

	alphaTokes["And"] = AND;
	alphaTokes["Or"]  = OR;
	alphaTokes["Xor"] = XOR;
	alphaTokes["Not"] = NOT;
	alphaTokes["Shl"] = SHL;
	alphaTokes["Shr"] = SHR;
	alphaTokes["Sar"] = SAR;

	std::map<std::string, int>::const_iterator it;
	for (it = alphaTokes.begin(); it != alphaTokes.end(); ++it) {
		lowerTokes[tolower(it->first)] = it->second;
	}
	made = true;
}

Toker::Toker(std::istream& in) : in(in), curr_row(-1)
{
	makeKeywords();
	nextline();
}

std::map<std::string, int>& Toker::getKeywords()
{
	makeKeywords();
	return alphaTokes;
}

int Toker::pos()
{
	return ((curr_row) << 16) | (tokes[curr_toke].from);
}

int Toker::curr()
{
	return tokes[curr_toke].n;
}

std::string Toker::text()
{
	int from = tokes[curr_toke].from, to = tokes[curr_toke].to;
	return line.substr(from, to - from);
}

int Toker::lookAhead(int n)
{
	return tokes[curr_toke + n].n;
}

void Toker::nextline()
{
	++curr_row;
	curr_toke = 0;
	tokes.clear();
	if (in.eof()) {
		line.resize(1);
		line[0] = EOF;
		tokes.push_back(Toke(EOF, 0, 1));
		return;
	}

	getline(in, line);
	line += '\n';
	chars_toked += line.size();

	for (int k = 0; k < line.size();) {
		int c = line[k], from = k;
		if (c == '\n') {
			tokes.push_back(Toke(c, from, ++k));
			continue;
		}
		if (isspace(c)) {
			++k;
			continue;
		}
		if (c == ';') {
			for (++k; line[k] != '\n'; ++k) {
			}
			continue;
		}
		if (c == '.' && isdigit(line[k + 1])) {
			for (k += 2; isdigit(line[k]); ++k) {
			}
			tokes.push_back(Toke(FLOATCONST, from, k));
			continue;
		}
		if (isdigit(c)) {
			for (++k; isdigit(line[k]); ++k) {
			}
			if (line[k] == '.') {
				for (++k; isdigit(line[k]); ++k) {
				}
				tokes.push_back(Toke(FLOATCONST, from, k));
				continue;
			}
			tokes.push_back(Toke(INTCONST, from, k));
			continue;
		}
		if (c == '%' && (line[k + 1] == '0' || line[k + 1] == '1')) {
			for (k += 2; line[k] == '0' || line[k] == '1'; ++k) {
			}
			tokes.push_back(Toke(BINCONST, from, k));
			continue;
		}
		if (c == '$' && isxdigit(line[k + 1])) {
			for (k += 2; isxdigit(line[k]); ++k) {
			}
			tokes.push_back(Toke(HEXCONST, from, k));
			continue;
		}
		if (isalpha(c)) {
			for (++k; isalnum(line[k]) || line[k] == '_'; ++k) {
			}

			std::string ident = tolower(line.substr(from, k - from));

			if (line[k] == ' ' && isalpha(line[k + 1])) {
				int t = k;
				for (t += 2; isalnum(line[t]) || line[t] == '_'; ++t) {
				}
				std::string s = tolower(line.substr(from, t - from));
				if (lowerTokes.find(s) != lowerTokes.end()) {
					k     = t;
					ident = s;
				}
			}

			std::map<std::string, int>::iterator it = lowerTokes.find(ident);

			if (it == lowerTokes.end()) {
				for (int n = from; n < k; ++n)
					line[n] = tolower(line[n]);
				tokes.push_back(Toke(IDENT, from, k));
				continue;
			}

			tokes.push_back(Toke(it->second, from, k));
			continue;
		}
		if (c == '\"') {
			for (++k; line[k] != '\"' && line[k] != '\n'; ++k) {
			}
			if (line[k] == '\"')
				++k;
			tokes.push_back(Toke(STRINGCONST, from, k));
			continue;
		}
		int n = line[k + 1];
		if ((c == '<' && n == '>') || (c == '>' && n == '<')) {
			tokes.push_back(Toke(NE, from, k += 2));
			continue;
		}
		if ((c == '<' && n == '=') || (c == '=' && n == '<')) {
			tokes.push_back(Toke(LE, from, k += 2));
			continue;
		}
		if ((c == '>' && n == '=') || (c == '=' && n == '>')) {
			tokes.push_back(Toke(GE, from, k += 2));
			continue;
		}
		tokes.push_back(Toke(c, from, ++k));
	}
	if (!tokes.size())
		exit(0);
}

int Toker::next()
{
	if (++curr_toke == tokes.size())
		nextline();
	return curr();
}