Latest stuff, rewriting lexer
This commit is contained in:
Michael Fabian 'Xaymar' Dirks
@@ -0,0 +1,581 @@
|
||||
// AUTOGENERATED COPYRIGHT HEADER START
|
||||
// Copyright (C) NaN-NaN undefined
|
||||
// Copyright (C) 2024 Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
|
||||
// AUTOGENERATED COPYRIGHT HEADER END
|
||||
|
||||
const ignoreList = [
|
||||
/^\.git$/gi,
|
||||
/^cmake\/clang$/gi,
|
||||
/^cmake\/version$/gi,
|
||||
/^third-party$/gi,
|
||||
]
|
||||
const sectionStart = "AUTOGENERATED COPYRIGHT HEADER START";
|
||||
const sectionEnd = "AUTOGENERATED COPYRIGHT HEADER END";
|
||||
const formatStyleList = {
|
||||
"#": {
|
||||
files: [
|
||||
"cmakelists.txt"
|
||||
], exts: [
|
||||
".clang-tidy",
|
||||
".clang-format",
|
||||
".cmake",
|
||||
".editorconfig",
|
||||
".gitignore",
|
||||
".gitmodules",
|
||||
".yml",
|
||||
".sh"
|
||||
],
|
||||
prepend: [
|
||||
`# ${sectionStart}`,
|
||||
],
|
||||
append: [
|
||||
`# ${sectionEnd}`,
|
||||
],
|
||||
prefix: "# ",
|
||||
suffix: "",
|
||||
},
|
||||
";": {
|
||||
files: [
|
||||
], exts: [
|
||||
".iss",
|
||||
".iss.in",
|
||||
],
|
||||
prepend: [
|
||||
`; ${sectionStart}`,
|
||||
],
|
||||
append: [
|
||||
`; ${sectionEnd}`,
|
||||
],
|
||||
prefix: "; ",
|
||||
suffix: "",
|
||||
},
|
||||
"//": {
|
||||
files: [
|
||||
], exts: [
|
||||
".c",
|
||||
".c.in",
|
||||
".cpp",
|
||||
".cpp.in",
|
||||
".h",
|
||||
".h.in",
|
||||
".hpp",
|
||||
".hpp.in",
|
||||
".js",
|
||||
".rc",
|
||||
".rc.in",
|
||||
".effect"
|
||||
],
|
||||
prepend: [
|
||||
`// ${sectionStart}`,
|
||||
],
|
||||
append: [
|
||||
`// ${sectionEnd}`,
|
||||
],
|
||||
prefix: "// ",
|
||||
suffix: "",
|
||||
},
|
||||
"<!---->": {
|
||||
files: [
|
||||
], exts: [
|
||||
".htm",
|
||||
".htm.in",
|
||||
".html",
|
||||
".html.in",
|
||||
".xml",
|
||||
".xml.in",
|
||||
".plist",
|
||||
".plist.in",
|
||||
".pkgproj",
|
||||
".pkgproj.in",
|
||||
".md"
|
||||
],
|
||||
prepend: [
|
||||
`<!-- ${sectionStart} -->`,
|
||||
],
|
||||
append: [
|
||||
`<!-- ${sectionEnd} -->`,
|
||||
],
|
||||
prefix: "<!-- ",
|
||||
suffix: " -->",
|
||||
}
|
||||
};
|
||||
const commitFormat="%aI|%aN <%aE>"
|
||||
const debug = false;
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// End of easily modified area, best not touch things beyond here.
|
||||
// --------------------------------------------------------------------------------
|
||||
const CHILD_PROCESS = require("node:child_process");
|
||||
const PROCESS = require("node:process");
|
||||
const PATH = require("node:path");
|
||||
const FS = require("node:fs");
|
||||
const FSPROMISES = require("node:fs/promises");
|
||||
const OS = require("os");
|
||||
|
||||
if (!debug)
|
||||
console.debug = function() {}
|
||||
|
||||
class RateLimiter {
|
||||
constructor(limit) {
|
||||
Object.defineProperty(this, "_maximum", {
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: 0
|
||||
});
|
||||
Object.defineProperty(this, "_available", {
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: 0
|
||||
});
|
||||
Object.defineProperty(this, "_instances", {
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: void 0
|
||||
});
|
||||
if (!limit) {
|
||||
this._maximum = Math.ceil(Math.max(1, OS.cpus().length / 3 * 2));
|
||||
}
|
||||
else {
|
||||
this._maximum = limit;
|
||||
}
|
||||
this._available = this._maximum;
|
||||
this._instances = [];
|
||||
}
|
||||
async queue(executor, ...args) {
|
||||
while (this._available == 0) {
|
||||
await Promise.race(this._instances);
|
||||
}
|
||||
--this._available;
|
||||
const data = {};
|
||||
data.task = new Promise((resolve, reject) => {
|
||||
try {
|
||||
if (executor.constructor.name == "Promise") {
|
||||
executor.then((res) => {
|
||||
resolve(res);
|
||||
}, (err) => {
|
||||
reject(err);
|
||||
});
|
||||
} else if (executor.constructor.name == "AsyncFunction") {
|
||||
executor(...args).then((res) => {
|
||||
resolve(res);
|
||||
}, (err) => {
|
||||
reject(err);
|
||||
});
|
||||
}
|
||||
else {
|
||||
resolve(executor(...args));
|
||||
}
|
||||
}
|
||||
catch (ex) {
|
||||
console.error(executor, executor.toString())
|
||||
reject(ex);
|
||||
}
|
||||
});
|
||||
data.solver = data.task.finally(() => {
|
||||
const taskIndex = this._instances.indexOf(data.task);
|
||||
if (taskIndex >= 0) {
|
||||
this._instances.splice(taskIndex, 1);
|
||||
}
|
||||
const solverIndex = this._instances.indexOf(data.solver);
|
||||
if (solverIndex >= 0) {
|
||||
this._instances.splice(solverIndex, 1);
|
||||
}
|
||||
++this._available;
|
||||
});
|
||||
this._instances.push(data.solver);
|
||||
return await data.solver;
|
||||
}
|
||||
}
|
||||
|
||||
let abortAllWork = false;
|
||||
let gitRL = new RateLimiter(1);
|
||||
let workRL = new RateLimiter();
|
||||
let gitCurrentFiles;
|
||||
let gitUserName;
|
||||
let gitUserMail;
|
||||
let gitDate = (new Date()).toISOString();
|
||||
|
||||
/** Run a process asynchronously, returning an array of messages.
|
||||
*
|
||||
* @param {*} path
|
||||
* @param {*} args
|
||||
* @param {*} options
|
||||
* @returns [ErrorCode, [{type, data, text}, ...]]
|
||||
*/
|
||||
async function runProcessAsync(path, args, options) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
return await new Promise((resolve, reject) => {
|
||||
let messages = [];
|
||||
|
||||
try {
|
||||
let proc = CHILD_PROCESS.spawn(path, args, Object.assign({
|
||||
"cwd": PROCESS.cwd(),
|
||||
"encoding": "utf8",
|
||||
}, options));
|
||||
|
||||
let pushOutput = function(type, chunk) {
|
||||
if (!chunk) return;
|
||||
|
||||
let lastMessage = undefined;
|
||||
if (messages.length == 0 || messages[messages.length - 1].type !== type) {
|
||||
lastMessage = {
|
||||
type: type,
|
||||
chunks: [],
|
||||
length: 0,
|
||||
};
|
||||
messages.push(lastMessage);
|
||||
} else {
|
||||
lastMessage = messages[messages.length - 1];
|
||||
}
|
||||
lastMessage.length += chunk.length;
|
||||
lastMessage.chunks.push(chunk);
|
||||
};
|
||||
|
||||
proc.stderr.on('data', (chunk) => { pushOutput("error", chunk); });
|
||||
proc.stderr.on('close', () => { pushOutput("output", proc.stderr.read()); });
|
||||
proc.stdout.on('data', (chunk) => { pushOutput("output", chunk); });
|
||||
proc.stdout.on('close', () => { pushOutput("output", proc.stdout.read()); });
|
||||
proc.on('error', (err) => {
|
||||
reject(err);
|
||||
})
|
||||
proc.on('exit', (code) => {
|
||||
// Merge chunks of messages into a single message.
|
||||
for (let message of messages) {
|
||||
message.data = Buffer.alloc(message.length);
|
||||
let offset = 0;
|
||||
for (let chunk of message.chunks) {
|
||||
Buffer.from(chunk).copy(message.data, offset, 0);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
message.chunks = undefined;
|
||||
message.length = undefined;
|
||||
message.text = message.data.toString();
|
||||
}
|
||||
resolve([code, messages]);
|
||||
});
|
||||
} catch (ex) {
|
||||
reject(ex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function git_isIgnored(path) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
let rpath = PATH.relative(process.cwd(), path).replaceAll(PATH.sep, PATH.posix.sep);
|
||||
for (let ignore of ignoreList) {
|
||||
if (ignore instanceof RegExp) {
|
||||
if (ignore.global) {
|
||||
let matches = rpath.matchAll(ignore);
|
||||
for (let match of matches) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
if (rpath.match(ignore) !== null) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} else if (rpath.startsWith(ignore)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
let result = await gitRL.queue(runProcessAsync, "git", ["check-ignore", path], {});
|
||||
return (result[0] == 0)
|
||||
}
|
||||
|
||||
async function git_getCurrentAuthor() {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
if (!gitUserName) {
|
||||
let result = await gitRL.queue(runProcessAsync, "git", ["config", "user.name"], {});
|
||||
gitUserName = result[1][0].text.split('\n')[0];
|
||||
}
|
||||
if (!gitUserMail) {
|
||||
let result = await gitRL.queue(runProcessAsync, "git", ["config", "user.email"], {});
|
||||
gitUserMail = result[1][0].text.split('\n')[0];
|
||||
}
|
||||
return commitFormat.replace("%aI", gitDate).replace("%aN", gitUserName).replace("%aE", gitUserMail);
|
||||
}
|
||||
|
||||
async function git_isInCurrentCommit(file) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
if (!gitCurrentFiles) {
|
||||
let files = [];
|
||||
let result = await gitRL.queue(runProcessAsync, "git", ["diff", "--name-only", "--cached"], {});
|
||||
if (result[0] != 0) {
|
||||
let output = "";
|
||||
for (let message of result[1]) {
|
||||
output += message.text;
|
||||
}
|
||||
throw new Error(output);
|
||||
} else {
|
||||
lines = "";
|
||||
for (let message of result[1]) {
|
||||
if (message.type == "output")
|
||||
lines += message.text;
|
||||
}
|
||||
files = lines.split(lines.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
|
||||
}
|
||||
|
||||
gitCurrentFiles = new Set(files);
|
||||
for (let file of files) {
|
||||
if (file.length == 0) continue;
|
||||
file = PATH.relative(PROCESS.cwd(), PATH.resolve(file));
|
||||
gitCurrentFiles.add(file);
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize file given to us.
|
||||
file = PATH.relative(PROCESS.cwd(), PATH.resolve(file));
|
||||
|
||||
// Check and return if it's in here.
|
||||
return gitCurrentFiles.has(file);
|
||||
}
|
||||
|
||||
async function git_retrieveAuthors(file) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
// git --no-pager log --date-order --reverse "--format=format:%aI|%aN <%aE>" -- file
|
||||
let lines = [];
|
||||
let result = await gitRL.queue(runProcessAsync, "git", ["--no-pager", "log", "--follow", `--format=format:${commitFormat}`, "--", file], {});
|
||||
if (result[0] != 0) {
|
||||
let output = "";
|
||||
for (let message of result[1]) {
|
||||
output += message.text;
|
||||
}
|
||||
throw new Error(output);
|
||||
} else {
|
||||
let buffered = "";
|
||||
for (let message of result[1]) {
|
||||
if (message.type == "output")
|
||||
buffered += message.text;
|
||||
}
|
||||
lines = buffered.split(buffered.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
|
||||
}
|
||||
|
||||
if (await git_isInCurrentCommit(file)) {
|
||||
lines.push(await git_getCurrentAuthor());
|
||||
}
|
||||
|
||||
let authors = new Map();
|
||||
for (let line of lines) {
|
||||
let [date, name] = line.split("|");
|
||||
|
||||
let author = authors.get(name);
|
||||
if (author) {
|
||||
let dt = new Date(date)
|
||||
if (author.from > dt)
|
||||
author.from = dt;
|
||||
if (author.to < dt)
|
||||
author.to = dt;
|
||||
} else {
|
||||
authors.set(name, {
|
||||
from: new Date(date),
|
||||
to: new Date(date),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return authors;
|
||||
}
|
||||
|
||||
async function generateCopyright(file) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
let authors = await git_retrieveAuthors(file)
|
||||
authors = new Map([...authors].sort((a, b) => {
|
||||
if (a[1].from < b[1].from) {
|
||||
return -1;
|
||||
} else if (a[1].from > b[1].from) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}));
|
||||
let lines = [];
|
||||
for (let entry of authors) {
|
||||
let from = entry[1].from.getUTCFullYear();
|
||||
let to = entry[1].to.getUTCFullYear();
|
||||
lines.push(`Copyright (C) ${from != to ? `${from}-${to}` : to} ${entry[0]}`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function makeHeader(file, copyright) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
let file_name = PATH.basename(file).toLocaleLowerCase();
|
||||
let file_exts = file_name.substring(file_name.indexOf("."));
|
||||
|
||||
let styles = formatStyleList;
|
||||
|
||||
for (let key in styles) {
|
||||
let style = [key, styles[key]];
|
||||
if (style[1].files.includes(file_name)
|
||||
|| style[1].files.includes(file)
|
||||
|| style[1].exts.includes(file_exts)) {
|
||||
let header = [];
|
||||
header.push(...style[1].prepend);
|
||||
for (let line of copyright) {
|
||||
header.push(`${style[1].prefix}${line}${style[1].suffix}`);
|
||||
}
|
||||
header.push(...style[1].append);
|
||||
return header;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Unrecognized file format.")
|
||||
}
|
||||
|
||||
async function updateFile(file) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
await workRL.queue(async () => {
|
||||
try {
|
||||
if (abortAllWork) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Copyright information.
|
||||
let copyright = await generateCopyright(file);
|
||||
let header = undefined;
|
||||
try {
|
||||
header = makeHeader(file, copyright);
|
||||
} catch (ex) {
|
||||
console.log(`Skipping file '${file}': ${ex.message}`);
|
||||
return;
|
||||
}
|
||||
console.log(`Updating file '${file}'...`);
|
||||
|
||||
// File contents.
|
||||
let content = await FSPROMISES.readFile(file);
|
||||
let eol = (content.indexOf("\r\n") != -1 ? OS.EOL : "\n");
|
||||
let insert = Buffer.from(header.join(eol) + eol);
|
||||
|
||||
// Find the starting point.
|
||||
let startHeader = content.indexOf(sectionStart);
|
||||
startHeader = content.lastIndexOf(eol, startHeader);
|
||||
startHeader += Buffer.from(eol).byteLength;
|
||||
|
||||
// Find the ending point.
|
||||
let endHeader = content.indexOf(sectionEnd);
|
||||
endHeader = content.indexOf(eol, endHeader);
|
||||
endHeader += Buffer.from(eol).byteLength;
|
||||
|
||||
if (abortAllWork) {
|
||||
return;
|
||||
}
|
||||
|
||||
let fd = await FSPROMISES.open(file, "w");
|
||||
let fp = [];
|
||||
if ((startHeader >= 0) && (endHeader > startHeader)) {
|
||||
let pos = 0;
|
||||
if (startHeader > 0) {
|
||||
fd.write(content, 0, startHeader, 0);
|
||||
pos += startHeader;
|
||||
}
|
||||
fd.write(insert, 0, undefined, pos);
|
||||
pos += insert.byteLength;
|
||||
fd.write(content, endHeader, undefined, pos);
|
||||
} else {
|
||||
fd.write(insert, 0, undefined, 0);
|
||||
fd.write(content, 0, undefined, insert.byteLength);
|
||||
}
|
||||
await fd.close();
|
||||
} catch (ex) {
|
||||
console.error(`Error processing '${file}'!: ${ex}`);
|
||||
abortAllWork = true;
|
||||
PROCESS.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function scanPath(path) {
|
||||
console.debug(arguments.callee.name, Array.from(arguments));
|
||||
// Abort here if the user aborted the process, or if the path is ignored.
|
||||
if (abortAllWork) {
|
||||
return;
|
||||
}
|
||||
|
||||
let promises = [];
|
||||
|
||||
await workRL.queue(async () => {
|
||||
let files = await FSPROMISES.readdir(path, { "withFileTypes": true });
|
||||
for (let file of files) {
|
||||
if (abortAllWork) {
|
||||
break;
|
||||
}
|
||||
|
||||
let fullname = PATH.join(path, file.name);
|
||||
if (await git_isIgnored(fullname)) {
|
||||
console.log(`Ignoring path '${fullname}'...`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (file.isDirectory()) {
|
||||
console.log(`Scanning path '${fullname}'...`);
|
||||
promises.push(scanPath(fullname));
|
||||
} else {
|
||||
promises.push(updateFile(fullname));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
(async function () {
|
||||
PROCESS.on("SIGINT", () => {
|
||||
abortAllWork = true;
|
||||
PROCESS.exitCode = 1;
|
||||
console.log("Sanely aborting all pending work...");
|
||||
})
|
||||
|
||||
const root_path = PROCESS.cwd();
|
||||
let is_git_directory = false;
|
||||
|
||||
console.debug(root_path, PROCESS.argv, PROCESS.execArgv);
|
||||
|
||||
var args = PROCESS.argv.slice(2);
|
||||
while (args.length > 0) {
|
||||
// Try to place ourselves where git actually is.
|
||||
while (!is_git_directory) {
|
||||
if (abortAllWork) {
|
||||
return;
|
||||
}
|
||||
|
||||
let entries = await FSPROMISES.readdir(PROCESS.cwd());
|
||||
if (entries.includes(".git")) {
|
||||
console.log(`Found .git at '${process.cwd()}'.`);
|
||||
is_git_directory = true;
|
||||
} else {
|
||||
PROCESS.chdir(PATH.resolve(PATH.join(PROCESS.cwd(), "..")));
|
||||
}
|
||||
}
|
||||
|
||||
// If that failed, we just exit now.
|
||||
if (!is_git_directory) {
|
||||
console.error("Failed to figure out where git is, is this even a repository?");
|
||||
PROCESS.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Then proceed with normal work.
|
||||
let path = PATH.normalize(PATH.relative(process.cwd(), PATH.resolve(root_path, args[0])));
|
||||
|
||||
// If we're handed a path, scan said path, otherwise update the file itself.
|
||||
let pathStat = await FSPROMISES.stat(path);
|
||||
if (pathStat) {
|
||||
if (await git_isIgnored(path)) {
|
||||
console.log(`Ignoring path '${path}'...`);
|
||||
} else if(pathStat.isDirectory()) {
|
||||
console.log(`Scanning path '${path}'...`);
|
||||
await scanPath(path);
|
||||
} else {
|
||||
await updateFile(path);
|
||||
}
|
||||
}
|
||||
|
||||
// Slice off the first argument and continue.
|
||||
args = args.slice(1);
|
||||
}
|
||||
console.log("Done");
|
||||
})();
|
||||
Reference in New Issue
Block a user