xaymar/js-encoding-samples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drastically speed things up with parallelization

Browse Source
This commit is contained in:
Michael Fabian 'Xaymar' Dirks
2020-10-25 19:16:04 +01:00
parent 3b0cdeaaac
commit ccc0483115
3 changed files with 112 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
encoder_h264_nvenc.js
+8 -8
View File
@@ -11,7 +11,7 @@ class h264_nvenc extends encoder {
}
available() {
console.time("Checking...");
if (global.debug) console.time("Checking...");
let res = ff.ffmpegSync([
"-hide_banner", "-v", "quiet",
"-f", "lavfi",
@@ -20,7 +20,7 @@ class h264_nvenc extends encoder {
"-f", "null",
"-"
]);
console.timeEnd("Checking...");
if (global.debug) console.timeEnd("Checking...");
if (res.status != 0) {
return false;
}
@@ -28,7 +28,7 @@ class h264_nvenc extends encoder {
}
available() {
console.time("Checking...");
if (global.debug) console.time("Checking...");
let res = this.ffmpeg.ffmpegSync([
"-hide_banner", "-v", "error",
"-f", "lavfi",
@@ -37,7 +37,7 @@ class h264_nvenc extends encoder {
"-f", "null",
"-"
]);
console.timeEnd("Checking...");
if (global.debug) console.timeEnd("Checking...");
if (res.status != 0) {
return false;
}
@@ -57,7 +57,7 @@ class h264_nvenc extends encoder {
return `${name};version=${version}`;
}
console.time("Generating...");
if (global.debug) console.time("Generating...");
this.indexes = {};
this.combinations = [];
for (let preset of this.settings.presets) {
@@ -127,8 +127,8 @@ class h264_nvenc extends encoder {
JSON.stringify(this.indexes, null, null),
{encoding: "utf8"}
);
console.timeEnd("Generating...");
console.log(`Combinations: ${this.combinations.length}`)
if (global.debug) console.timeEnd("Generating...");
if (global.debug) console.log(`Combinations: ${this.combinations.length}`)
}
pool() {
@@ -140,7 +140,7 @@ class h264_nvenc extends encoder {
}
extra() {
return ["-gpu", settings.gpu];
return ["-gpu", this.settings.gpu];
}
}
encoder_libx264.js
+5 -5
View File
@@ -34,7 +34,7 @@ class libx264 extends encoder {
}
available() {
console.time("Checking...");
if (global.debug) console.time("Checking...");
let res = this.ffmpeg.ffmpegSync([
"-hide_banner", "-v", "error",
"-f", "lavfi",
@@ -43,7 +43,7 @@ class libx264 extends encoder {
"-f", "null",
"-"
]);
console.timeEnd("Checking...");
if (global.debug) console.timeEnd("Checking...");
if (res.status != 0) {
return false;
}
@@ -63,7 +63,7 @@ class libx264 extends encoder {
return `${name};version=${version}`;
}
console.time("Generating...");
if (global.debug) console.time("Generating...");
this.indexes = {};
this.combinations = [];
for (let preset of this.settings.presets) {
@@ -99,8 +99,8 @@ class libx264 extends encoder {
JSON.stringify(this.indexes, null, null),
{encoding: "utf8"}
);
console.timeEnd("Generating...");
console.log(`Combinations: ${this.count()}`)
if (global.debug) console.timeEnd("Generating...");
if (global.debug) console.log(`Combinations: ${this.count()}`)
}
pool() {
ves.js
+99 -97
View File
@@ -121,7 +121,7 @@ async function load_videos(config, ff) { // Load Videos
data.name = name;
data.info = json;
data.caches = new Map();
data.caches = {};
// File Information
data.file_name = `${name}.mkv`;
@@ -285,96 +285,98 @@ async function transcode(config, ff, videos, encoders) {
// Build queues per cache.
console.group("Queueing...")
console.time("Subtotal");
let promises = [];
for (let video_key of videos.keys()) {
console.group(video_key);
console.time(video_key);
let video = videos.get(video_key);
for (let cache_key of video.caches.keys()) {
let cache = video.caches.get(cache_key);
console.time(cache_key);
promises.push(new Promise(async (resolve, reject) => {
let video = videos.get(video_key);
let video_promises = [];
for (let encoder_key of encoders.keys()) {
let encoder = encoders.get(encoder_key);
console.time(video_key);
for (let cache_key of video.caches.keys()) {
video_promises.push(new Promise(async (resolve1) => {
let cache = video.caches.get(cache_key);
let queue_promises = [];
let queue_commands = new poolqueue();
let queue_files = new poolqueue();
for (let idx = 0; idx < encoder.count(); idx++) {
let command_promises = [];
let command = encoder.get(idx, cache.width, cache.height, cache.framerate);
for (let bitrate of config.options.bitrates) {
for (let kfinterval of config.options.keyframeinterval) {
command_promises.push(new Promise((resolve, reject) => {
let file = path.join(
config.paths.output,
video.name,
cache_key,
encoder_key,
bitrate.toFixed(0),
(cache.framerate * kfinterval).toFixed(0),
`${command.hash}.mkv`
);
let file_json = path.join(
config.paths.output,
video.name,
cache_key,
encoder_key,
bitrate.toFixed(0),
(cache.framerate * kfinterval).toFixed(0),
`${command.hash}.json`
);
let queue_commands = new poolqueue();
let queue_files = new poolqueue();
let queue_promises = new Array();
// Check if this file already exists at the target location.
if (fs.existsSync(file_json)) {
// Don't need to check for a video here, since we only care about results.
if (global.debug) console.debug(`${file} already completed.`);
resolve();
return;
for (let encoder_key of encoders.keys()) {
let encoder = encoders.get(encoder_key);
let encoder_pool = encoder.pool();
let encoder_extra = encoder.extra();
for (let idx = 0; idx < encoder.count(); idx++) {
let command = encoder.get(idx, cache.width, cache.height, cache.framerate);
for (let bitrate of config.options.bitrates) {
for (let kfinterval of config.options.keyframeinterval) {
queue_promises.push(new Promise((resolve2) => {
let file = path.join(
config.paths.output,
video.name,
cache_key,
encoder_key,
bitrate.toFixed(0),
(cache.framerate * kfinterval).toFixed(0),
`${command.hash}.mkv`
);
let file_json = path.join(
config.paths.output,
video.name,
cache_key,
encoder_key,
bitrate.toFixed(0),
(cache.framerate * kfinterval).toFixed(0),
`${command.hash}.json`
);
// Check if this file already exists at the target location.
if (fs.existsSync(file_json)) {
// Don't need to check for a video here, since we only care about results.
if (global.debug) console.debug(`${file} already completed.`);
resolve2(false);
return;
}
let line = [
"-map", "0:v:0",
"-an",
"-c:v", encoder_key,
"-g", (cache.framerate * kfinterval).toFixed(0),
"-b:v", `${bitrate}k`,
"-minrate", "0",
"-maxrate", "0",
"-bufsize", `${2 * bitrate}k`,
].concat(command.options).concat(encoder_extra).concat([file]);
queue_commands.push(encoder_pool, line, command.cost);
queue_files.push(encoder_pool, [file, file_json], command.cost);
resolve2(true);
}));
}
let line = [
"-map", "0:v:0",
"-an",
"-c:v", encoder_key,
"-g", (cache.framerate * kfinterval).toFixed(0),
"-b:v", `${bitrate}k`,
"-minrate", "0",
"-maxrate", "0",
"-bufsize", `${2 * bitrate}k`,
].concat(command.options).concat(encoder.extra()).concat([file]);
queue_commands.push(
encoder.pool(),
line,
command.cost,
);
queue_files.push(
encoder.pool(),
[file, file_json],
command.cost,
);
resolve();
}));
}
}
}
queue_promises.push(async function () {
await Promise.allSettled(command_promises);
});
}
await Promise.allSettled(queue_promises);
cache.queues.set(encoder_key, {
commands: queue_commands.finalize(),
files: queue_files.finalize(),
});
await Promise.allSettled(queue_promises);
let data = {
commands: queue_commands.finalize(),
files: queue_files.finalize(),
};
cache.queues = data;
resolve1(true);
}));
}
console.timeEnd(cache_key);
}
console.timeEnd(video_key);
console.groupEnd();
await Promise.allSettled(video_promises);
console.timeEnd(video_key);
resolve();
}));
}
await Promise.allSettled(promises);
console.timeEnd("Subtotal");
console.groupEnd();
@@ -386,25 +388,25 @@ async function transcode(config, ff, videos, encoders) {
// 4. Delete encoded files.
// 5. Repeat until queues empty, no more caches for video, and no more videos.
/*
console.group("Queueing...")
console.time("Subtotal");
for (let video_key of videos.keys()) {
console.group(video_key);
console.time(video_key);
let video = videos.get(video_key);
for (let cache_key of video.caches.keys()) {
let cache = video.caches.get(cache_key);
console.time(cache_key);
console.timeEnd(cache_key);
/*
console.group("Queueing...")
console.time("Subtotal");
for (let video_key of videos.keys()) {
console.group(video_key);
console.time(video_key);
let video = videos.get(video_key);
for (let cache_key of video.caches.keys()) {
let cache = video.caches.get(cache_key);
console.time(cache_key);
console.timeEnd(cache_key);
}
console.timeEnd(video_key);
console.groupEnd();
}
console.timeEnd(video_key);
console.timeEnd("Subtotal");
console.groupEnd();
}
console.timeEnd("Subtotal");
console.groupEnd();
*/
*/
/*
for (let video_name in videos) {
console.time(video_name);