xaymar/js-encoding-samples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drastically speed things up with parallelization

Browse Source
This commit is contained in:
Michael Fabian 'Xaymar' Dirks
2020-10-25 19:16:04 +01:00
parent 3b0cdeaaac
commit ccc0483115
3 changed files with 112 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
encoder_h264_nvenc.js
+8 -8
View File
@@ -11,7 +11,7 @@ class h264_nvenc extends encoder {
} }
available() { available() {
console.time("Checking..."); if (global.debug) console.time("Checking...");
let res = ff.ffmpegSync([ let res = ff.ffmpegSync([
"-hide_banner", "-v", "quiet", "-hide_banner", "-v", "quiet",
"-f", "lavfi", "-f", "lavfi",
@@ -20,7 +20,7 @@ class h264_nvenc extends encoder {
"-f", "null", "-f", "null",
"-" "-"
]); ]);
console.timeEnd("Checking..."); if (global.debug) console.timeEnd("Checking...");
if (res.status != 0) { if (res.status != 0) {
return false; return false;
} }
@@ -28,7 +28,7 @@ class h264_nvenc extends encoder {
} }
available() { available() {
console.time("Checking..."); if (global.debug) console.time("Checking...");
let res = this.ffmpeg.ffmpegSync([ let res = this.ffmpeg.ffmpegSync([
"-hide_banner", "-v", "error", "-hide_banner", "-v", "error",
"-f", "lavfi", "-f", "lavfi",
@@ -37,7 +37,7 @@ class h264_nvenc extends encoder {
"-f", "null", "-f", "null",
"-" "-"
]); ]);
console.timeEnd("Checking..."); if (global.debug) console.timeEnd("Checking...");
if (res.status != 0) { if (res.status != 0) {
return false; return false;
} }
@@ -57,7 +57,7 @@ class h264_nvenc extends encoder {
return `${name};version=${version}`; return `${name};version=${version}`;
} }
console.time("Generating..."); if (global.debug) console.time("Generating...");
this.indexes = {}; this.indexes = {};
this.combinations = []; this.combinations = [];
for (let preset of this.settings.presets) { for (let preset of this.settings.presets) {
@@ -127,8 +127,8 @@ class h264_nvenc extends encoder {
JSON.stringify(this.indexes, null, null), JSON.stringify(this.indexes, null, null),
{encoding: "utf8"} {encoding: "utf8"}
); );
console.timeEnd("Generating..."); if (global.debug) console.timeEnd("Generating...");
console.log(`Combinations: ${this.combinations.length}`) if (global.debug) console.log(`Combinations: ${this.combinations.length}`)
} }
pool() { pool() {
@@ -140,7 +140,7 @@ class h264_nvenc extends encoder {
} }
extra() { extra() {
return ["-gpu", settings.gpu]; return ["-gpu", this.settings.gpu];
} }
} }
encoder_libx264.js
+5 -5
View File
@@ -34,7 +34,7 @@ class libx264 extends encoder {
} }
available() { available() {
console.time("Checking..."); if (global.debug) console.time("Checking...");
let res = this.ffmpeg.ffmpegSync([ let res = this.ffmpeg.ffmpegSync([
"-hide_banner", "-v", "error", "-hide_banner", "-v", "error",
"-f", "lavfi", "-f", "lavfi",
@@ -43,7 +43,7 @@ class libx264 extends encoder {
"-f", "null", "-f", "null",
"-" "-"
]); ]);
console.timeEnd("Checking..."); if (global.debug) console.timeEnd("Checking...");
if (res.status != 0) { if (res.status != 0) {
return false; return false;
} }
@@ -63,7 +63,7 @@ class libx264 extends encoder {
return `${name};version=${version}`; return `${name};version=${version}`;
} }
console.time("Generating..."); if (global.debug) console.time("Generating...");
this.indexes = {}; this.indexes = {};
this.combinations = []; this.combinations = [];
for (let preset of this.settings.presets) { for (let preset of this.settings.presets) {
@@ -99,8 +99,8 @@ class libx264 extends encoder {
JSON.stringify(this.indexes, null, null), JSON.stringify(this.indexes, null, null),
{encoding: "utf8"} {encoding: "utf8"}
); );
console.timeEnd("Generating..."); if (global.debug) console.timeEnd("Generating...");
console.log(`Combinations: ${this.count()}`) if (global.debug) console.log(`Combinations: ${this.count()}`)
} }
pool() { pool() {
ves.js
+44 -42
View File
@@ -121,7 +121,7 @@ async function load_videos(config, ff) { // Load Videos
data.name = name; data.name = name;
data.info = json; data.info = json;
data.caches = new Map(); data.caches = {};
// File Information // File Information
data.file_name = `${name}.mkv`; data.file_name = `${name}.mkv`;
@@ -285,26 +285,30 @@ async function transcode(config, ff, videos, encoders) {
// Build queues per cache. // Build queues per cache.
console.group("Queueing...") console.group("Queueing...")
console.time("Subtotal"); console.time("Subtotal");
let promises = [];
for (let video_key of videos.keys()) { for (let video_key of videos.keys()) {
console.group(video_key); promises.push(new Promise(async (resolve, reject) => {
console.time(video_key);
let video = videos.get(video_key); let video = videos.get(video_key);
let video_promises = [];
console.time(video_key);
for (let cache_key of video.caches.keys()) { for (let cache_key of video.caches.keys()) {
video_promises.push(new Promise(async (resolve1) => {
let cache = video.caches.get(cache_key); let cache = video.caches.get(cache_key);
console.time(cache_key);
let queue_commands = new poolqueue();
let queue_files = new poolqueue();
let queue_promises = new Array();
for (let encoder_key of encoders.keys()) { for (let encoder_key of encoders.keys()) {
let encoder = encoders.get(encoder_key); let encoder = encoders.get(encoder_key);
let encoder_pool = encoder.pool();
let queue_promises = []; let encoder_extra = encoder.extra();
let queue_commands = new poolqueue();
let queue_files = new poolqueue();
for (let idx = 0; idx < encoder.count(); idx++) { for (let idx = 0; idx < encoder.count(); idx++) {
let command_promises = [];
let command = encoder.get(idx, cache.width, cache.height, cache.framerate); let command = encoder.get(idx, cache.width, cache.height, cache.framerate);
for (let bitrate of config.options.bitrates) { for (let bitrate of config.options.bitrates) {
for (let kfinterval of config.options.keyframeinterval) { for (let kfinterval of config.options.keyframeinterval) {
command_promises.push(new Promise((resolve, reject) => { queue_promises.push(new Promise((resolve2) => {
let file = path.join( let file = path.join(
config.paths.output, config.paths.output,
video.name, video.name,
@@ -328,7 +332,8 @@ async function transcode(config, ff, videos, encoders) {
if (fs.existsSync(file_json)) { if (fs.existsSync(file_json)) {
// Don't need to check for a video here, since we only care about results. // Don't need to check for a video here, since we only care about results.
if (global.debug) console.debug(`${file} already completed.`); if (global.debug) console.debug(`${file} already completed.`);
resolve();
resolve2(false);
return; return;
} }
@@ -341,40 +346,37 @@ async function transcode(config, ff, videos, encoders) {
"-minrate", "0", "-minrate", "0",
"-maxrate", "0", "-maxrate", "0",
"-bufsize", `${2 * bitrate}k`, "-bufsize", `${2 * bitrate}k`,
].concat(command.options).concat(encoder.extra()).concat([file]); ].concat(command.options).concat(encoder_extra).concat([file]);
queue_commands.push( queue_commands.push(encoder_pool, line, command.cost);
encoder.pool(), queue_files.push(encoder_pool, [file, file_json], command.cost);
line,
command.cost, resolve2(true);
); }));
queue_files.push( }
encoder.pool(), }
[file, file_json], }
command.cost, }
);
await Promise.allSettled(queue_promises);
let data = {
commands: queue_commands.finalize(),
files: queue_files.finalize(),
};
cache.queues = data;
resolve1(true);
}));
}
await Promise.allSettled(video_promises);
console.timeEnd(video_key);
resolve(); resolve();
})); }));
} }
} await Promise.allSettled(promises);
queue_promises.push(async function () {
await Promise.allSettled(command_promises);
});
}
await Promise.allSettled(queue_promises);
cache.queues.set(encoder_key, {
commands: queue_commands.finalize(),
files: queue_files.finalize(),
});
}
console.timeEnd(cache_key);
}
console.timeEnd(video_key);
console.groupEnd();
}
console.timeEnd("Subtotal"); console.timeEnd("Subtotal");
console.groupEnd(); console.groupEnd();
@@ -386,7 +388,7 @@ async function transcode(config, ff, videos, encoders) {
// 4. Delete encoded files. // 4. Delete encoded files.
// 5. Repeat until queues empty, no more caches for video, and no more videos. // 5. Repeat until queues empty, no more caches for video, and no more videos.
/* /*
console.group("Queueing...") console.group("Queueing...")
console.time("Subtotal"); console.time("Subtotal");
for (let video_key of videos.keys()) { for (let video_key of videos.keys()) {
@@ -404,7 +406,7 @@ async function transcode(config, ff, videos, encoders) {
} }
console.timeEnd("Subtotal"); console.timeEnd("Subtotal");
console.groupEnd(); console.groupEnd();
*/ */
/* /*
for (let video_name in videos) { for (let video_name in videos) {
console.time(video_name); console.time(video_name);