xaymar/js-encoding-samples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changes:

Browse Source 
* Added configurable scenecut.
* Removed bframes 0 and 1 from nvenc default config.
* Added b_ref_mode configuration option.
* Fixed path parallelization.
* Attempted libvmaf parallelization.
This commit is contained in:
Michael Fabian 'Xaymar' Dirks
2020-10-25 22:26:04 +01:00
parent ccc0483115
commit 6e2ac5e513
5 changed files with 190 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.json
+15 -7
View File
@@ -25,16 +25,17 @@
"film",
"grain",
"animation"
],
"scenecut": [
false
]
},
"h264_nvenc": {
"enabled": true,
"pool": "nvenc",
"parallel": 2,
"parallel": 3,
"gpu": -1,
"presets": [
"p5",
"p6",
"p7"
],
"tunes": [
@@ -45,18 +46,22 @@
16,
32
],
"bf": [
0,
1,
"bframes": [
2,
3,
4
],
"bframe_reference_mode": [
"middle"
],
"scenecut": [
false
]
},
"hevc_nvenc": {
"enabled": false,
"pool": "nvenc",
"parallel": 2,
"parallel": 3,
"gpu": -1
},
"h264_amf": {
@@ -182,6 +187,9 @@
}
},
"options": {
"vmaf": {
"model": "vmaf_4k_rb_v0.6.2.pkl"
},
"resolutions": [
[
1280,
encoder_h264_nvenc.js
+6 -6
View File
@@ -63,12 +63,12 @@ class h264_nvenc extends encoder {
for (let preset of this.settings.presets) {
for (let tune of this.settings.tunes) {
for (let rcla of this.settings["rc-lookahead"]) {
for (let ia of [0, 1]) {
if ((rcla == 0) && (ia != 0)) { // Requires lookahead.
for (let scenecut of this.settings.scenecut) {
if ((rcla == 0) && (scenecut == true)) { // Requires lookahead.
continue;
}
for (let bf of this.settings.bf) {
for (let bfm of ["disabled", "middle"]) {
for (let bf of this.settings.bframes) {
for (let bfm of this.settings.bframe_reference_mode) {
if ((bf == 0) && (bfm != "disabled")) { // Requires B-Frames
continue;
}
@@ -82,7 +82,7 @@ class h264_nvenc extends encoder {
"-rc", "cbr",
"-cbr", 1,
"-rc-lookahead", rcla,
"-no-scenecut", 1 - ia,
"-no-scenecut", scenecut == true ? 0 : 1,
"-bf", bf,
"-b_ref_mode", bfm,
"-b_adapt", 1,
@@ -124,7 +124,7 @@ class h264_nvenc extends encoder {
fs.writeFileSync(
path.join(this.config.paths.output, "h264_nvenc.json"),
JSON.stringify(this.indexes, null, null),
JSON.stringify(this.indexes, null, '\t'),
{encoding: "utf8"}
);
if (global.debug) console.timeEnd("Generating...");
encoder_libx264.js
+31 -23
View File
@@ -68,35 +68,43 @@ class libx264 extends encoder {
this.combinations = [];
for (let preset of this.settings.presets) {
for (let tune of this.settings.tunes) {
let _opts = [
"-profile:v", "high",
"-preset", preset,
"-x264-params", `nal-hrd=cbr:force-cfr=1`,
"-ssim", "0",
"-threads", this.settings.threads,
];
if (tune) {
_opts.push("-tune", tune);
for (let scenecut of this.settings.scenecut) {
let _opts = [
"-profile:v", "high",
"-preset", preset,
"-x264-params", `nal-hrd=cbr:force-cfr=1`,
"-ssim", "0",
"-threads", this.settings.threads,
];
if (tune) {
_opts.push("-tune", tune);
}
if (scenecut == false) {
_opts.push("-sc_threshold", 0);
} else if ((typeof scenecut) == "number") {
_opts.push("-sc_threshold", scenecut.toFixed(0));
}
let _name = name(_opts);
let _hash = crypto.createHash("sha256").update(_name).digest("hex");
let _cost = 1.0 * this.settings.cost_scale * this.cost.preset[preset] * this.cost.tune[tune] * this.cost.threads;
let combo = {
name: _name,
hash: _hash,
options: _opts,
cost: _cost,
};
this.indexes[_hash] = combo.options;
this.combinations.push(combo);
}
let _name = name(_opts);
let _hash = crypto.createHash("sha256").update(_name).digest("hex");
let _cost = 1.0 * this.settings.cost_scale * this.cost.preset[preset] * this.cost.tune[tune] * this.cost.threads;
let combo = {
name: _name,
hash: _hash,
options: _opts,
cost: _cost,
};
this.indexes[_hash] = combo.options;
this.combinations.push(combo);
}
}
fs.writeFileSync(
path.join(this.config.paths.output, "libx264.json"),
JSON.stringify(this.indexes, null, null),
JSON.stringify(this.indexes, null, '\t'),
{encoding: "utf8"}
);
if (global.debug) console.timeEnd("Generating...");
ffmpeg.js
+15
View File
@@ -1,6 +1,7 @@
const path = require('path');
const process = require('process');
const child_process = require('child_process');
const os = require('os');
function parse_duration(str) {
// Duration is in the form HH:MM:SS.fraction
@@ -152,6 +153,20 @@ class ffmpeg {
})
});
}
consolify(_file) {
//_file = path.resolve(_file);
if (os.platform() == 'win32') {
// Need to turn:
// C:\myfiles\model.pkl
// into:
// C\\:/myfiles/model.pkl
_file = _file.replace(/\\/g, '/').replace(':', '\\\\:');
} else {
// No further work needs to be done?
}
return _file;
}
}
module.exports = ffmpeg;
ves.js
+123 -74
View File
@@ -6,13 +6,6 @@ global.debug = false;
// --------------------------------------------------------------------------------
// Actual Code
/*
Comparing VMAF instantly:
- Saves disk space - no need to keep copies around!
fn: .\ffmpeg.exe -i "..\..\cache\arma_3-002-1280x720x60.00.mkv" -i "..\..\videos\arma_3-002.mkv" -filter_complex_threads 8 -filter_complex [0:v:0]scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=1920:h=1080,colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[main];[1:v:0]colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[ref];[main][ref]libvmaf=model_path=../vmaf/vmaf_4k_rb_v0.6.2.pkl:log_fmt=json:log_path=here2.json:enable_conf_interval=1:shortest=1[out] -map [out] -f null -
*/
// Import modules
const ffmpeg = require('./ffmpeg.js');
const poolqueue = require('./poolqueue.js');
@@ -271,20 +264,10 @@ async function create_caches(config, ff, videos, encoders) { // Create Caches
return videos;
}
async function transcode(config, ff, videos, encoders) {
console.group("Transcoding...");
async function queue(config, ff, videos, encoders) {
console.group("Queueing...");
console.time("Total");
/* Needs a massive overhaul:
- Check if the .json file for a configuration already exists, if not continue.
- Do not store transcodes for longer than needed, we only need to compare against source.
- Use poolqueue to still allow for parallelization.
- '-g' and bitrate options are controlled _from here_.
*/
// Build queues per cache.
console.group("Queueing...")
console.time("Subtotal");
let promises = [];
for (let video_key of videos.keys()) {
promises.push(new Promise(async (resolve, reject) => {
@@ -349,7 +332,7 @@ async function transcode(config, ff, videos, encoders) {
].concat(command.options).concat(encoder_extra).concat([file]);
queue_commands.push(encoder_pool, line, command.cost);
queue_files.push(encoder_pool, [file, file_json], command.cost);
queue_files.push(encoder_pool, [[file, file_json]], command.cost);
resolve2(true);
}));
@@ -377,9 +360,11 @@ async function transcode(config, ff, videos, encoders) {
}));
}
await Promise.allSettled(promises);
console.timeEnd("Subtotal");
console.timeEnd("Total");
console.groupEnd();
}
async function work(config, ff, videos, encoders) {
// Process from here on out.
// LOOP
// 1. Pull out front of the command and file queue.
@@ -388,53 +373,122 @@ async function transcode(config, ff, videos, encoders) {
// 4. Delete encoded files.
// 5. Repeat until queues empty, no more caches for video, and no more videos.
/*
console.group("Queueing...")
console.time("Subtotal");
for (let video_key of videos.keys()) {
console.group(video_key);
console.time(video_key);
let video = videos.get(video_key);
for (let cache_key of video.caches.keys()) {
let cache = video.caches.get(cache_key);
console.time(cache_key);
console.timeEnd(cache_key);
}
console.timeEnd(video_key);
console.groupEnd();
}
console.timeEnd("Subtotal");
console.groupEnd();
*/
/*
for (let video_name in videos) {
console.time(video_name);
console.group(video_name);
let video = videos[video_name];
for (let cache of video.caches) {
let key_cache = `${cache.x}x${cache.y}x${cache.fps.toFixed(2)}`;
console.time(key_cache);
console.group(key_cache);
for (let cmd of cache.commands) {
let opts = [
"-y",
"-hide_banner",
"-v", "error",
"-hwaccel", "auto",
"-i", cache.file
].concat(cmd);
let res = ff.ffmpegSync(opts);
console.log(res.stdout.toString(), res.stderr.toString());
}
console.groupEnd();
console.timeEnd(key_cache);
}
console.groupEnd();
console.timeEnd(video_name);
}
*/
let vmaf_model = ff.consolify(path.resolve(path.join(config.paths.ffmpeg, "vmaf", config.options.vmaf.model)));
console.group("Processing...")
console.time("Total");
for (let video_key of videos.keys()) {
console.group(video_key);
console.time(video_key);
let video = videos.get(video_key);
for (let cache_key of video.caches.keys()) {
console.time(cache_key);
let cache = video.caches.get(cache_key);
let length = cache.queues.commands.length;
console.log(`0.0% (0 / ${length}): 0.000s`);
while (cache.queues.commands.length > 0) {
let commands = cache.queues.commands.shift();
let files = cache.queues.files.shift();
let prc1 = length - cache.queues.commands.length;
let prc2 = prc1 / length * 100.0;
let LABEL = `${prc2.toFixed(1)}% (${prc1} / ${length})`
console.time(LABEL)
console.group();
// Create directories.
for (let file of files) {
fs.mkdirSync(path.dirname(file[0]), { recursive: true });
}
// Encode
{
console.time("Encoding");
let opts = [
"-y",
"-hide_banner",
"-v", "error",
"-hwaccel", "auto",
"-i", cache.file
].concat(commands);
let res = ff.ffmpegSync(opts);
if (res.status != 0) {
console.log(res.stdout.toString(), res.stderr.toString());
continue;
}
console.timeEnd("Encoding");
}
/*
Comparing VMAF instantly:
- Saves disk space - no need to keep copies around!
fn: .\ffmpeg.exe -i "..\..\cache\arma_3-002-1280x720x60.00.mkv" -i "..\..\videos\arma_3-002.mkv" -filter_complex_threads 8 -filter_complex [0:v:0]scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=1920:h=1080,colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[main];[1:v:0]colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[ref];[main][ref]libvmaf=model_path=../vmaf/vmaf_4k_rb_v0.6.2.pkl:log_fmt=json:log_path=here2.json:enable_conf_interval=1:shortest=1[out] -map [out] -f null -
*/
// Process
{
console.time("Processing");
let opts = [
"-hide_banner",
"-v", "warning",
"-hwaccel", "auto",
"-i", video.file,
];
let filter = "";
let references = [];
// Build Filter Graph
if (files.length > 1) {
filter = `[0:v:0]split=${files.length}`
for (let idx = 0; idx < files.length; idx++) {
filter = `${filter}[ref:${idx}]`;
references[idx] = `[ref:${idx}]`;
}
filter = `${filter}`
} else {
references[0] = "[0:v:0]";
}
// Rescale and Resample all inputs and compare them.
for (let idx = 0; idx < files.length; idx++) {
let file = files[idx];
opts.push("-i", file[0]);
if (filter != "")
filter = `${filter};` // [temp:${idx}];[temp:${idx}]
filter = `${filter}[${idx}:v:0]scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=${video.resolution.width.toFixed(0)}:h=${video.resolution.height.toFixed(0)},colorspace=space=${video.color.matrix}:trc=${video.color.trc}:primaries=${video.color.primaries}:range=${video.color.range},format=pix_fmts=yuv444p,fps=fps=${video.framerate.toFixed(2)},[ref:${idx}]libvmaf=model_path=${vmaf_model}:log_fmt=json:log_path=${ff.consolify(file[1])}:enable_conf_interval=1:n_threads=2[main:${idx}]`
}
opts.push("-filter_complex", filter);
for (let idx = 0; idx < files.length; idx++) {
opts.push(
"-map", `[main:${idx}]`,
"-f", "null",
"-"
)
}
console.log(opts);
let res = ff.ffmpegSync(opts);
if (res.status != 0) {
console.log(res.stdout.toString(), res.stderr.toString());
continue;
}
console.timeEnd("Processing");
}
console.log(filter);
console.groupEnd();
console.timeEnd(LABEL);
}
console.timeEnd(cache_key);
}
console.timeEnd(video_key);
console.groupEnd();
}
console.timeEnd("Total");
console.groupEnd();
}
@@ -450,13 +504,8 @@ async function main() {
await load_encoders(config, ff).then((p) => { encoders = p; });
await load_videos(config, ff).then((p) => { videos = p; });
await create_caches(config, ff, videos, encoders);
await transcode(config, ff, videos, encoders);
// Queue
//await queue_transcodes(config, ff, videos, encoders);
// Transcode
await queue(config, ff, videos, encoders);
await work(config, ff, videos, encoders);
}
main();