From 43952dd162c269196a1f1b719ef1a98e2aa64ebb Mon Sep 17 00:00:00 2001 From: Michael Fabian 'Xaymar' Dirks Date: Sun, 25 Oct 2020 18:08:58 +0100 Subject: [PATCH] Initial Work --- .editorconfig | 10 + .gitignore | 10 + config.json | 214 +++++++++++++++++++++ encoder.js | 40 ++++ encoder_h264_nvenc.js | 153 +++++++++++++++ encoder_libx264.js | 118 ++++++++++++ ffmpeg.js | 157 +++++++++++++++ ffmpeg/README.md | 2 + poolqueue.js | 45 +++++ ves.js | 436 ++++++++++++++++++++++++++++++++++++++++++ videos/README.md | 1 + 11 files changed, 1186 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 config.json create mode 100644 encoder.js create mode 100644 encoder_h264_nvenc.js create mode 100644 encoder_libx264.js create mode 100644 ffmpeg.js create mode 100644 ffmpeg/README.md create mode 100644 poolqueue.js create mode 100644 ves.js create mode 100644 videos/README.md diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..794e044 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file. +[*] +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 +indent_style = tab +indent_size = 4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..84f246e --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +# FFmpeg Binaries - see README.md! +ffmpeg/* + +# Video Inputs - see README.md! +videos/* + +# Generated Files +output/* +cache/* +desktop.ini diff --git a/config.json b/config.json new file mode 100644 index 0000000..a2186c6 --- /dev/null +++ b/config.json @@ -0,0 +1,214 @@ +{ + "paths": { + "ffmpeg": "./ffmpeg/", + "videos": "./videos/", + "cache": "./cache/", + "output": "./output/" + }, + "encoders": { + "libx264": { + "enabled": true, + "pool": "cpu", + "cost_scale": 100.0, + "threads": 8, + "presets": [ + "veryfast", + "faster", + "fast", + "medium", + "slow", + "slower", + "veryslow" + ], + "tunes": [ + null, + "film", + "grain", + "animation" + ] + }, + "h264_nvenc": { + "enabled": true, + "pool": "nvenc", + "parallel": 2, + "presets": [ + "p5", + "p6", + "p7" + ], + "tunes": [ + "hq" + ], + "rc-lookahead": [ + 0, + 16, + 32 + ], + "bf": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + "hevc_nvenc": { + "enabled": false, + "pool": "nvenc", + "parallel": 3 + }, + "h264_amf": { + "enabled": false, + "pool": "amf", + "parallel": 3 + }, + "hevc_amf": { + "enabled": false, + "pool": "amf", + "parallel": 3 + }, + "h264_qsv": { + "enabled": false, + "pool": "qsv", + "parallel": 3 + }, + "hevc_qsv": { + "enabled": false, + "pool": "qsv", + "parallel": 3 + }, + "vp9_qsv": { + "enabled": false, + "pool": "qsv", + "parallel": 3 + }, + "libvpx-vp9": { + "enabled": false, + "pool": "cpu" + }, + "libaom-av1": { + "enabled": false, + "pool": "cpu" + } + }, + "videos": { + "arma_3-001": { + "enabled": false + }, + "arma_3-002": { + "enabled": true + }, + "black_mesa-001": { + "enabled": false + }, + "black_mesa-002": { + "enabled": false + }, + "black_mesa-003": { + "enabled": false + }, + "dota_2-001": { + "enabled": false + }, + "grip-001": { + "enabled": false + }, + "grip-002": { + "enabled": false + }, + "grip-003": { + "enabled": false + }, + "beat_hazard-001": { + "enabled": false + }, + "beat_hazard-002": { + "enabled": false + }, + "celeste-001": { + "enabled": false + }, + "celeste-002": { + "enabled": false + }, + "celeste-003": { + "enabled": false + }, + "final_fantasy_xiv-001": { + "enabled": false + }, + "final_fantasy_xiv-002": { + "enabled": false + }, + "final_fantasy_xiv-003": { + "enabled": false + }, + "final_fantasy_xiv-004": { + "enabled": true + }, + "forza_4_horizon-001": { + "enabled": false + }, + "forza_4_horizon-002": { + "enabled": false + }, + "noita-001": { + "enabled": false + }, + "noita-002": { + "enabled": false + }, + "risk_of_rain_2-001": { + "enabled": false + }, + "satisfactory-001": { + "enabled": false + }, + "space_engineers-001": { + "enabled": false + }, + "space_engineers-002": { + "enabled": false + }, + "space_engineers-003": { + "enabled": false + } + }, + "options": { + "resolutions": [ + [ + 1280, + 720 + ], + [ + 1536, + 864 + ], + [ + 1920, + 1080 + ], + [ + 2560, + 1440 + ], + [ + 3840, + 2160 + ] + ], + "framerate_scalings": [ + 1.0, + 0.5 + ], + "bitrates": [ + 3500, + 6000, + 8000 + ], + "keyframeinterval": [ + 1.0, + 2.0 + ] + } +} \ No newline at end of file diff --git a/encoder.js b/encoder.js new file mode 100644 index 0000000..39c41ed --- /dev/null +++ b/encoder.js @@ -0,0 +1,40 @@ +class encoder { + constructor(ffmpeg, config, settings) { + // FFmpeg + this.ffmpeg = ffmpeg; + + // Configuration + this.config = config; + + // Encoder Settings + this.settings = settings; // Settings + + // Check if this encoder is available. + if (!this.available()) { + throw new ReferenceError("Encoder is not available"); + } + } + + available() { + return true; + } + + load() { + this.indexes = {}; + this.combinations = []; + } + + pool() { + return "default"; + } + + count() { + return this.combinations.length; + } + + get(index, width, height, framerate) { + throw new Error("Not Implemented"); + } +} + +module.exports = encoder; diff --git a/encoder_h264_nvenc.js b/encoder_h264_nvenc.js new file mode 100644 index 0000000..df9dac8 --- /dev/null +++ b/encoder_h264_nvenc.js @@ -0,0 +1,153 @@ +const encoder = require('./encoder.js'); +const path = require('path'); +const fs = require('fs'); +const crypto = require('crypto'); + +let version = 1; + +class h264_nvenc extends encoder { + constructor(ffmpeg, config, settings) { + super(ffmpeg, config, settings); + } + + available() { + console.time("Checking..."); + let res = ff.ffmpegSync([ + "-hide_banner", "-v", "quiet", + "-f", "lavfi", + "-i", "color=size=256x256:duration=1:rate=30:color=black", + "-c:v", "h264_nvenc", + "-f", "null", + "-" + ]); + console.timeEnd("Checking..."); + if (res.status != 0) { + return false; + } + return true; + } + + available() { + console.time("Checking..."); + let res = this.ffmpeg.ffmpegSync([ + "-hide_banner", "-v", "error", + "-f", "lavfi", + "-i", "color=size=256x256:duration=1:rate=30:color=black", + "-c:v", "h264_nvenc", + "-f", "null", + "-" + ]); + console.timeEnd("Checking..."); + if (res.status != 0) { + return false; + } + return true; + } + + load() { + let name = function(opts) { + let name = ""; + for (let idx = 0; idx < opts.length; idx += 2) { + let opt = opts[idx]; + let val = opts[idx + 1]; + if (name.length != 0) + name += ";"; + name += `${opt.substr(1)}=${val}`; + } + return `${name};version=${version}`; + } + + console.time("Generating..."); + this.indexes = {}; + this.combinations = []; + for (let preset of this.settings.presets) { + for (let tune of this.settings.tunes) { + for (let rcla of this.settings["rc-lookahead"]) { + for (let ia of [0, 1]) { + if ((rcla == 0) && (ia != 0)) { // Requires lookahead. + continue; + } + for (let bf of this.settings.bf) { + for (let bfm of ["disabled", "middle"]) { + if ((bf == 0) && (bfm != "disabled")) { // Requires B-Frames + continue; + } + for (let mp of [0, 1, 2]) { + for (let taq of [0, 1]) { + for (let saqs of [0, 7, 15]) { + let _opts = [ + "-profile:v", "high", + "-preset", preset, + "-tune", tune, + "-rc", "cbr", + "-cbr", 1, + "-rc-lookahead", rcla, + "-no-scenecut", 1 - ia, + "-bf", bf, + "-b_ref_mode", bfm, + "-b_adapt", 1, + "-multipass", mp, + "-temporal_aq", taq, + ]; + if (saqs == 0) { + _opts.push( + "-spatial-aq", 0, + ); + } else { + _opts.push( + "-spatial-aq", 1, + "-aq-strength", saqs, + ); + } + + let _name = name(_opts); + let _hash = crypto.createHash("sha256").update(_name).digest("hex"); + let _cost = (1.0 / this.settings.parallel) * 1.01; + let combo = { + name: _name, + hash: _hash, + options: _opts, + cost: _cost, + }; + + this.indexes[_hash] = combo.options; + this.combinations.push(combo); + } + } + } + } + } + } + } + } + } + + fs.writeFileSync( + path.join(this.config.paths.output, "h264_nvenc.json"), + JSON.stringify(this.indexes, null, null), + {encoding: "utf8"} + ); + console.timeEnd("Generating..."); + console.log(`Combinations: ${this.combinations.length}`) + } + + pool() { + return this.settings.pool; + } + + get(index, width, height, framerate) { + return this.combinations[index]; + } + +/* +for (let br of _config.bitrates) + "-b:v", `${br.toFixed(0)}k`, + "-bufsize", `${(br * 2).toFixed(0)}k`, + "-minrate", "0", + "-maxrate", `${br.toFixed(0)}k`, +for (let kfm of _config.keyframe_multiplier) + "-g", (_cache.fps * kfm).toFixed(0), +*/ +} + +module.exports = h264_nvenc; diff --git a/encoder_libx264.js b/encoder_libx264.js new file mode 100644 index 0000000..cf784f6 --- /dev/null +++ b/encoder_libx264.js @@ -0,0 +1,118 @@ +const encoder = require('./encoder.js'); +const path = require('path'); +const fs = require('fs'); +const crypto = require('crypto'); + +let version = 1; + +class libx264 extends encoder { + constructor(ffmpeg, config, settings) { + super(ffmpeg, config, settings); + + // Cost per option chosen. + this.cost = { + preset: { // Measured at 2560x1440x60 with no tune + "ultrafast": 0.603, // 520 fps / 30 fps, 116 fps / 60 fps + "superfast": 0.686, // 383 fps / 30 fps, 102 fps / 60 fps + "veryfast": 0.737, // 378 fps / 30 fps, 95 fps / 60 fps + "faster": 0.854, // 339 fps / 30 fps, 82 fps / 60 fps + "fast": 0.972, // 334 fps / 30 fps, 72 fps / 60 fps + "medium": 1.000, // 368 fps / 30 fps, 70 fps / 60 fps + "slow": 1.321, // 359 fps / 30 fps, 53 fps / 60 fps + "slower": 2.414, // 329 fps / 30 fps, 29 fps / 60 fps + "veryslow": 4.667, // 270 fps / 30 fps, 15 fps / 60 fps + "placebo": 18.42, // 125 fps / 30 fps, 3.8 fps / 60 fps + }, + tune: { // Measured at 2560x1440x60 medium + null: 1.000, // 70 fps / 60 fps + "film": 1.029, // 68 fps / 60 fps + "animation": 1.077, // 65 fps / 60 fps + "grain": 1.061, // 66 fps / 60 fps + }, + threads: (32.0 / this.settings.threads), + } + } + + available() { + console.time("Checking..."); + let res = this.ffmpeg.ffmpegSync([ + "-hide_banner", "-v", "error", + "-f", "lavfi", + "-i", "color=size=256x256:duration=1:rate=30:color=black", + "-c:v", "libx264", + "-f", "null", + "-" + ]); + console.timeEnd("Checking..."); + if (res.status != 0) { + return false; + } + return true; + } + + load() { + let name = function(opts) { + let name = ""; + for (let idx = 0; idx < opts.length; idx += 2) { + let opt = opts[idx]; + let val = opts[idx + 1]; + if (name.length != 0) + name += ";"; + name += `${opt.substr(1)}=${val}`; + } + return `${name};version=${version}`; + } + + console.time("Generating..."); + this.indexes = {}; + this.combinations = []; + for (let preset of this.settings.presets) { + for (let tune of this.settings.tunes) { + let _opts = [ + "-profile:v", "high", + "-preset", preset, + "-x264-params", `nal-hrd=cbr:force-cfr=1`, + "-ssim", "0", + "-threads", this.settings.threads, + ]; + if (tune) { + _opts.push("-tune", tune); + } + + let _name = name(_opts); + let _hash = crypto.createHash("sha256").update(_name).digest("hex"); + let _cost = 1.0 * this.settings.cost_scale * this.cost.preset[preset] * this.cost.tune[tune] * this.cost.threads; + let combo = { + name: _name, + hash: _hash, + options: _opts, + cost: _cost, + }; + + this.indexes[_hash] = combo.options; + this.combinations.push(combo); + } + } + + fs.writeFileSync( + path.join(this.config.paths.output, "libx264.json"), + JSON.stringify(this.indexes, null, null), + {encoding: "utf8"} + ); + console.timeEnd("Generating..."); + console.log(`Combinations: ${this.count()}`) + } + + pool() { + return this.settings.pool; + } + + get(index, width, height, framerate) { + let result = Object.assign(new Object(), this.combinations[index]); + result.cost *= framerate / 60.0; + result.cost *= (width * height) / (2560 * 1440); + return result; + } +} + +module.exports = libx264; diff --git a/ffmpeg.js b/ffmpeg.js new file mode 100644 index 0000000..5c69fe1 --- /dev/null +++ b/ffmpeg.js @@ -0,0 +1,157 @@ +const path = require('path'); +const process = require('process'); +const child_process = require('child_process'); + +function parse_duration(str) { + // Duration is in the form HH:MM:SS.fraction + let parts = str.split(":"); + for (let idx = 0; idx < parts.length; idx++) { + parts[idx] = parseFloat(parts[idx]); + } + return ((parts[0] * 60) + parts[1]) * 60 + parts[2]; +} + +class ffmpeg { + constructor(ffpath) { + // Figure out the location of FFmpeg + if (!ffpath) + ffpath = process.cwd(); + this.bin = { + ffmpeg: path.join(ffpath, "bin/ffmpeg.exe"), + ffprobe: path.join(ffpath, "bin/ffprobe.exe") + }; + } + + _probeParse(buffer) { + let parsed = JSON.parse(buffer); + if (parsed.streams && (parsed.streams.length > 0)) { + for (let idx = 0; idx < parsed.streams.length; idx++) { + let stream = parsed.streams[idx]; + if (stream.tags && stream.tags["DURATION"]) { + stream.duration = parse_duration(stream.tags["DURATION"]); + } + } + } + return parsed; + } + + probeSync(file) { + let result = child_process.spawnSync(this.bin.ffprobe, + [ + "-hide_banner", + "-v", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + "-i", file + ] + ); + return this._probeParse(result.stdout); + } + + probe(file) { + return new Promise((resolve, reject) => { + let proc = child_process.spawn(this.bin.ffprobe, + [ + "-hide_banner", + "-v", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + "-i", file + ], + {stdio: ['pipe', 'pipe', 'pipe']} + ); + let buffer = ""; + proc.stdout.on('data', (data) => { + buffer += data.toString(); + }); + proc.on('close', (code, signal) => { + if (code == 0) { + try { + resolve(this._probeParse(buffer)); + } catch (ex) { + reject(ex); + } + } else { + if (code == null) { + reject(signal); + } else { + reject(code); + } + } + }); + proc.on('error', (error) => { + reject(error); + }) + }); + } + + get_encoder_caps(encoder) { + let result = { + hardware: false, + formats: [], + devices: [] + }; + + // Output is a list of options. + let temp = child_process.execFileSync(this.bin.ffmpeg, + [ + "-hide_banner", + "-v", "quiet", + "-h", `encoder=${encoder}` + ] + ).toString(); + let lines = temp.split('\r\n'); + for (let line of lines) { + if (line.includes("General capabilities:")) { + result.hardware = (line.includes("hardware")); + } else if (line.includes("Supported hardware devices:")) { + let data = line.substr(line.indexOf(':') + 2); + result.devices = data.split(' '); + } else if (line.includes("Supported pixel formats:")) { + let data = line.substr(line.indexOf(':') + 2); + result.formats = data.split(' '); + } + } + return result; + } + + ffmpegSync(params) { + return child_process.spawnSync(this.bin.ffmpeg, params); + } + + ffmpeg(params) { + return new Promise((resolve, reject) => { + let proc = child_process.spawn(this.bin.ffmpeg, params); + let buf_stdout = ""; + let buf_stderr = ""; + proc.stdout.on('data', (data) => { + buf_stdout += data.toString(); + }); + proc.stderr.on('data', (data) => { + buf_stderr += data.toString(); + }); + proc.on('close', (code, signal) => { + if (code == 0) { + try { + resolve([code, buf_stdout, buf_stderr]); + } catch (ex) { + reject([ex, buf_stdout, buf_stderr]); + } + } else { + if (code == null) { + reject([signal, buf_stdout, buf_stderr]); + } else { + reject([code, buf_stdout, buf_stderr]); + } + } + }); + proc.on('error', (error) => { + reject([error, buf_stdout, buf_stderr]); + }) + }); + } +} + +module.exports = ffmpeg; diff --git a/ffmpeg/README.md b/ffmpeg/README.md new file mode 100644 index 0000000..f55b959 --- /dev/null +++ b/ffmpeg/README.md @@ -0,0 +1,2 @@ +# FFmpeg + VMAF +Download the latest prebuilt FFmpeg for Windows package and extract it here. diff --git a/poolqueue.js b/poolqueue.js new file mode 100644 index 0000000..18e6040 --- /dev/null +++ b/poolqueue.js @@ -0,0 +1,45 @@ + +class poolqueue { + constructor() { + this.queues = new Array(); + this.index = {}; + this.costs = {}; + } + + clear() { + this.queues = new Array(); + this.index = {}; + this.costs = {}; + } + + push(name, command, cost) { + if (!this.index[name]) { + this.index[name] = 0; + } + if (!this.costs[name]) { + this.costs[name] = 0; + } + + // Make sure Queue exists + if (!this.queues[this.index[name]]) { + this.queues[this.index[name]] = new Array(); + if (global.debug) console.debug(`${name}: Created new pool.`); + } + + // Push the new command. + this.queues[this.index[name]] = this.queues[this.index[name]].concat(command); + + this.costs[name] += cost; + if (this.costs[name] > 1.0) { + this.index[name]++; + if (global.debug) console.debug(`${name}: Incremented index to ${this.index[name]} due to cost ${this.costs[name]}.`); + this.costs[name] = 0.0; + } + } + + finalize() { + return this.queues; + } +} + +module.exports = poolqueue; diff --git a/ves.js b/ves.js new file mode 100644 index 0000000..cd16dd7 --- /dev/null +++ b/ves.js @@ -0,0 +1,436 @@ +// -------------------------------------------------------------------------------- +// Development options (for user options, see config.json) +global.debug = false; + + +// -------------------------------------------------------------------------------- +// Actual Code + +/* +Comparing VMAF instantly: +- Saves disk space - no need to keep copies around! +fn: .\ffmpeg.exe -i "..\..\cache\arma_3-002-1280x720x60.00.mkv" -i "..\..\videos\arma_3-002.mkv" -filter_complex_threads 8 -filter_complex [0:v:0]scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=1920:h=1080,colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[main];[1:v:0]colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[ref];[main][ref]libvmaf=model_path=../vmaf/vmaf_4k_rb_v0.6.2.pkl:log_fmt=json:log_path=here2.json:enable_conf_interval=1:shortest=1[out] -map [out] -f null - + +*/ + +// Import modules +const ffmpeg = require('./ffmpeg.js'); +const poolqueue = require('./poolqueue.js'); + +const fs = require('fs'); +const path = require('path'); +const process = require('process'); +const util = require('util'); +const { config } = require('process'); + +// Define some Helpers +function float_eq(a, b, edge) { return (Math.abs(a - b) <= edge); } +function float_lt(a, b, edge) { return ((a + edge) < b); } +function float_le(a, b, edge) { return float_lt(a, b, edge) || float_eq(a, b, edge); } +function float_gt(a, b, edge) { return ((a - edge) > b); } +function float_ge(a, b, edge) { return float_gt(a, b, edge) || float_eq(a, b, edge); } +Object.size = function(obj) { var size = 0, key; for (key in obj) { if (obj.hasOwnProperty(key)) { size++; } } return size; }; + +// Actual Code +async function load_config() { // Load Configuration + console.group("Loading configuration..."); + console.time("Total"); + + let config = require('./config.json'); + // Resolve and create directories. + for (let name in config.paths) { + config.paths[name] = path.resolve(config.paths[name]); + if (!fs.existsSync(config.paths[name])) { + fs.mkdirSync(config.paths[name]); + } + } + + console.timeEnd("Total"); + console.groupEnd(); + return config; +} + +async function load_encoders(config, ff) { // Load Encoders + console.group("Loading encoders..."); + console.time("Total"); + + let encoders = new Map(); + for (let name in config.encoders) { + let encoder = config.encoders[name]; + + // Is the encoder disabled, or invalid? + if (!encoder || !encoder.enabled) { + if (global.debug) console.debug(`${name} is disabled or invalid.`); + continue; + } + + // Does the encoder exist? + if (!fs.existsSync(path.join(".", `encoder_${name}.js`))) { + console.error(`${name} does not exist.`); + continue; + } + + // Attempt to load the encoder, if possible. + console.group(name); + console.time("Subtotal"); + try { + let instance = new (require(`./encoder_${name}.js`))(ff, config, encoder); + instance.load(); + encoders.set(name, instance); + } catch (ex) { + console.log(ex); + } + console.timeEnd("Subtotal"); + console.groupEnd(); + } + + console.timeEnd("Total"); + console.groupEnd(); + return encoders; +} + +async function load_videos(config, ff) { // Load Videos + console.group("Loading videos..."); + console.time("Total"); + + let videos = new Map(); + let promises = []; // Asynchronous loading. + for (let name in config.videos) { + let video = config.videos[name]; + + // Is the video disabled, or invalid? + if (!video || !video.enabled) { + if (global.debug) console.debug(`${name} is disabled or invalid.`); + continue; + } + + // Does the video exist? + if (!fs.existsSync(path.join(config.paths.videos, `${name}.mkv`))) { + console.error(`${name} does not exist.`); + continue; + } + + // Load video information asynchronously. + promises.push(new Promise((resolve, reject) => { + console.time(name); + + let file_name = path.join(config.paths.videos, `${name}.mkv`); + let probeprom = ff.probe(file_name); + probeprom.then((json) => { + let data = new Object(); + + data.name = name; + data.info = json; + data.caches = new Map(); + + // File Information + data.file_name = `${name}.mkv`; + data.file_ext = path.extname(data.file_name); + data.file_base = path.basename(data.file_name, data.file_ext); + data.file = path.join(config.paths.videos, data.file_name); + + // Video Information + data.resolution = { width: data.info.streams[0].width, height: data.info.streams[0].height }; + data.framerate = eval(data.info.streams[0].r_frame_rate); + data.duration = data.info.streams[0].duration; + + // Color Information + data.color = {}; + data.color.range = data.info.streams[0].color_range ? data.info.streams[0].color_range : 'tv'; + data.color.trc = data.info.streams[0].color_transfer ? data.info.streams[0].color_transfer : 'bt709'; + data.color.primaries = data.info.streams[0].color_primaries ? data.info.streams[0].color_primaries : 'bt709'; + data.color.matrix = data.info.streams[0].color_space ? data.info.streams[0].color_space : 'bt709'; + + videos.set(data.name, data); + console.timeEnd(name); + resolve(data); + }); + })); + } + + await Promise.allSettled(promises); + + // Sort the videos list again. (We don't care about what order the user wants!) + videos = new Map([...videos].sort((a, b) => a[0] > b[0] ? 1 : -1)); + + console.timeEnd("Total"); + console.groupEnd(); + return videos; +} + +async function create_caches(config, ff, videos, encoders) { // Create Caches + console.group("Creating caches..."); + console.time("Total"); + + for (let name of videos.keys()) { + let video = videos.get(name); + + console.group(name); + console.time("Subtotal"); + + video.caches = new Map(); + for (let resolution of config.options.resolutions) { + let aspect_ratio = (video.resolution.height / video.resolution.width); + for (let framerate_scaling of config.options.framerate_scalings) { + let height = (Math.round(resolution[0] * aspect_ratio * 0.5) * 2); + // Skip resolutions that are too large horizontally. + if (resolution[0] > video.resolution.width) { + if (global.debug) console.debug(`${resolution[0]}x${height}x${(video.framerate * framerate_scaling)} skipped.`); + continue; + } + + // Create Cache Information + let cache = { + framerate: (video.framerate * framerate_scaling), + width: resolution[0], + height: height + }; + cache.framerate_s = cache.framerate.toFixed(2); + cache.duration = Math.floor(video.duration * video.framerate * framerate_scaling) / cache.framerate; + let key = `${cache.width}x${cache.height}x${cache.framerate_s}`; + cache.file = path.join(config.paths.cache, `${video.file_base}-${key}.mkv`); + cache.queues = new Map(); + + video.caches.set(`${key}`, cache); + } + } + + for (let key of video.caches.keys()) { + let cache = video.caches.get(key); + + // Check if cache file already exists and is correct. + if (fs.existsSync(cache.file)) { + let info = ff.probeSync(cache.file); + if ((info.streams) + && (info.streams.length > 0) + && (info.streams[0].width == cache.width) + && (info.streams[0].height == cache.height) + && float_eq(eval(info.streams[0].r_frame_rate), cache.framerate, 0.01) + && float_eq(info.streams[0].duration, video.duration, 0.1)) { + if (global.debug) console.debug(`${key} already exists.`); + continue; + } + } + + console.time(`${key} created`); + let command = [ + "-y", + "-hide_banner", + "-v", "error", + "-i", video.file, + "-filter_complex", `fps=fps=${cache.framerate_s},scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=${cache.width}:h=${cache.height},colorspace=all=bt709:range=tv:format=yuv420p`, + "-an", + ]; + if (encoders.has("h264_nvenc")) { + command.push( + "-c:v", "h264_nvenc", + "-profile:v", "high", + "-preset", "p1", + "-tune", "lossless", + "-rc", "constqp", + "-rc-lookahead", "0", + "-multipass", "0", + "-b:v", "0", + "-minrate", "0", + "-maxrate", "0", + "-bufsize", "0", + "-qp", "0", + "-init_qpI", "0", + "-init_qpP", "0", + "-init_qpB", "0", + "-bf", "0", + "-g", `15`, + ); + } else { + command.push( + "-c:v", "libx264", + "-profile:v", "high", + "-preset", "veryfast", + "-crf", "0", + "-b:v", "0", + "-minrate", "0", + "-maxrate", "0", + "-bufsize", "0", + "-g", `15`, + ); + } + command.push(cache.file); + let res = ff.ffmpegSync(command); + if (res.status != 0) { + console.log(res.stderr.toString()); + } + if (global.debug) console.log(res.stdout.toString(), res.stderr.toString()); + console.timeEnd(`${key} created`); + } + + console.timeEnd("Subtotal"); + console.groupEnd(name); + } + + console.timeEnd("Total"); + console.groupEnd(); + return videos; +} + +async function transcode(config, ff, videos, encoders) { + console.group("Transcoding..."); + console.time("Total"); + + /* Needs a massive overhaul: + - Check if the .json file for a configuration already exists, if not continue. + - Do not store transcodes for longer than needed, we only need to compare against source. + - Use poolqueue to still allow for parallelization. + - '-g' and bitrate options are controlled _from here_. + */ + + // Build queues per cache. + console.group("Queueing...") + console.time("Subtotal"); + for (let video_key of videos.keys()) { + console.group(video_key); + console.time(video_key); + let video = videos.get(video_key); + for (let cache_key of video.caches.keys()) { + let cache = video.caches.get(cache_key); + console.time(cache_key); + + for (let encoder_key of encoders.keys()) { + let encoder = encoders.get(encoder_key); + + let queue_promises = []; + let queue_commands = new poolqueue(); + let queue_files = new poolqueue(); + for (let idx = 0; idx < encoder.count(); idx++) { + let command_promises = []; + let command = encoder.get(idx, cache.width, cache.height, cache.framerate); + for (let bitrate of config.options.bitrates) { + for (let kfinterval of config.options.keyframeinterval) { + command_promises.push(new Promise((resolve, reject) => { + let file = path.join( + config.paths.output, + video.name, + cache_key, + encoder_key, + bitrate.toFixed(0), + (cache.framerate * kfinterval).toFixed(0), + `${command.hash}.mkv` + ); + let file_json = path.join( + config.paths.output, + video.name, + cache_key, + encoder_key, + bitrate.toFixed(0), + (cache.framerate * kfinterval).toFixed(0), + `${command.hash}.json` + ); + + // Check if this file already exists at the target location. + if (fs.existsSync(file_json)) { + // Don't need to check for a video here, since we only care about results. + if (global.debug) console.debug(`${file} already completed.`); + resolve(); + return; + } + + let line = [ + "-map", "0:v:0", + "-an", + "-c:v", encoder_key, + "-g", (cache.framerate * kfinterval).toFixed(0), + "-b:v", `${bitrate}k`, + "-minrate", "0", + "-maxrate", "0", + "-bufsize", `${2 * bitrate}k`, + ].concat(command.options).concat([file]); + + queue_commands.push( + encoder.pool(), + line, + command.cost, + ); + queue_files.push( + encoder.pool(), + [file, file_json], + command.cost, + ); + + resolve(); + })); + } + } + queue_promises.push(async function() { + await Promise.allSettled(command_promises); + }); + } + await Promise.allSettled(queue_promises); + + cache.queues.set(encoder_key, { + commands: queue_commands.finalize(), + files: queue_files.finalize(), + }); + } + + console.timeEnd(cache_key); + } + console.timeEnd(video_key); + console.groupEnd(); + } + console.timeEnd("Subtotal"); + console.groupEnd(); + + + + /* + for (let video_name in videos) { + console.time(video_name); + console.group(video_name); + let video = videos[video_name]; + for (let cache of video.caches) { + let key_cache = `${cache.x}x${cache.y}x${cache.fps.toFixed(2)}`; + console.time(key_cache); + console.group(key_cache); + for (let cmd of cache.commands) { + let opts = [ + "-y", + "-hide_banner", + "-v", "error", + "-hwaccel", "auto", + "-i", cache.file + ].concat(cmd); + let res = ff.ffmpegSync(opts); + console.log(res.stdout.toString(), res.stderr.toString()); + } + console.groupEnd(); + console.timeEnd(key_cache); + } + console.groupEnd(); + console.timeEnd(video_name); + } + */ + + console.timeEnd("Total"); + console.groupEnd(); +} + +async function main() { + let config; + let ff; + let encoders; + let videos; + + await load_config().then((p) => { config = p; }); + ff = new ffmpeg(config.paths.ffmpeg); + await load_encoders(config, ff).then((p) => { encoders = p; }); + await load_videos(config, ff).then((p) => { videos = p; }); + await create_caches(config, ff, videos, encoders); + await transcode(config, ff, videos, encoders); + + + // Queue + //await queue_transcodes(config, ff, videos, encoders); + + // Transcode +} + +main(); diff --git a/videos/README.md b/videos/README.md new file mode 100644 index 0000000..acff627 --- /dev/null +++ b/videos/README.md @@ -0,0 +1 @@ +Place input video files here. \ No newline at end of file