Changes:

* Added configurable scenecut.
* Removed bframes 0 and 1 from nvenc default config.
* Added b_ref_mode configuration option.
* Fixed path parallelization.
* Attempted libvmaf parallelization.

config.json

@@ -25,16 +25,17 @@
 				"film",
 				"grain",
 				"animation"
+			],
+			"scenecut": [
+				false
 			]
 		},
 		"h264_nvenc": {
 			"enabled": true,
 			"pool": "nvenc",
-			"parallel": 2,
+			"parallel": 3,
 			"gpu": -1,
 			"presets": [
-				"p5",
-				"p6",
 				"p7"
 			],
 			"tunes": [
@@ -45,18 +46,22 @@
 				16,
 				32
 			],
-			"bf": [
+			"bframes": [
-				0,
-				1,
 				2,
 				3,
 				4
+			],
+			"bframe_reference_mode": [
+				"middle"
+			],
+			"scenecut": [
+				false
 			]
 		},
 		"hevc_nvenc": {
 			"enabled": false,
 			"pool": "nvenc",
-			"parallel": 2,
+			"parallel": 3,
 			"gpu": -1
 		},
 		"h264_amf": {
@@ -182,6 +187,9 @@
 		}
 	},
 	"options": {
+		"vmaf": {
+			"model": "vmaf_4k_rb_v0.6.2.pkl"
+		},
 		"resolutions": [
 			[
 				1280,

encoder_h264_nvenc.js

@@ -63,12 +63,12 @@ class h264_nvenc extends encoder {
 		for (let preset of this.settings.presets) {
 			for (let tune of this.settings.tunes) {
 				for (let rcla of this.settings["rc-lookahead"]) {
-					for (let ia of [0, 1]) {
+					for (let scenecut of this.settings.scenecut) {
-						if ((rcla == 0) && (ia != 0)) { // Requires lookahead.
+						if ((rcla == 0) && (scenecut == true)) { // Requires lookahead.
 							continue;
 						}
-						for (let bf of this.settings.bf) {
+						for (let bf of this.settings.bframes) {
-							for (let bfm of ["disabled", "middle"]) {
+							for (let bfm of this.settings.bframe_reference_mode) {
 								if ((bf == 0) && (bfm != "disabled")) { // Requires B-Frames
 									continue;
 								}
@@ -82,7 +82,7 @@ class h264_nvenc extends encoder {
 												"-rc", "cbr",
 												"-cbr", 1,
 												"-rc-lookahead", rcla,
-												"-no-scenecut", 1 - ia,
+												"-no-scenecut", scenecut == true ? 0 : 1,
 												"-bf", bf,
 												"-b_ref_mode", bfm,
 												"-b_adapt", 1,
@@ -124,7 +124,7 @@ class h264_nvenc extends encoder {
 		
 		fs.writeFileSync(
 			path.join(this.config.paths.output, "h264_nvenc.json"),
-			JSON.stringify(this.indexes, null, null),
+			JSON.stringify(this.indexes, null, '\t'),
 			{encoding: "utf8"}
 		);
 		if (global.debug) console.timeEnd("Generating...");

encoder_libx264.js

@@ -68,35 +68,43 @@ class libx264 extends encoder {
 		this.combinations = [];
 		for (let preset of this.settings.presets) {
 			for (let tune of this.settings.tunes) {
-				let _opts = [
+				for (let scenecut of this.settings.scenecut) {
-					"-profile:v", "high",
+					let _opts = [
-					"-preset", preset,
+						"-profile:v", "high",
-					"-x264-params", `nal-hrd=cbr:force-cfr=1`,
+						"-preset", preset,
-					"-ssim", "0",
+						"-x264-params", `nal-hrd=cbr:force-cfr=1`,
-					"-threads", this.settings.threads,
+						"-ssim", "0",
-				];
+						"-threads", this.settings.threads,						
-				if (tune) {
+					];
-					_opts.push("-tune", tune);
+					if (tune) {
+						_opts.push("-tune", tune);
+					}
+					if (scenecut == false) {
+						_opts.push("-sc_threshold", 0);
+					} else if ((typeof scenecut) == "number") {
+						_opts.push("-sc_threshold", scenecut.toFixed(0));
+					}
+	
+					let _name = name(_opts);
+					let _hash = crypto.createHash("sha256").update(_name).digest("hex");
+					let _cost = 1.0 * this.settings.cost_scale * this.cost.preset[preset] * this.cost.tune[tune] * this.cost.threads;
+					let combo = {
+						name: _name,
+						hash: _hash,
+						options: _opts,
+						cost: _cost,
+					};
+	
+					this.indexes[_hash] = combo.options;
+					this.combinations.push(combo);
+
 				}
-
-				let _name = name(_opts);
-				let _hash = crypto.createHash("sha256").update(_name).digest("hex");
-				let _cost = 1.0 * this.settings.cost_scale * this.cost.preset[preset] * this.cost.tune[tune] * this.cost.threads;
-				let combo = {
-					name: _name,
-					hash: _hash,
-					options: _opts,
-					cost: _cost,
-				};
-
-				this.indexes[_hash] = combo.options;
-				this.combinations.push(combo);
 			}
 		}
 		
 		fs.writeFileSync(
 			path.join(this.config.paths.output, "libx264.json"),
-			JSON.stringify(this.indexes, null, null),
+			JSON.stringify(this.indexes, null, '\t'),
 			{encoding: "utf8"}
 		);
 		if (global.debug) console.timeEnd("Generating...");

ffmpeg.js

@@ -1,6 +1,7 @@
 const path = require('path');
 const process = require('process');
 const child_process = require('child_process');
+const os = require('os');
 
 function parse_duration(str) {
 	// Duration is in the form HH:MM:SS.fraction
@@ -152,6 +153,20 @@ class ffmpeg {
 			})
 		});
 	}
+
+	consolify(_file) {
+		//_file = path.resolve(_file);
+		if (os.platform() == 'win32') {
+			// Need to turn:
+			//   C:\myfiles\model.pkl
+			// into:
+			//   C\\:/myfiles/model.pkl
+			_file = _file.replace(/\\/g, '/').replace(':', '\\\\:');
+		} else {
+			// No further work needs to be done?
+		}
+		return _file;
+	}
 }
 
 module.exports = ffmpeg;

ves.js

@@ -6,13 +6,6 @@ global.debug = false;
 // --------------------------------------------------------------------------------
 // Actual Code
 
-/*
-Comparing VMAF instantly:
-- Saves disk space - no need to keep copies around!
-fn: .\ffmpeg.exe -i "..\..\cache\arma_3-002-1280x720x60.00.mkv" -i "..\..\videos\arma_3-002.mkv" -filter_complex_threads 8 -filter_complex [0:v:0]scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=1920:h=1080,colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[main];[1:v:0]colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[ref];[main][ref]libvmaf=model_path=../vmaf/vmaf_4k_rb_v0.6.2.pkl:log_fmt=json:log_path=here2.json:enable_conf_interval=1:shortest=1[out] -map [out] -f null -
-
-*/
-
 // Import modules
 const ffmpeg = require('./ffmpeg.js');
 const poolqueue = require('./poolqueue.js');
@@ -271,20 +264,10 @@ async function create_caches(config, ff, videos, encoders) { // Create Caches
 	return videos;
 }
 
-async function transcode(config, ff, videos, encoders) {
+async function queue(config, ff, videos, encoders) {
-	console.group("Transcoding...");
+	console.group("Queueing...");
 	console.time("Total");
 
-	/* Needs a massive overhaul:
-	- Check if the .json file for a configuration already exists, if not continue.
-	- Do not store transcodes for longer than needed, we only need to compare against source.
-	- Use poolqueue to still allow for parallelization. 
-	- '-g' and bitrate options are controlled _from here_.
-	*/
-
-	// Build queues per cache.
-	console.group("Queueing...")
-	console.time("Subtotal");
 	let promises = [];
 	for (let video_key of videos.keys()) {
 		promises.push(new Promise(async (resolve, reject) => {
@@ -349,7 +332,7 @@ async function transcode(config, ff, videos, encoders) {
 										].concat(command.options).concat(encoder_extra).concat([file]);
 
 										queue_commands.push(encoder_pool, line, command.cost);
-										queue_files.push(encoder_pool, [file, file_json], command.cost);
+										queue_files.push(encoder_pool, [[file, file_json]], command.cost);
 
 										resolve2(true);
 									}));
@@ -377,9 +360,11 @@ async function transcode(config, ff, videos, encoders) {
 		}));
 	}
 	await Promise.allSettled(promises);
-	console.timeEnd("Subtotal");
+	console.timeEnd("Total");
 	console.groupEnd();
+}
 
+async function work(config, ff, videos, encoders) {
 	// Process from here on out.
 	// LOOP
 	// 1. Pull out front of the command and file queue.
@@ -388,53 +373,122 @@ async function transcode(config, ff, videos, encoders) {
 	// 4. Delete encoded files.
 	// 5. Repeat until queues empty, no more caches for video, and no more videos.
 
-	/*
+	let vmaf_model = ff.consolify(path.resolve(path.join(config.paths.ffmpeg, "vmaf", config.options.vmaf.model)));
-		console.group("Queueing...")
-		console.time("Subtotal");
-		for (let video_key of videos.keys()) {
-			console.group(video_key);
-			console.time(video_key);
-			let video = videos.get(video_key);
-			for (let cache_key of video.caches.keys()) {
-				let cache = video.caches.get(cache_key);
-				console.time(cache_key);
-	
-				console.timeEnd(cache_key);
-			}
-			console.timeEnd(video_key);
-			console.groupEnd();
-		}
-		console.timeEnd("Subtotal");
-		console.groupEnd();
-	*/
-	/*
-	for (let video_name in videos) {
-		console.time(video_name);
-		console.group(video_name);
-		let video = videos[video_name];
-		for (let cache of video.caches) {
-			let key_cache = `${cache.x}x${cache.y}x${cache.fps.toFixed(2)}`;
-			console.time(key_cache);
-			console.group(key_cache);
-			for (let cmd of cache.commands) {
-				let opts = [
-					"-y",
-					"-hide_banner",
-					"-v", "error",
-					"-hwaccel", "auto",
-					"-i", cache.file
-				].concat(cmd);
-				let res = ff.ffmpegSync(opts);
-				console.log(res.stdout.toString(), res.stderr.toString());
-			}
-			console.groupEnd();
-			console.timeEnd(key_cache);
-		}
-		console.groupEnd();
-		console.timeEnd(video_name);
-	}
-	*/
 
+	console.group("Processing...")
+	console.time("Total");
+	for (let video_key of videos.keys()) {
+		console.group(video_key);
+		console.time(video_key);
+		let video = videos.get(video_key);
+		for (let cache_key of video.caches.keys()) {
+			console.time(cache_key);
+			let cache = video.caches.get(cache_key);
+
+			let length = cache.queues.commands.length;
+			console.log(`0.0% (0 / ${length}): 0.000s`);
+			while (cache.queues.commands.length > 0) {
+				let commands = cache.queues.commands.shift();
+				let files = cache.queues.files.shift();
+
+				let prc1 = length - cache.queues.commands.length;
+				let prc2 = prc1 / length * 100.0;
+				let LABEL = `${prc2.toFixed(1)}% (${prc1} / ${length})`
+				console.time(LABEL)
+				console.group();
+
+				// Create directories.
+				for (let file of files) {
+					fs.mkdirSync(path.dirname(file[0]), { recursive: true });
+				}
+
+				// Encode
+				{
+					console.time("Encoding");
+					let opts = [
+						"-y",
+						"-hide_banner",
+						"-v", "error",
+						"-hwaccel", "auto",
+						"-i", cache.file
+					].concat(commands);
+					let res = ff.ffmpegSync(opts);
+					if (res.status != 0) {
+						console.log(res.stdout.toString(), res.stderr.toString());
+						continue;
+					}
+					console.timeEnd("Encoding");
+				}
+
+				/*
+				Comparing VMAF instantly:
+				- Saves disk space - no need to keep copies around!
+				fn: .\ffmpeg.exe -i "..\..\cache\arma_3-002-1280x720x60.00.mkv" -i "..\..\videos\arma_3-002.mkv" -filter_complex_threads 8 -filter_complex [0:v:0]scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=1920:h=1080,colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[main];[1:v:0]colorspace=all=bt709:range=pc,format=pix_fmts=yuv444p[ref];[main][ref]libvmaf=model_path=../vmaf/vmaf_4k_rb_v0.6.2.pkl:log_fmt=json:log_path=here2.json:enable_conf_interval=1:shortest=1[out] -map [out] -f null -
+				
+				*/
+
+				// Process
+				{
+					console.time("Processing");
+					let opts = [
+						"-hide_banner",
+						"-v", "warning",
+						"-hwaccel", "auto",
+						"-i", video.file,
+					];
+					let filter = "";
+					let references = [];
+
+					// Build Filter Graph
+					if (files.length > 1) {
+						filter = `[0:v:0]split=${files.length}`
+						for (let idx = 0; idx < files.length; idx++) {
+							filter = `${filter}[ref:${idx}]`;
+							references[idx] = `[ref:${idx}]`;
+						}
+						filter = `${filter}`
+					} else {
+						references[0] = "[0:v:0]";
+					}
+
+					// Rescale and Resample all inputs and compare them.
+					for (let idx = 0; idx < files.length; idx++) {
+						let file = files[idx];						
+						opts.push("-i", file[0]);
+
+						if (filter != "")
+							filter = `${filter};` // [temp:${idx}];[temp:${idx}]
+						filter = `${filter}[${idx}:v:0]scale=flags=bicubic+full_chroma_inp+full_chroma_int:w=${video.resolution.width.toFixed(0)}:h=${video.resolution.height.toFixed(0)},colorspace=space=${video.color.matrix}:trc=${video.color.trc}:primaries=${video.color.primaries}:range=${video.color.range},format=pix_fmts=yuv444p,fps=fps=${video.framerate.toFixed(2)},[ref:${idx}]libvmaf=model_path=${vmaf_model}:log_fmt=json:log_path=${ff.consolify(file[1])}:enable_conf_interval=1:n_threads=2[main:${idx}]`
+					}
+
+					opts.push("-filter_complex", filter);
+					for (let idx = 0; idx < files.length; idx++) {
+						opts.push(
+							"-map", `[main:${idx}]`,
+							"-f", "null",
+							"-"
+						)
+					}
+
+					console.log(opts);
+					let res = ff.ffmpegSync(opts);
+					if (res.status != 0) {
+						console.log(res.stdout.toString(), res.stderr.toString());
+						continue;
+					}
+					console.timeEnd("Processing");
+				}
+
+				console.log(filter);
+
+				console.groupEnd();
+				console.timeEnd(LABEL);
+			}
+			console.timeEnd(cache_key);
+		}
+		console.timeEnd(video_key);
+		console.groupEnd();
+	}
 	console.timeEnd("Total");
 	console.groupEnd();
 }
@@ -450,13 +504,8 @@ async function main() {
 	await load_encoders(config, ff).then((p) => { encoders = p; });
 	await load_videos(config, ff).then((p) => { videos = p; });
 	await create_caches(config, ff, videos, encoders);
-	await transcode(config, ff, videos, encoders);
+	await queue(config, ff, videos, encoders);
-
+	await work(config, ff, videos, encoders);
-
-	// Queue
-	//await queue_transcodes(config, ff, videos, encoders);
-
-	// Transcode
 }
 
 main();