ui/prores_aw_handler: Override automatic color format when profile demands it

Certain ProRes profiles require a very specific color format, otherwise the avcodec_open2 will fail with EPERM, which is very confusing.

.github/workflows/main.yml

@@ -0,0 +1,34 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [windows-2016, windows-2019]
+        include:
+          - os: windows-2016
+            generator_32: "Visual Studio 15 2017"
+            generator_64: "Visual Studio 15 2017 Win64"
+            sysversion: "10.0.17763.0"
+          - os: windows-2019
+            generator_32: 
+            generator_64: "Visual Studio 16 2019"
+            sysversion: "10.0.18362.0"
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Clone Repository
+      uses: actions/checkout@v1      
+    - name: Update Submodules
+      run: git submodule update --init --force --recursive
+    - name: Install Node.JS 10.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 10
+    - name: Build
+      env:
+        CMAKE_GENERATOR_32: ${{ matrix.generator_32 }}
+        CMAKE_GENERATOR_64: ${{ matrix.generator_64 }}
+        CMAKE_SYSTEM_VERSION: ${{ matrix.sysversion }}
+      run: node ./ci/builder.js

CMakeLists.txt

@@ -155,7 +155,7 @@ mark_as_advanced(FORCE OBS_NATIVE OBS_PACKAGE OBS_REFERENCE OBS_DOWNLOAD)
 
 if(NOT TARGET libobs)
 	set(${PropertyPrefix}OBS_STUDIO_DIR "" CACHE PATH "OBS Studio Source/Package Directory")
-	set(${PropertyPrefix}OBS_DOWNLOAD_VERSION "23.2.1-ci" CACHE STRING "OBS Studio Version to download")
+	set(${PropertyPrefix}OBS_DOWNLOAD_VERSION "24.0.0-rc2-ci" CACHE STRING "OBS Studio Version to download")
 endif()
 
 if(NOT ${PropertyPrefix}OBS_NATIVE)
@@ -272,6 +272,8 @@ set(PROJECT_PRIVATE
 	"${PROJECT_SOURCE_DIR}/source/codecs/hevc.cpp"
 	"${PROJECT_SOURCE_DIR}/source/codecs/h264.hpp"
 	"${PROJECT_SOURCE_DIR}/source/codecs/h264.cpp"
+	"${PROJECT_SOURCE_DIR}/source/codecs/prores.hpp"
+	"${PROJECT_SOURCE_DIR}/source/codecs/prores.cpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/avframe-queue.cpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/avframe-queue.hpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/swscale.hpp"

appveyor.yml

@@ -7,11 +7,16 @@ matrix:
 # Build Image & Environment
 platform: x64
 
+# Build Tags only
+skip_non_tags: true
+
 image:
   - Visual Studio 2017
 
 environment:
   CMAKE_SYSTEM_VERSION: 10.0.17134.0
+  CMAKE_GENERATOR_32: "Visual Studio 15 2017"
+  CMAKE_GENERATOR_64: "Visual Studio 15 2017 Win64"
   PACKAGE_PREFIX: obs-ffmpeg-encoder
   INNOSETUP_URL: http://www.jrsoftware.org/download.php/is.exe
   CURL_VERSION: 7.39.0
@@ -32,6 +37,7 @@ build_script:
 - cmd: node ci/builder.js
 
 after_build:
+- cmd: node ci/packager.js
 - cmd: ci/appveyor-package.bat
 
 # Testing

ci/builder.js

@@ -2,108 +2,94 @@
 
 const process = require('process');
 const runner = require('./runner.js');
+let env = process.env;
 
-// Steps
+let x32_steps = [];
-let configure_runners = [];
+let x64_steps = [];
-let build_runners = [];
-let package_runners = [];
 
-{
+if ((process.platform == "win32") || (process.platform == "win64")) {
-	let cmake_configure_extra = [
+	// Windows
+	let extra_conf = [
 		`-DCMAKE_SYSTEM_VERSION=${process.env.CMAKE_SYSTEM_VERSION}`,
+		`-DCMAKE_PACKAGE_NAME=obs-ffmpeg-encoder`,
 		'-DCMAKE_INSTALL_PREFIX="build/distrib/"',
 		'-DCMAKE_PACKAGE_PREFIX="build/"',
-		`-DCMAKE_PACKAGE_NAME="${process.env.PACKAGE_PREFIX}"`,
 	];
-	let cmake_build_extra = [
+	let extra_build = [
+
 	];
 
-	// Configuration depends on platform
+	if(process.env.APPVEYOR) {
-	if (process.platform == "win32" || process.platform == "win64") {
+		extra_build.concat(['--', '/logger:"C:\\Program Files\\AppVeyor\\BuildAgent\\Appveyor.MSBuildLogger.dll"']);
-		configure_runners.push(new runner('32-bit', 'cmake', [
+	}
+
+	if ((process.env.CMAKE_GENERATOR_32 !== undefined) && (process.env.CMAKE_GENERATOR_32 !== "")) {
+		x32_steps.push(
+			[ 'cmake', [
 				'-H.',
 				'-Bbuild/32',
-				`-G"Visual Studio 15 2017"`,
+				`-G"${process.env.CMAKE_GENERATOR_32}"`,
-			].concat(cmake_configure_extra)));
+			].concat(extra_conf), env ]
-		configure_runners.push(new runner('64-bit', 'cmake', [
+		);
+		x32_steps.push(
+			[ 'cmake', [
+				'--build', 'build/32', 
+				'--config', 'RelWithDebInfo',
+				'--target', 'INSTALL'
+			].concat(extra_build), env ]
+		);
+	}
+	if ((process.env.CMAKE_GENERATOR_64 !== undefined) && (process.env.CMAKE_GENERATOR_64 !== "")) {
+		x64_steps.push(
+			[ 'cmake', [
 				'-H.',
 				'-Bbuild/64',
-				`-G"Visual Studio 15 2017 Win64"`,
+				`-G"${process.env.CMAKE_GENERATOR_64}"`,
-				'-T"host=x64"',
+				'-T"host=x64"'
-			].concat(cmake_configure_extra)));
+			].concat(extra_conf), env ]
-		
+		);
-		// Extra build steps for AppVeyor on Windows for Logging purposes.
+		x64_steps.push(
-		if(process.env.APPVEYOR) {
+			[ 'cmake', [
-			cmake_build_extra.concat(['--', '/logger:"C:\\Program Files\\AppVeyor\\BuildAgent\\Appveyor.MSBuildLogger.dll"']);
+				'--build', 'build/64', 
-		}
+				'--config', 'RelWithDebInfo',
-	} else if (process.platform == "linux") {
+				'--target', 'INSTALL'
-		configure_runners.push(new runner('32-bit', 'cmake', [
+			].concat(extra_build), env ]
-				'-H.',
+		);
-				'-Bbuild32',
-				`-G"Unix Makefiles"`,
-				`-DCOPIED_DEPENDENCIES=false`,
-			].concat(cmake_configure_extra),
-			{ ...process.env, ...{
-				CFLAGS: `${process.env.COMPILER_FLAGS_32}`,
-				CXXFLAGS: `${process.env.COMPILER_FLAGS_32}`,
-			}}));
-		configure_runners.push(new runner('64-bit', 'cmake', [
-				'-H.',
-				'-Bbuild64',
-				`-G"Unix Makefiles"`,
-				`-DCOPIED_DEPENDENCIES=false`,
-			].concat(cmake_configure_extra),
-			{ ...process.env, ...{
-				CFLAGS: `${process.env.COMPILER_FLAGS_64}`,
-				CXXFLAGS: `${process.env.COMPILER_FLAGS_64}`,
-			}}));
 	}
-	
+} else {
-	build_runners.push(new runner('32-bit', 'cmake', [
+	// Unix
-		'--build', 'build/32', 
+
-		'--config', 'RelWithDebInfo',
-		'--target', 'INSTALL'
-	].concat(cmake_build_extra)));
-	build_runners.push(new runner('64-bit', 'cmake', [
-		'--build', 'build/64', 
-		'--config', 'RelWithDebInfo',
-		'--target', 'INSTALL'
-	].concat(cmake_build_extra)));
-	package_runners.push(new runner('32-bit', 'cmake', [
-		'--build', 'build/32',
-		'--target', 'PACKAGE_7Z',
-		'--config', 'RelWithDebInfo'
-	].concat(cmake_build_extra)));
-	package_runners.push(new runner('64-bit', 'cmake', [
-		'--build', 'build/64',
-		'--target', 'PACKAGE_ZIP',
-		'--config', 'RelWithDebInfo'
-	].concat(cmake_build_extra)));
 }
 
-// Run Configure steps.
+function runRunners(runnerArray, name) {
-let configure_promises = [];
+	return new Promise(async (resolve, reject) => {
-for (let config of configure_runners) {
+		let local = runnerArray.reverse();
-	configure_promises.push(config.run());
+		while (local.length > 0) {	
-}
+			try {
-Promise.all(configure_promises).then(function(result) {    
+				let task = local.pop();
-	let build_promises = [];
+				let work = new runner(name, task[0], task[1], task[2]);
-	for (let build of build_runners) {
+				await work.run();
-		build_promises.push(build.run());
+			} catch (e) {
-	}
+				reject(e);
-	Promise.all(build_promises).then(function(result) {    
+				return;
-		let package_promises = [];
+			}
-		for (let pack of package_runners) {
-			package_promises.push(pack.run());
 		}
-		Promise.all(package_promises).then(function(result) {    
+		resolve(0);
-			process.exit(result);
-		}).catch(function(result) {
-			process.exit(result);
-		});
-	}).catch(function(result) {
-		process.exit(result);
 	});
-}).catch(function(result) {
+}
-	process.exit(result);
+
-});
+let promises = [];
+promises.push(runRunners(x32_steps, "32-Bit"));
+promises.push(runRunners(x64_steps, "64-Bit"));
+Promise.all(promises).then(
+	res => {
+		process.exit(0);
+	},
+	err => {
+		console.log(err);
+		process.exit(1);
+	}
+).catch(err => {
+	console.log(err);
+	process.exit(1);
+})

ci/packager.js

@@ -0,0 +1,63 @@
+"use strict";
+
+const process = require('process');
+const runner = require('./runner.js');
+
+function runRunners(runnerArray, name) {
+	return new Promise(async (resolve, reject) => {
+		let local = runnerArray.reverse();
+		while (local.length > 0) {	
+			let task = local.pop();
+			let work = new runner(name, task[0], task[1], task[2]);
+			await work.run();
+		}
+		resolve(0);
+	});
+}
+
+let env = process.env;
+let steps = [];
+
+if ((process.env.CMAKE_GENERATOR_64 !== undefined) && (process.env.CMAKE_GENERATOR_64 !== "")) {
+	steps.push(
+		[ 'cmake', [
+			'--build', 'build/64',
+			'--config', 'RelWithDebInfo',
+			'--target', 'PACKAGE_7Z'
+		], env ]
+	);
+	steps.push(
+		[ 'cmake', [
+			'--build', 'build/64',
+			'--config', 'RelWithDebInfo',
+			'--target', 'PACKAGE_ZIP'
+		], env ]
+	);
+} else if ((process.env.CMAKE_GENERATOR_32 !== undefined) && (process.env.CMAKE_GENERATOR_32 !== "")) {
+	steps.push(
+		[ 'cmake', [
+			'--build', 'build/32',
+			'--config', 'RelWithDebInfo',
+			'--target', 'PACKAGE_7Z'
+		], env ]
+	);
+	steps.push(
+		[ 'cmake', [
+			'--build', 'build/32',
+			'--config', 'RelWithDebInfo',
+			'--target', 'PACKAGE_ZIP'
+		], env ]
+	);
+}
+
+let promises = [];
+promises.push(runRunners(steps, "32-Bit"));
+Promise.all(promises).then(
+	res => {
+		process.exit(0);
+	},
+	err => {
+		console.log(err);
+		process.exit(1);
+	}
+)

ci/runner.js

@@ -18,7 +18,6 @@ class Runner {
 	run() {
         let self = this;
         return new Promise(function(resolve, reject) {
-			console.log(self.cmd, self.args);
             self.proc = cp.spawn(
                 self.cmd, self.args, {
                     windowsVerbatimArguments: true,

data/locale/en-US.ini

@@ -48,20 +48,6 @@ KeyFrames.IntervalType.Frames="Frames"
 KeyFrames.IntervalType.Seconds="Seconds"
 KeyFrames.Interval="Interval"
 
-# Apple ProRes
-AppleProRes.Profile="Profile"
-AppleProRes.Profile.APCO="Proxy"
-AppleProRes.Profile.APCS="LT"
-AppleProRes.Profile.APCN="Standard Definition"
-AppleProRes.Profile.APCH="High Quality"
-AppleProRes.Profile.AP4H="4444"
-
-# ProRes
-ProRes.Profile.Proxy="Proxy (PXY)"
-ProRes.Profile.Light="Light (LT)"
-ProRes.Profile.Standard="Standard"
-ProRes.Profile.HighQuality="High Quality (HQ)"
-
 # Codec: H264
 Codec.H264="H264"
 Codec.H264.Profile="Profile"
@@ -82,6 +68,15 @@ Codec.HEVC.Tier.main="Main"
 Codec.HEVC.Tier.high="High"
 Codec.HEVC.Level="Level"
 
+# Codec: Apple ProRes
+Codec.ProRes.Profile="Profile"
+Codec.ProRes.Profile.APCO="422 Proxy/PXY (APCO)"
+Codec.ProRes.Profile.APCS="422 Light/LT (APCS)"
+Codec.ProRes.Profile.APCN="422 Standard (APCN)"
+Codec.ProRes.Profile.APCH="422 High Quality/HQ (APCH)"
+Codec.ProRes.Profile.AP4H="4444 Standard (AP4H)"
+Codec.ProRes.Profile.AP4X="4444 Extra Quality/XQ (AP4X)"
+
 # NVENC
 NVENC.Preset="Preset"
 NVENC.Preset.Default="Default"

source/codecs/prores.cpp

@@ -0,0 +1,22 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "prores.hpp"

source/codecs/prores.hpp

@@ -0,0 +1,32 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+// Codec: ProRes
+#define P_PRORES "Codec.ProRes"
+#define P_PRORES_PROFILE "Codec.ProRes.Profile"
+#define P_PRORES_PROFILE_APCS "Codec.ProRes.Profile.APCS"
+#define P_PRORES_PROFILE_APCO "Codec.ProRes.Profile.APCO"
+#define P_PRORES_PROFILE_APCN "Codec.ProRes.Profile.APCN"
+#define P_PRORES_PROFILE_APCH "Codec.ProRes.Profile.APCH"
+#define P_PRORES_PROFILE_AP4H "Codec.ProRes.Profile.AP4H"
+#define P_PRORES_PROFILE_AP4X "Codec.ProRes.Profile.AP4X"

source/encoder.hpp

@@ -24,6 +24,7 @@
 #include <condition_variable>
 #include <mutex>
 #include <thread>
+#include <vector>
 #include "ffmpeg/avframe-queue.hpp"
 #include "ffmpeg/swscale.hpp"
 
@@ -38,13 +39,21 @@ extern "C" {
 }
 
 namespace obsffmpeg {
+	class unsupported_gpu_exception : public std::runtime_error {
+		public:
+		unsupported_gpu_exception(const std::string& reason) : runtime_error(reason) {}
+	};
+
+	struct encoder_info {
+		std::string      uid;
+		std::string      codec;
+		std::string      readable_name;
+		obs_encoder_info oei;
+	};
+
 	class encoder_factory {
-		struct info {
+		encoder_info   info;
-			std::string      uid;
+		encoder_info   info_fallback;
-			std::string      codec;
-			std::string      readable_name;
-			obs_encoder_info oei;
-		} info;
 		const AVCodec* avcodec_ptr;
 
 		public:
@@ -53,13 +62,15 @@ namespace obsffmpeg {
 
 		void register_encoder();
 
-		const char* get_name();
-
 		void get_defaults(obs_data_t* settings);
 
 		void get_properties(obs_properties_t* props);
 
 		const AVCodec* get_avcodec();
+
+		const encoder_info& get_info();
+
+		const encoder_info& get_fallback();
 	};
 
 	class encoder {
@@ -83,7 +94,7 @@ namespace obsffmpeg {
 		std::vector<uint8_t> _sei_data;
 
 		public:
-		encoder(obs_data_t* settings, obs_encoder_t* encoder);
+		encoder(obs_data_t* settings, obs_encoder_t* encoder, bool is_texture_encode = false);
 		virtual ~encoder();
 
 		public: // OBS API

source/ffmpeg/tools.cpp

@@ -20,6 +20,8 @@
 // SOFTWARE.
 
 #include "tools.hpp"
+#include <list>
+#include <map>
 #include <sstream>
 #include <stdexcept>
 
@@ -159,59 +161,49 @@ const char* ffmpeg::tools::get_error_description(int error)
 	return "Not Translated Yet";
 }
 
+static std::map<video_format, AVPixelFormat> obs_to_av_format_map = {
+    {VIDEO_FORMAT_I420, AV_PIX_FMT_YUV420P},  // YUV 4:2:0
+    {VIDEO_FORMAT_NV12, AV_PIX_FMT_NV12},     // NV12 Packed YUV
+    {VIDEO_FORMAT_YVYU, AV_PIX_FMT_YVYU422},  // YVYU Packed YUV
+    {VIDEO_FORMAT_YUY2, AV_PIX_FMT_YUYV422},  // YUYV Packed YUV
+    {VIDEO_FORMAT_UYVY, AV_PIX_FMT_UYVY422},  // UYVY Packed YUV
+    {VIDEO_FORMAT_RGBA, AV_PIX_FMT_RGBA},     //
+    {VIDEO_FORMAT_BGRA, AV_PIX_FMT_BGRA},     //
+    {VIDEO_FORMAT_BGRX, AV_PIX_FMT_BGR0},     //
+    {VIDEO_FORMAT_Y800, AV_PIX_FMT_GRAY8},    //
+    {VIDEO_FORMAT_I444, AV_PIX_FMT_YUV444P},  //
+    {VIDEO_FORMAT_BGR3, AV_PIX_FMT_BGR24},    //
+    {VIDEO_FORMAT_I422, AV_PIX_FMT_YUV422P},  //
+    {VIDEO_FORMAT_I40A, AV_PIX_FMT_YUVA420P}, //
+    {VIDEO_FORMAT_I42A, AV_PIX_FMT_YUVA422P}, //
+    {VIDEO_FORMAT_YUVA, AV_PIX_FMT_YUVA444P}, //
+                                              //{VIDEO_FORMAT_AYUV, AV_PIX_FMT_AYUV444P}, //
+};
+
 AVPixelFormat ffmpeg::tools::obs_videoformat_to_avpixelformat(video_format v)
 {
-	switch (v) {
+	auto found = obs_to_av_format_map.find(v);
-		// 32-Bits
+	if (found != obs_to_av_format_map.end()) {
-	case VIDEO_FORMAT_BGRX:
+		return found->second;
-		return AV_PIX_FMT_BGR0;
-	case VIDEO_FORMAT_BGRA:
-		return AV_PIX_FMT_BGRA;
-	case VIDEO_FORMAT_RGBA:
-		return AV_PIX_FMT_RGBA;
-	case VIDEO_FORMAT_I444:
-		return AV_PIX_FMT_YUV444P;
-	case VIDEO_FORMAT_YUY2:
-		return AV_PIX_FMT_YUYV422;
-	case VIDEO_FORMAT_YVYU:
-		return AV_PIX_FMT_YVYU422;
-	case VIDEO_FORMAT_UYVY:
-		return AV_PIX_FMT_UYVY422;
-	case VIDEO_FORMAT_I420:
-		return AV_PIX_FMT_YUV420P;
-	case VIDEO_FORMAT_NV12:
-		return AV_PIX_FMT_NV12;
 	}
-	throw std::invalid_argument("unknown format");
+	return AV_PIX_FMT_NONE;
 }
 
 video_format ffmpeg::tools::avpixelformat_to_obs_videoformat(AVPixelFormat v)
 {
-	switch (v) {
+	for (const auto& kv : obs_to_av_format_map) {
-	case AV_PIX_FMT_YUV420P:
+		if (kv.second == v)
-		return VIDEO_FORMAT_I420;
+			return kv.first;
-	case AV_PIX_FMT_NV12:
-		return VIDEO_FORMAT_NV12;
-	case AV_PIX_FMT_YVYU422:
-		return VIDEO_FORMAT_YVYU;
-	case AV_PIX_FMT_YUYV422:
-		return VIDEO_FORMAT_YUY2;
-	case AV_PIX_FMT_UYVY422:
-		return VIDEO_FORMAT_UYVY;
-	case AV_PIX_FMT_RGBA:
-		return VIDEO_FORMAT_RGBA;
-	case AV_PIX_FMT_BGRA:
-		return VIDEO_FORMAT_BGRA;
-	case AV_PIX_FMT_BGR0:
-		return VIDEO_FORMAT_BGRX;
-	case AV_PIX_FMT_GRAY8:
-		return VIDEO_FORMAT_Y800;
-	case AV_PIX_FMT_YUV444P:
-		return VIDEO_FORMAT_I444;
 	}
 	return VIDEO_FORMAT_NONE;
 }
 
+AVPixelFormat ffmpeg::tools::get_least_lossy_format(const AVPixelFormat* haystack, AVPixelFormat needle)
+{
+	int data_loss = 0;
+	return avcodec_find_best_pix_fmt_of_list(haystack, needle, 0, &data_loss);
+}
+
 AVColorSpace ffmpeg::tools::obs_videocolorspace_to_avcolorspace(video_colorspace v)
 {
 	switch (v) {
@@ -235,3 +227,154 @@ AVColorRange ffmpeg::tools::obs_videorangetype_to_avcolorrange(video_range_type
 	}
 	throw std::invalid_argument("unknown range");
 }
+
+bool ffmpeg::tools::can_hardware_encode(const AVCodec* codec)
+{
+	AVPixelFormat hardware_formats[] = {AV_PIX_FMT_D3D11};
+
+	for (const AVPixelFormat* fmt = codec->pix_fmts; (fmt != nullptr) && (*fmt != AV_PIX_FMT_NONE); fmt++) {
+		for (auto cmp : hardware_formats) {
+			if (*fmt == cmp) {
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+std::vector<AVPixelFormat> ffmpeg::tools::get_software_formats(const AVPixelFormat* list)
+{
+	AVPixelFormat hardware_formats[] = {
+#if FF_API_VAAPI
+		AV_PIX_FMT_VAAPI_MOCO,
+		AV_PIX_FMT_VAAPI_IDCT,
+#endif
+		AV_PIX_FMT_VAAPI,
+		AV_PIX_FMT_DXVA2_VLD,
+		AV_PIX_FMT_VDPAU,
+		AV_PIX_FMT_QSV,
+		AV_PIX_FMT_MMAL,
+		AV_PIX_FMT_D3D11VA_VLD,
+		AV_PIX_FMT_CUDA,
+		AV_PIX_FMT_XVMC,
+		AV_PIX_FMT_VIDEOTOOLBOX,
+		AV_PIX_FMT_MEDIACODEC,
+		AV_PIX_FMT_D3D11,
+		AV_PIX_FMT_OPENCL,
+	};
+
+	std::vector<AVPixelFormat> fmts;
+	for (auto fmt = list; fmt && (*fmt != AV_PIX_FMT_NONE); fmt++) {
+		bool is_blacklisted = false;
+		for (auto blacklisted : hardware_formats) {
+			if (*fmt == blacklisted)
+				is_blacklisted = true;
+		}
+		if (!is_blacklisted)
+			fmts.push_back(*fmt);
+	}
+
+	fmts.push_back(AV_PIX_FMT_NONE);
+
+	return std::move(fmts);
+}
+
+static std::map<std::pair<AVPixelFormat, AVPixelFormat>, double_t> format_compatibility = {
+    {{AV_PIX_FMT_NV12, AV_PIX_FMT_NV12}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_NV12, AV_PIX_FMT_NV21}, 65535.0},
+
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVA420P}, 65535.0},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P9}, 58981.5},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10}, 53083.35},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P12}, 47775.015},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P14}, 42997.5135},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P16}, 38697.76215},
+
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P9}, 65535.0},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P10}, 58981.5},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P16}, 53083.35},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUV420P}, 32767.0},
+
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUVA422P}, 65535.0},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P9}, 58981.5},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P10}, 53083.35},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P12}, 47775.015},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P14}, 42997.5135},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P16}, 38697.76215},
+
+    {{AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA422P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P9}, 65535.0},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P10}, 58981.5},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P16}, 53083.35},
+    {{AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUV422P}, 32767.0},
+
+    {{AV_PIX_FMT_YVYU422, AV_PIX_FMT_YVYU422}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YVYU422, AV_PIX_FMT_YUYV422}, 65535.0},
+
+    {{AV_PIX_FMT_UYVY422, AV_PIX_FMT_UYVY422}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_UYVY422, AV_PIX_FMT_YVYU422}, 65535.0},
+
+    {{AV_PIX_FMT_YUYV422, AV_PIX_FMT_YUYV422}, std::numeric_limits<double_t>::max()},
+
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVA444P}, 65535.0},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P9}, 58981.5},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P10}, 53083.35},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P12}, 47775.015},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P14}, 42997.5135},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P16}, 38697.76215},
+
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P9}, 65535.0},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P10}, 58981.5},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P16}, 53083.35},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUV444P}, 32767.0},
+
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_RGBA}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_RGB0}, 65535.0},
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_0RGB}, 32767.0},
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_RGB24}, 16384.0},
+
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_BGRA}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_BGR0}, 65535.0},
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_0BGR}, 32767.0},
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_BGR24}, 16384.0},
+
+    {{AV_PIX_FMT_BGR0, AV_PIX_FMT_BGR0}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_BGR0, AV_PIX_FMT_BGRA}, 65535.0},
+    {{AV_PIX_FMT_BGR0, AV_PIX_FMT_BGR24}, 32767.0},
+
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY8}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9}, 65535.0},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY10}, 58981.5},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY12}, 53083.35},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY14}, 47775.015},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY16}, 42997.5135},
+
+    {{AV_PIX_FMT_BGR24, AV_PIX_FMT_BGR24}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_BGR24, AV_PIX_FMT_RGB24}, 32767.0},
+};
+
+AVPixelFormat ffmpeg::tools::get_best_compatible_format(const AVPixelFormat* list, AVPixelFormat source)
+{
+	double_t      score = std::numeric_limits<double_t>::min();
+	AVPixelFormat best  = source;
+
+	for (auto fmt = list; fmt && (*fmt != AV_PIX_FMT_NONE); fmt++) {
+		auto found = format_compatibility.find(std::pair{source, *fmt});
+		if (found != format_compatibility.end()) {
+			score = found->second;
+			best  = *fmt;
+		}
+	}
+
+	if (score <= 0) {
+		int data_loss = 0;
+		return avcodec_find_best_pix_fmt_of_list(list, source, 0, &data_loss);
+	}
+
+	return best;
+}

source/ffmpeg/tools.hpp

@@ -25,9 +25,11 @@
 
 #include <obs.h>
 #include <string>
+#include <vector>
 
 extern "C" {
 #include <libavutil/pixfmt.h>
+#include <libavcodec/avcodec.h>
 }
 
 namespace ffmpeg {
@@ -44,9 +46,17 @@ namespace ffmpeg {
 
 		video_format avpixelformat_to_obs_videoformat(AVPixelFormat v);
 
+		AVPixelFormat get_least_lossy_format(const AVPixelFormat* haystack, AVPixelFormat needle);
+
 		AVColorSpace obs_videocolorspace_to_avcolorspace(video_colorspace v);
 
 		AVColorRange obs_videorangetype_to_avcolorrange(video_range_type v);
+
+		bool can_hardware_encode(const AVCodec* codec);
+
+		std::vector<AVPixelFormat> get_software_formats(const AVPixelFormat* list);
+
+		AVPixelFormat get_best_compatible_format(const AVPixelFormat* list, AVPixelFormat source);
 	} // namespace tools
 } // namespace ffmpeg
 

source/ui/handler.cpp

@@ -22,3 +22,21 @@
 #include "handler.hpp"
 
 void obsffmpeg::ui::handler::override_visible_name(const AVCodec*, std::string&) {}
+
+void obsffmpeg::ui::handler::override_info(obs_encoder_info* main, obs_encoder_info* fallback) {}
+
+void obsffmpeg::ui::handler::override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+                                                  const AVCodec* codec, AVCodecContext* context)
+{}
+
+void obsffmpeg::ui::handler::log_options(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context) {}
+
+void obsffmpeg::ui::handler::import_from_ffmpeg(const std::string ffmpeg, obs_data_t* settings, const AVCodec* codec,
+                                                AVCodecContext* context)
+{}
+
+std::string obsffmpeg::ui::handler::export_for_ffmpeg(obs_data_t* settings, const AVCodec* codec,
+                                                      AVCodecContext* context)
+{
+	return std::string();
+}

source/ui/handler.hpp

@@ -24,7 +24,10 @@
 #include <string>
 
 extern "C" {
+#include <obs.h>
+
 #include <obs-data.h>
+#include <obs-encoder.h>
 #include <obs-properties.h>
 #pragma warning(push)
 #pragma warning(disable : 4244)
@@ -38,6 +41,11 @@ namespace obsffmpeg {
 			public:
 			virtual void override_visible_name(const AVCodec* codec, std::string& name);
 
+			virtual void override_info(obs_encoder_info* main, obs_encoder_info* fallback);
+
+			virtual void override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+			                                  const AVCodec* codec, AVCodecContext* context);
+
 			virtual void get_defaults(obs_data_t* settings, const AVCodec* codec,
 			                          AVCodecContext* context) = 0;
 
@@ -45,6 +53,14 @@ namespace obsffmpeg {
 			                            AVCodecContext* context) = 0;
 
 			virtual void update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context) = 0;
+
+			virtual void log_options(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context);
+
+			virtual void import_from_ffmpeg(const std::string ffmpeg, obs_data_t* settings,
+			                                const AVCodec* codec, AVCodecContext* context);
+
+			virtual std::string export_for_ffmpeg(obs_data_t* settings, const AVCodec* codec,
+			                                      AVCodecContext* context);
 		};
 	} // namespace ui
 } // namespace obsffmpeg

source/ui/prores_aw_handler.cpp

@@ -20,6 +20,7 @@
 // SOFTWARE.
 
 #include "prores_aw_handler.hpp"
+#include "codecs/prores.hpp"
 #include "plugin.hpp"
 #include "utility.hpp"
 
@@ -27,13 +28,6 @@ extern "C" {
 #include <obs-module.h>
 }
 
-#define P_PROFILE "AppleProRes.Profile"
-#define P_PROFILE_APCO "AppleProRes.Profile.APCO"
-#define P_PROFILE_APCS "AppleProRes.Profile.APCS"
-#define P_PROFILE_APCN "AppleProRes.Profile.APCN"
-#define P_PROFILE_APCH "AppleProRes.Profile.APCH"
-#define P_PROFILE_AP4H "AppleProRes.Profile.AP4H"
-
 INITIALIZER(prores_aw_handler_init)
 {
 	obsffmpeg::initializers.push_back([]() {
@@ -41,44 +35,72 @@ INITIALIZER(prores_aw_handler_init)
 	});
 };
 
+void obsffmpeg::ui::prores_aw_handler::override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+                                                            const AVCodec* codec, AVCodecContext* context)
+{
+	std::string profile = "";
+
+	int profile_id = static_cast<int>(obs_data_get_int(settings, P_PRORES_PROFILE));
+	for (auto ptr = codec->profiles; ptr->profile != FF_PROFILE_UNKNOWN; ptr++) {
+		if (ptr->profile == profile_id) {
+			profile = ptr->name;
+			break;
+		}
+	}
+
+	std::unordered_map<AVPixelFormat, std::list<std::string>> valid_formats = {
+	    {AV_PIX_FMT_YUV422P10, {"apco", "apcs", "apcn", "apch"}}, {AV_PIX_FMT_YUV444P10, {"ap4h", "ap4x"}}};
+
+	for (auto kv : valid_formats) {
+		for (auto name : kv.second) {
+			if (profile == name) {
+				target_format = kv.first;
+			}
+		}
+	}
+}
+
 void obsffmpeg::ui::prores_aw_handler::get_defaults(obs_data_t* settings, const AVCodec*, AVCodecContext*)
 {
-	obs_data_set_default_int(settings, P_PROFILE, 0);
+	obs_data_set_default_int(settings, P_PRORES_PROFILE, 0);
 }
 
 void obsffmpeg::ui::prores_aw_handler::get_properties(obs_properties_t* props, const AVCodec* codec,
                                                       AVCodecContext* context)
 {
 	if (!context) {
-		auto p = obs_properties_add_list(props, P_PROFILE, TRANSLATE(P_PROFILE), OBS_COMBO_TYPE_LIST,
+		auto p = obs_properties_add_list(props, P_PRORES_PROFILE, TRANSLATE(P_PRORES_PROFILE),
-		                                 OBS_COMBO_FORMAT_INT);
+		                                 OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);
-		obs_property_set_long_description(p, TRANSLATE(DESC(P_PROFILE)));
+		obs_property_set_long_description(p, TRANSLATE(DESC(P_PRORES_PROFILE)));
 		for (auto ptr = codec->profiles; ptr->profile != FF_PROFILE_UNKNOWN; ptr++) {
 			if (strcmp("apco", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCO),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCO),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("apcs", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCS),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCS),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("apcn", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCN),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCN),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("apch", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCH),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCH),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("ap4h", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_AP4H),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_AP4H),
+				                          static_cast<int64_t>(ptr->profile));
+			} else if (strcmp("ap4x", ptr->name) == 0) {
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_AP4X),
 				                          static_cast<int64_t>(ptr->profile));
 			} else {
 				obs_property_list_add_int(p, ptr->name, ptr->profile);
 			}
 		}
 	} else {
-		obs_property_set_enabled(obs_properties_get(props, P_PROFILE), false);
+		obs_property_set_enabled(obs_properties_get(props, P_PRORES_PROFILE), false);
 	}
 }
 
 void obsffmpeg::ui::prores_aw_handler::update(obs_data_t* settings, const AVCodec*, AVCodecContext* context)
 {
-	context->profile = static_cast<int>(obs_data_get_int(settings, P_PROFILE));
+	context->profile = static_cast<int>(obs_data_get_int(settings, P_PRORES_PROFILE));
 }

source/ui/prores_aw_handler.hpp

@@ -34,6 +34,9 @@ namespace obsffmpeg {
 	namespace ui {
 		class prores_aw_handler : public handler {
 			public:
+			virtual void override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+			                                  const AVCodec* codec, AVCodecContext* context) override;
+
 			virtual void get_defaults(obs_data_t* settings, const AVCodec* codec,
 			                          AVCodecContext* context) override;