ui/prores_aw_handler: Override automatic color format when profile demands it

Certain ProRes profiles require a very specific color format, otherwise the avcodec_open2 will fail with EPERM, which is very confusing.

CMakeLists.txt

@@ -272,6 +272,8 @@ set(PROJECT_PRIVATE
 	"${PROJECT_SOURCE_DIR}/source/codecs/hevc.cpp"
 	"${PROJECT_SOURCE_DIR}/source/codecs/h264.hpp"
 	"${PROJECT_SOURCE_DIR}/source/codecs/h264.cpp"
+	"${PROJECT_SOURCE_DIR}/source/codecs/prores.hpp"
+	"${PROJECT_SOURCE_DIR}/source/codecs/prores.cpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/avframe-queue.cpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/avframe-queue.hpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/swscale.hpp"

data/locale/en-US.ini

@@ -48,20 +48,6 @@ KeyFrames.IntervalType.Frames="Frames"
 KeyFrames.IntervalType.Seconds="Seconds"
 KeyFrames.Interval="Interval"
 
-# Apple ProRes
-AppleProRes.Profile="Profile"
-AppleProRes.Profile.APCO="Proxy"
-AppleProRes.Profile.APCS="LT"
-AppleProRes.Profile.APCN="Standard Definition"
-AppleProRes.Profile.APCH="High Quality"
-AppleProRes.Profile.AP4H="4444"
-
-# ProRes
-ProRes.Profile.Proxy="Proxy (PXY)"
-ProRes.Profile.Light="Light (LT)"
-ProRes.Profile.Standard="Standard"
-ProRes.Profile.HighQuality="High Quality (HQ)"
-
 # Codec: H264
 Codec.H264="H264"
 Codec.H264.Profile="Profile"
@@ -82,6 +68,15 @@ Codec.HEVC.Tier.main="Main"
 Codec.HEVC.Tier.high="High"
 Codec.HEVC.Level="Level"
 
+# Codec: Apple ProRes
+Codec.ProRes.Profile="Profile"
+Codec.ProRes.Profile.APCO="422 Proxy/PXY (APCO)"
+Codec.ProRes.Profile.APCS="422 Light/LT (APCS)"
+Codec.ProRes.Profile.APCN="422 Standard (APCN)"
+Codec.ProRes.Profile.APCH="422 High Quality/HQ (APCH)"
+Codec.ProRes.Profile.AP4H="4444 Standard (AP4H)"
+Codec.ProRes.Profile.AP4X="4444 Extra Quality/XQ (AP4X)"
+
 # NVENC
 NVENC.Preset="Preset"
 NVENC.Preset.Default="Default"

source/codecs/prores.cpp

@@ -0,0 +1,22 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "prores.hpp"

source/codecs/prores.hpp

@@ -0,0 +1,32 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+// Codec: ProRes
+#define P_PRORES "Codec.ProRes"
+#define P_PRORES_PROFILE "Codec.ProRes.Profile"
+#define P_PRORES_PROFILE_APCS "Codec.ProRes.Profile.APCS"
+#define P_PRORES_PROFILE_APCO "Codec.ProRes.Profile.APCO"
+#define P_PRORES_PROFILE_APCN "Codec.ProRes.Profile.APCN"
+#define P_PRORES_PROFILE_APCH "Codec.ProRes.Profile.APCH"
+#define P_PRORES_PROFILE_AP4H "Codec.ProRes.Profile.AP4H"
+#define P_PRORES_PROFILE_AP4X "Codec.ProRes.Profile.AP4X"

source/encoder.cpp

@@ -25,6 +25,7 @@
 #include <sstream>
 #include <thread>
 #include <util/profiler.hpp>
+#include <vector>
 #include "codecs/hevc.hpp"
 #include "ffmpeg/tools.hpp"
 #include "plugin.hpp"
@@ -44,6 +45,13 @@ extern "C" {
 #pragma warning(pop)
 }
 
+//#define DEBUG_CALL_ORDER
+// Call Order should be:
+// - create_texture/create
+// - get_video_info
+// - encode_texture/encode
+// I don't understand what get_video_info is actually for in this order, as this postpones initialization to encode...
+
 // FFmpeg
 #define ST_FFMPEG "FFmpeg"
 #define ST_FFMPEG_CUSTOMSETTINGS "FFmpeg.CustomSettings"
@@ -53,7 +61,226 @@ extern "C" {
 
 enum class keyframe_type { SECONDS, FRAMES };
 
-obsffmpeg::encoder_factory::encoder_factory(const AVCodec* codec) : avcodec_ptr(codec), info()
+static void* _create(obs_data_t* settings, obs_encoder_t* encoder) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX", __FUNCTION_NAME__, settings, encoder);
+#endif
+	return reinterpret_cast<void*>(new obsffmpeg::encoder(settings, encoder));
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return nullptr;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return nullptr;
+}
+
+static void* _create_texture(obs_data_t* settings, obs_encoder_t* encoder) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX", __FUNCTION_NAME__, settings, encoder);
+#endif
+	return reinterpret_cast<void*>(new obsffmpeg::encoder(settings, encoder, true));
+} catch (const obsffmpeg::unsupported_gpu_exception& ex) {
+	obsffmpeg::encoder_factory* fac =
+	    reinterpret_cast<obsffmpeg::encoder_factory*>(obs_encoder_get_type_data(encoder));
+	PLOG_WARNING("<%s> GPU not supported for hardware encoding, falling back to software.",
+	             fac->get_avcodec()->name);
+	return obs_encoder_create_rerouted(encoder, fac->get_fallback().oei.id);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return nullptr;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return nullptr;
+}
+
+static void _destroy(void* ptr) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX", __FUNCTION_NAME__, ptr);
+#endif
+	if (ptr)
+		delete reinterpret_cast<obsffmpeg::encoder*>(ptr);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+}
+
+static const char* _get_name(void* type_data) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX", __FUNCTION_NAME__, type_data);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder_factory*>(type_data)->get_info().readable_name.c_str();
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return nullptr;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return nullptr;
+}
+
+static const char* _get_name_fallback(void* type_data) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX", __FUNCTION_NAME__, type_data);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder_factory*>(type_data)->get_fallback().readable_name.c_str();
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return nullptr;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return nullptr;
+}
+
+static void _get_defaults(obs_data_t* settings, void* type_data) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX", __FUNCTION_NAME__, settings, type_data);
+#endif
+	reinterpret_cast<obsffmpeg::encoder_factory*>(type_data)->get_defaults(settings);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+};
+
+static obs_properties_t* _get_properties(void* ptr, void* type_data) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX", __FUNCTION_NAME__, ptr, type_data);
+#endif
+	obs_properties_t* props = obs_properties_create();
+	if (type_data != nullptr) {
+		reinterpret_cast<obsffmpeg::encoder_factory*>(type_data)->get_properties(props);
+	}
+	if (ptr != nullptr) {
+		reinterpret_cast<obsffmpeg::encoder*>(ptr)->get_properties(props);
+	}
+	return props;
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return reinterpret_cast<obs_properties_t*>(0);
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return reinterpret_cast<obs_properties_t*>(0);
+}
+
+static bool _update(void* ptr, obs_data_t* settings) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX", __FUNCTION_NAME__, ptr, settings);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder*>(ptr)->update(settings);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return false;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return false;
+}
+
+static bool _get_sei_data(void* ptr, uint8_t** sei_data, size_t* size) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX %llX", __FUNCTION_NAME__, ptr, sei_data, size);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder*>(ptr)->get_sei_data(sei_data, size);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return false;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return false;
+}
+
+static bool _get_extra_data(void* ptr, uint8_t** extra_data, size_t* size) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX %llX", __FUNCTION_NAME__, ptr, extra_data, size);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder*>(ptr)->get_extra_data(extra_data, size);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return false;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return false;
+}
+
+static void _get_video_info(void* ptr, struct video_scale_info* info) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX", __FUNCTION_NAME__, ptr, info);
+#endif
+	reinterpret_cast<obsffmpeg::encoder*>(ptr)->get_video_info(info);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+}
+
+static bool _encode(void* ptr, struct encoder_frame* frame, struct encoder_packet* packet,
+                    bool* received_packet) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX %llX %llX", __FUNCTION_NAME__, ptr, frame, packet, received_packet);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder*>(ptr)->video_encode(frame, packet, received_packet);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return false;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return false;
+}
+
+static bool _encode_texture(void* ptr, uint32_t handle, int64_t pts, uint64_t lock_key, uint64_t* next_key,
+                            struct encoder_packet* packet, bool* received_packet) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %lI %llI %llU %llX %llX %llX", __FUNCTION_NAME__, ptr, handle, pts, lock_key, next_key, packet,
+	          received_packet);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder*>(ptr)->video_encode_texture(handle, pts, lock_key, next_key, packet,
+	                                                                        received_packet);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return false;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return false;
+}
+
+static void _get_audio_info(void* ptr, struct audio_convert_info* info) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX", __FUNCTION_NAME__, ptr, info);
+#endif
+	reinterpret_cast<obsffmpeg::encoder*>(ptr)->get_audio_info(info);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+}
+
+static size_t _get_frame_size(void* ptr) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX", __FUNCTION_NAME__, ptr);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder*>(ptr)->get_frame_size();
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return 0;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return 0;
+}
+
+static bool _encode_audio(void* ptr, struct encoder_frame* frame, struct encoder_packet* packet,
+                          bool* received_packet) noexcept try {
+#ifdef DEBUG_CALL_ORDER
+	PLOG_INFO("%s %llX %llX %llX %llX", __FUNCTION_NAME__, ptr, frame, packet, received_packet);
+#endif
+	return reinterpret_cast<obsffmpeg::encoder*>(ptr)->audio_encode(frame, packet, received_packet);
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return false;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return false;
+}
+
+obsffmpeg::encoder_factory::encoder_factory(const AVCodec* codec) : avcodec_ptr(codec), info(), info_fallback()
 {
 	// Unique Id is FFmpeg name.
 	info.uid = avcodec_ptr->name;
@@ -96,6 +323,21 @@ obsffmpeg::encoder_factory::encoder_factory(const AVCodec* codec) : avcodec_ptr(
 		info.oei.caps |= OBS_ENCODER_CAP_DEPRECATED;
 	}
 #endif
+
+	// Hardware encoder?
+	if (ffmpeg::tools::can_hardware_encode(avcodec_ptr)) {
+		info_fallback.uid           = info.uid + "_sw";
+		info_fallback.codec         = info.codec;
+		info_fallback.readable_name = info.readable_name + " (Software)";
+
+		// Copy capabilities and hide from view.
+		info_fallback.oei.id    = info_fallback.uid.c_str();
+		info_fallback.oei.codec = info.oei.codec;
+		info_fallback.oei.caps  = info.oei.caps;
+		//info_fallback.oei.caps |= OBS_ENCODER_CAP_DEPRECATED;
+
+		info.oei.caps |= OBS_ENCODER_CAP_PASS_TEXTURE;
+	}
 }
 
 obsffmpeg::encoder_factory::~encoder_factory() {}
@@ -112,190 +354,55 @@ void obsffmpeg::encoder_factory::register_encoder()
 	}
 
 	// Register functions.
-	info.oei.create = [](obs_data_t* settings, obs_encoder_t* encoder) {
+	info.oei.destroy         = _destroy;
-		try {
+	info.oei.get_name        = _get_name;
-			return reinterpret_cast<void*>(new obsffmpeg::encoder(settings, encoder));
+	info.oei.get_defaults2   = _get_defaults;
-		} catch (std::exception const& e) {
+	info.oei.get_properties2 = _get_properties;
-			PLOG_ERROR("exception: %s", e.what());
+	info.oei.update          = _update;
-			return reinterpret_cast<void*>(0);
+	info.oei.get_sei_data    = _get_sei_data;
-		} catch (...) {
+	info.oei.get_extra_data  = _get_extra_data;
-			PLOG_ERROR("unknown exception");
-			return reinterpret_cast<void*>(0);
-		}
-	};
-	info.oei.destroy = [](void* ptr) {
-		try {
-			delete reinterpret_cast<encoder*>(ptr);
-		} catch (std::exception const& e) {
-			PLOG_ERROR("exception: %s", e.what());
-			throw e;
-		} catch (...) {
-			PLOG_ERROR("unknown exception");
-			throw;
-		}
-	};
-	info.oei.get_name = [](void* type_data) {
-		try {
-			return reinterpret_cast<encoder_factory*>(type_data)->get_name();
-		} catch (std::exception const& e) {
-			PLOG_ERROR("exception: %s", e.what());
-			throw e;
-		} catch (...) {
-			PLOG_ERROR("unknown exception");
-			throw;
-		}
-	};
-	info.oei.get_defaults2 = [](obs_data_t* settings, void* type_data) {
-		try {
-			reinterpret_cast<encoder_factory*>(type_data)->get_defaults(settings);
-		} catch (std::exception const& e) {
-			PLOG_ERROR("exception: %s", e.what());
-			throw e;
-		} catch (...) {
-			PLOG_ERROR("unknown exception");
-			throw;
-		}
-	};
-	info.oei.get_properties2 = [](void* ptr, void* type_data) {
-		try {
-			obs_properties_t* props = obs_properties_create();
-			if (type_data != nullptr) {
-				reinterpret_cast<encoder_factory*>(type_data)->get_properties(props);
-			}
-			if (ptr != nullptr) {
-				reinterpret_cast<encoder*>(ptr)->get_properties(props);
-			}
-			return props;
-		} catch (std::exception const& e) {
-			PLOG_ERROR("exception: %s", e.what());
-			throw e;
-		} catch (...) {
-			PLOG_ERROR("unknown exception");
-			throw;
-		}
-	};
-	info.oei.update = [](void* ptr, obs_data_t* settings) {
-		try {
-			return reinterpret_cast<encoder*>(ptr)->update(settings);
-		} catch (std::exception const& e) {
-			PLOG_ERROR("exception: %s", e.what());
-			throw e;
-		} catch (...) {
-			PLOG_ERROR("unknown exception");
-			throw;
-		}
-	};
-	info.oei.get_sei_data = [](void* ptr, uint8_t** sei_data, size_t* size) {
-		try {
-			return reinterpret_cast<encoder*>(ptr)->get_sei_data(sei_data, size);
-		} catch (std::exception const& e) {
-			PLOG_ERROR("exception: %s", e.what());
-			throw e;
-		} catch (...) {
-			PLOG_ERROR("unknown exception");
-			throw;
-		}
-	};
-	info.oei.get_extra_data = [](void* ptr, uint8_t** extra_data, size_t* size) {
-		try {
-			return reinterpret_cast<encoder*>(ptr)->get_extra_data(extra_data, size);
-		} catch (std::exception const& e) {
-			PLOG_ERROR("exception: %s", e.what());
-			throw e;
-		} catch (...) {
-			PLOG_ERROR("unknown exception");
-			throw;
-		}
-	};
 
 	if (avcodec_ptr->type == AVMediaType::AVMEDIA_TYPE_VIDEO) {
-		info.oei.get_video_info = [](void* ptr, struct video_scale_info* info) {
+		info.oei.get_video_info = _get_video_info;
-			try {
-				reinterpret_cast<encoder*>(ptr)->get_video_info(info);
-			} catch (std::exception const& e) {
-				PLOG_ERROR("exception: %s", e.what());
-				throw e;
-			} catch (...) {
-				PLOG_ERROR("unknown exception");
-				throw;
-			}
-		};
-		info.oei.encode = [](void* ptr, struct encoder_frame* frame, struct encoder_packet* packet,
-		                     bool* received_packet) {
-			try {
-				return reinterpret_cast<encoder*>(ptr)->video_encode(frame, packet, received_packet);
-			} catch (std::exception const& e) {
-				PLOG_ERROR("exception: %s", e.what());
-				throw e;
-			} catch (...) {
-				PLOG_ERROR("unknown exception");
-				throw;
-			}
-		};
-		info.oei.encode_texture = [](void* ptr, uint32_t handle, int64_t pts, uint64_t lock_key,
-		                             uint64_t* next_key, struct encoder_packet* packet, bool* received_packet) {
-			try {
-				return reinterpret_cast<encoder*>(ptr)->video_encode_texture(
-				    handle, pts, lock_key, next_key, packet, received_packet);
-			} catch (std::exception const& e) {
-				PLOG_ERROR("exception: %s", e.what());
-				throw e;
-			} catch (...) {
-				PLOG_ERROR("unknown exception");
-				throw;
-			}
-		};
-
 	} else if (avcodec_ptr->type == AVMediaType::AVMEDIA_TYPE_AUDIO) {
-		info.oei.get_audio_info = [](void* ptr, struct audio_convert_info* info) {
+		info.oei.get_audio_info = _get_audio_info;
-			try {
+		info.oei.get_frame_size = _get_frame_size;
-				reinterpret_cast<encoder*>(ptr)->get_audio_info(info);
+		info.oei.encode         = _encode_audio;
-			} catch (std::exception const& e) {
-				PLOG_ERROR("exception: %s", e.what());
-				throw e;
-			} catch (...) {
-				PLOG_ERROR("unknown exception");
-				throw;
-			}
-		};
-		info.oei.get_frame_size = [](void* ptr) {
-			try {
-				return reinterpret_cast<encoder*>(ptr)->get_frame_size();
-			} catch (std::exception const& e) {
-				PLOG_ERROR("exception: %s", e.what());
-				throw e;
-			} catch (...) {
-				PLOG_ERROR("unknown exception");
-				throw;
-			}
-		};
-		info.oei.encode = [](void* ptr, struct encoder_frame* frame, struct encoder_packet* packet,
-		                     bool* received_packet) {
-			try {
-				return reinterpret_cast<encoder*>(ptr)->audio_encode(frame, packet, received_packet);
-			} catch (std::exception const& e) {
-				PLOG_ERROR("exception: %s", e.what());
-				throw e;
-			} catch (...) {
-				PLOG_ERROR("unknown exception");
-				throw;
-			}
-		};
 	}
 
 	// Finally store ourself as type data.
 	info.oei.type_data = this;
 
+	if (ffmpeg::tools::can_hardware_encode(avcodec_ptr)) {
+		info.oei.create         = _create_texture;
+		info.oei.encode_texture = _encode_texture;
+
+		info_fallback.oei.type            = info.oei.type;
+		info_fallback.oei.create          = _create;
+		info_fallback.oei.destroy         = _destroy;
+		info_fallback.oei.get_name        = _get_name_fallback;
+		info_fallback.oei.get_defaults2   = _get_defaults;
+		info_fallback.oei.get_properties2 = _get_properties;
+		info_fallback.oei.update          = _update;
+		info_fallback.oei.get_sei_data    = _get_sei_data;
+		info_fallback.oei.get_extra_data  = _get_extra_data;
+		info_fallback.oei.get_video_info  = _get_video_info;
+		info_fallback.oei.encode          = _encode;
+		info_fallback.oei.type_data       = this;
+
+		obs_register_encoder(&info_fallback.oei);
+		PLOG_DEBUG("Registered software fallback for encoder #%llX", avcodec_ptr);
+	} else {
+		// Is not a GPU Encoder, don't implement fallback.
+		info.oei.create = _create;
+		info.oei.encode = _encode;
+	}
+
 	obs_register_encoder(&info.oei);
 	PLOG_DEBUG("Registered encoder #%llX with name '%s' and long name '%s' and caps %llX", avcodec_ptr,
 	           avcodec_ptr->name, avcodec_ptr->long_name, avcodec_ptr->capabilities);
 }
 
-const char* obsffmpeg::encoder_factory::get_name()
-{
-	return info.readable_name.c_str();
-}
-
 void obsffmpeg::encoder_factory::get_defaults(obs_data_t* settings)
 {
 	{ // Handler
@@ -320,12 +427,17 @@ void obsffmpeg::encoder_factory::get_defaults(obs_data_t* settings)
 	}
 }
 
-static bool modified_keyframes(obs_properties_t* props, obs_property_t*, obs_data_t* settings)
+static bool modified_keyframes(obs_properties_t* props, obs_property_t*, obs_data_t* settings) try {
-{
 	bool is_seconds = obs_data_get_int(settings, S_KEYFRAMES_INTERVALTYPE) == 0;
 	obs_property_set_visible(obs_properties_get(props, S_KEYFRAMES_INTERVAL_FRAMES), !is_seconds);
 	obs_property_set_visible(obs_properties_get(props, S_KEYFRAMES_INTERVAL_SECONDS), is_seconds);
 	return true;
+} catch (const std::exception& ex) {
+	PLOG_ERROR("Unexpected exception in function '%s': %s.", __FUNCTION_NAME__, ex.what());
+	return false;
+} catch (...) {
+	PLOG_ERROR("Unexpected exception in function '%s'.", __FUNCTION_NAME__);
+	return false;
 }
 
 void obsffmpeg::encoder_factory::get_properties(obs_properties_t* props)
@@ -424,9 +536,23 @@ const AVCodec* obsffmpeg::encoder_factory::get_avcodec()
 	return avcodec_ptr;
 }
 
-obsffmpeg::encoder::encoder(obs_data_t* settings, obs_encoder_t* encoder)
+const obsffmpeg::encoder_info& obsffmpeg::encoder_factory::get_info()
-    : _self(encoder), _lag_in_frames(0), _count_send_frames(0), _have_first_frame(false), _initialized(false)
 {
+	return info;
+}
+
+const obsffmpeg::encoder_info& obsffmpeg::encoder_factory::get_fallback()
+{
+	return info_fallback;
+}
+
+obsffmpeg::encoder::encoder(obs_data_t* settings, obs_encoder_t* encoder, bool is_texture_encode)
+    : _self(encoder), _lag_in_frames(0), _count_send_frames(0), _have_first_frame(false)
+{
+	if (is_texture_encode) {
+		throw obsffmpeg::unsupported_gpu_exception("not implemented yet");
+	}
+
 	_factory = reinterpret_cast<encoder_factory*>(obs_encoder_get_type_data(_self));
 
 	// Verify that the codec actually still exists.
@@ -466,11 +592,113 @@ obsffmpeg::encoder::encoder(obs_data_t* settings, obs_encoder_t* encoder)
 		}
 	}
 
+	// Create 8MB of precached Packet data for use later on.
+	av_init_packet(&_current_packet);
+	av_new_packet(&_current_packet, 8 * 1024 * 1024); // 8 MB precached Packet size.
+
+	if (_codec->type == AVMEDIA_TYPE_VIDEO) {
+		// Initialize Video Encoding
+		auto voi = video_output_get_info(obs_encoder_video(_self));
+
+		// Find a suitable Pixel Format.
+		AVPixelFormat _pixfmt_source = ffmpeg::tools::obs_videoformat_to_avpixelformat(voi->format);
+		AVPixelFormat _pixfmt_target =
+		    static_cast<AVPixelFormat>(obs_data_get_int(settings, ST_FFMPEG_COLORFORMAT));
+		if (_pixfmt_target == AV_PIX_FMT_NONE) {
+			// Find the best conversion format.
+			std::vector<AVPixelFormat> fmts = ffmpeg::tools::get_software_formats(_codec->pix_fmts);
+			_pixfmt_target = ffmpeg::tools::get_best_compatible_format(fmts.data(), _pixfmt_source);
+
+			{ // Allow Handler to override the automatic color format for sanity reasons.
+				auto ptr = obsffmpeg::find_codec_handler(_codec->name);
+				if (ptr) {
+					ptr->override_colorformat(_pixfmt_target, settings, _codec, _context);
+				}
+			}
+		} else {
+			// Use user override, guaranteed to be supported.
+			bool is_format_supported = false;
+			for (auto ptr = _codec->pix_fmts; *ptr != AV_PIX_FMT_NONE; ptr++) {
+				if (*ptr == _pixfmt_target) {
+					is_format_supported = true;
+				}
+			}
+
+			if (!is_format_supported) {
+				std::stringstream sstr;
+				sstr << "Color Format '" << ffmpeg::tools::get_pixel_format_name(_pixfmt_target)
+				     << "' is not supported by the encoder.";
+				throw std::exception(sstr.str().c_str());
+			}
+		}
+
+		_context->width                   = voi->width;
+		_context->height                  = voi->height;
+		_context->colorspace              = ffmpeg::tools::obs_videocolorspace_to_avcolorspace(voi->colorspace);
+		_context->color_range             = ffmpeg::tools::obs_videorangetype_to_avcolorrange(voi->range);
+		_context->pix_fmt                 = _pixfmt_target;
+		_context->field_order             = AV_FIELD_PROGRESSIVE;
+		_context->time_base.num           = voi->fps_den;
+		_context->time_base.den           = voi->fps_num;
+		_context->ticks_per_frame         = 1;
+		_context->sample_aspect_ratio.num = _context->sample_aspect_ratio.den = 1;
+
+		_swscale.set_source_size(_context->width, _context->height);
+		_swscale.set_source_color(_context->color_range, _context->colorspace);
+		_swscale.set_source_full_range(voi->range == VIDEO_RANGE_FULL);
+		_swscale.set_source_format(_pixfmt_source);
+
+		_swscale.set_target_size(_context->width, _context->height);
+		_swscale.set_target_color(_context->color_range, _context->colorspace);
+		_swscale.set_target_full_range(voi->range == VIDEO_RANGE_FULL);
+		_swscale.set_target_format(_pixfmt_target);
+
+		// Create Scaler
+		if (!_swscale.initialize(SWS_POINT)) {
+			std::stringstream sstr;
+			sstr << "Initializing scaler failed for conversion from '"
+			     << ffmpeg::tools::get_pixel_format_name(_swscale.get_source_format()) << "' to '"
+			     << ffmpeg::tools::get_pixel_format_name(_swscale.get_target_format())
+			     << "' with color space '"
+			     << ffmpeg::tools::get_color_space_name(_swscale.get_source_colorspace()) << "' and "
+			     << (_swscale.is_source_full_range() ? "full" : "partial") << " range.";
+			throw std::runtime_error(sstr.str());
+		}
+	}
+
+	{ // Log Encoder info
+		const char* id = obs_encoder_get_id(_self);
+		PLOG_INFO("[%s] Initializing...", id);
+		PLOG_INFO("[%s]   Video Input: %ldx%ld %s %s %s", id, _swscale.get_source_width(),
+		          _swscale.get_source_height(),
+		          ffmpeg::tools::get_pixel_format_name(_swscale.get_source_format()),
+		          ffmpeg::tools::get_color_space_name(_swscale.get_source_colorspace()),
+		          _swscale.is_source_full_range() ? "Full" : "Partial");
+		PLOG_INFO("[%s]   Video Output: %ldx%ld %s %s %s", id, _swscale.get_target_width(),
+		          _swscale.get_target_height(),
+		          ffmpeg::tools::get_pixel_format_name(_swscale.get_target_format()),
+		          ffmpeg::tools::get_color_space_name(_swscale.get_target_colorspace()),
+		          _swscale.is_target_full_range() ? "Full" : "Partial");
+		PLOG_INFO("[%s]   Framerate: %ld/%ld (%f", id, _context->time_base.num, _context->time_base.den,
+		          _context->time_base.num / _context->time_base.den);
+	}
+
 	// Update settings
 	update(settings);
 
-	av_init_packet(&_current_packet);
+	// Initialize Encoder
-	av_new_packet(&_current_packet, 8 * 1024 * 1024); // 8 MB precached Packet size.
+	int res = avcodec_open2(_context, _codec, NULL);
+	if (res < 0) {
+		std::stringstream sstr;
+		sstr << "Initializing encoder '" << _codec->name
+		     << "' failed with error: " << ffmpeg::tools::get_error_description(res) << " (code " << res << ")";
+		throw std::runtime_error(sstr.str());
+	}
+
+	// Create Frame queue
+	_frame_queue.set_pixel_format(_context->pix_fmt);
+	_frame_queue.set_resolution(_context->width, _context->height);
+	_frame_queue.precache(2);
 }
 
 obsffmpeg::encoder::~encoder()
@@ -497,85 +725,6 @@ obsffmpeg::encoder::~encoder()
 	_swscale.finalize();
 }
 
-bool obsffmpeg::encoder::initialize()
-{
-	if (_initialized)
-		return true;
-
-	// Detect supported
-	bool is_format_supported = false;
-	for (auto ptr = _codec->pix_fmts; *ptr != AV_PIX_FMT_NONE; ptr++) {
-		if (*ptr == _format) {
-			is_format_supported = true;
-		}
-	}
-	if (!is_format_supported) {
-		std::stringstream sstr;
-		sstr << "The color format " << ffmpeg::tools::get_pixel_format_name(_format)
-		     << " is not supported by the encoder.";
-		throw std::exception(sstr.str().c_str());
-	}
-
-	if (_codec->type == AVMEDIA_TYPE_VIDEO) {
-		auto encvideo = obs_encoder_video(_self);
-		auto voi      = video_output_get_info(encvideo);
-
-		// Set Resolution
-		_context->width  = static_cast<int>(_resolution.first);
-		_context->height = static_cast<int>(_resolution.second);
-
-		// Set Color Space, Format and Range
-		_context->colorspace  = ffmpeg::tools::obs_videocolorspace_to_avcolorspace(voi->colorspace);
-		_context->color_range = ffmpeg::tools::obs_videorangetype_to_avcolorrange(voi->range);
-		_context->pix_fmt     = _format;
-
-		// Set Framerate and Field Order
-		_context->field_order     = AV_FIELD_PROGRESSIVE;
-		_context->time_base.num   = voi->fps_den;
-		_context->time_base.den   = voi->fps_num;
-		_context->ticks_per_frame = 1;
-	}
-
-	// Initialize
-	int res = avcodec_open2(_context, _codec, NULL);
-	if (res < 0) {
-		std::stringstream sstr;
-		sstr << "Failed to initalized encoder '" << _codec->name
-		     << "' due to error: " << ffmpeg::tools::get_error_description(res) << "(" << res
-		     << ")";
-		    throw std::runtime_error(sstr.str().c_str());
-	}
-
-	// Initialize Scaler and Frame Queue
-	if (_codec->type == AVMEDIA_TYPE_VIDEO) {
-		_swscale.set_source_format(_format);
-		_swscale.set_target_format(_context->pix_fmt);
-		_swscale.set_target_size(_context->width, _context->height);
-		_swscale.set_source_size(_context->width, _context->height);
-		_swscale.set_target_size(_context->width, _context->height);
-		_swscale.set_source_color(_context->color_range, _context->colorspace);
-		_swscale.set_target_color(_context->color_range, _context->colorspace);
-
-		// Create Scaler
-		if (!_swscale.initialize(SWS_FAST_BILINEAR)) {
-			std::stringstream sstr;
-			sstr << "Failed to initialize frame scaler for pixel format '"
-			     << ffmpeg::tools::get_pixel_format_name(_context->pix_fmt) << "' with color space '"
-			     << ffmpeg::tools::get_color_space_name(_context->colorspace) << "' and "
-			     << (_swscale.is_source_full_range() ? "full" : "partial") << " color range.";
-			throw std::runtime_error(sstr.str().c_str());
-		}
-
-		// Create Frame queue
-		_frame_queue.set_pixel_format(_context->pix_fmt);
-		_frame_queue.set_resolution(_context->width, _context->height);
-		_frame_queue.precache(2);
-	}
-
-	_initialized = true;
-	return true;
-}
-
 void obsffmpeg::encoder::get_properties(obs_properties_t* props)
 {
 	{ // Handler
@@ -628,6 +777,14 @@ bool obsffmpeg::encoder::update(obs_data_t* settings)
 		av_opt_set_from_string(_context->priv_data, obs_data_get_string(settings, ST_FFMPEG_CUSTOMSETTINGS),
 		                       nullptr, "=", ";");
 	}
+
+	{ // Handler Logging
+		auto ptr = obsffmpeg::find_codec_handler(_codec->name);
+		if (ptr) {
+			ptr->log_options(settings, _codec, _context);
+		}
+	}
+
 	return false;
 }
 
@@ -645,18 +802,9 @@ bool obsffmpeg::encoder::audio_encode(encoder_frame*, encoder_packet*, bool*)
 
 void obsffmpeg::encoder::get_video_info(video_scale_info* vsi)
 {
-	obs_data_t* settings = obs_encoder_get_settings(_self);
+	vsi->width  = _swscale.get_source_width();
-
+	vsi->height = _swscale.get_source_height();
-	AVPixelFormat avformat  = static_cast<AVPixelFormat>(obs_data_get_int(settings, ST_FFMPEG_COLORFORMAT));
+	vsi->format = ffmpeg::tools::avpixelformat_to_obs_videoformat(_swscale.get_source_format());
-	video_format  obsformat = ffmpeg::tools::avpixelformat_to_obs_videoformat(avformat);
-
-	if (obsformat != VIDEO_FORMAT_NONE) {
-		vsi->format = obsformat;
-	}
-
-	_resolution.first  = vsi->width;
-	_resolution.second = vsi->height;
-	_format            = ffmpeg::tools::obs_videoformat_to_avpixelformat(vsi->format);
 }
 
 bool obsffmpeg::encoder::get_sei_data(uint8_t** data, size_t* size)
@@ -711,15 +859,6 @@ static inline void copy_data(encoder_frame* frame, AVFrame* vframe)
 
 bool obsffmpeg::encoder::video_encode(encoder_frame* frame, encoder_packet* packet, bool* received_packet)
 {
-	// Ensure that the encoder was initialized.
-	try {
-		if (!initialize())
-			return false;
-	} catch (std::exception& ex) {
-		PLOG_ERROR("%s", ex.what());
-		return false;
-	}
-
 	// Convert frame.
 	std::shared_ptr<AVFrame> vframe = _frame_queue.pop(); // Retrieve an empty frame.
 	{

source/encoder.hpp

@@ -24,6 +24,7 @@
 #include <condition_variable>
 #include <mutex>
 #include <thread>
+#include <vector>
 #include "ffmpeg/avframe-queue.hpp"
 #include "ffmpeg/swscale.hpp"
 
@@ -38,13 +39,21 @@ extern "C" {
 }
 
 namespace obsffmpeg {
-	class encoder_factory {
+	class unsupported_gpu_exception : public std::runtime_error {
-		struct info {
+		public:
+		unsupported_gpu_exception(const std::string& reason) : runtime_error(reason) {}
+	};
+
+	struct encoder_info {
 		std::string      uid;
 		std::string      codec;
 		std::string      readable_name;
 		obs_encoder_info oei;
-		} info;
+	};
+
+	class encoder_factory {
+		encoder_info   info;
+		encoder_info   info_fallback;
 		const AVCodec* avcodec_ptr;
 
 		public:
@@ -53,13 +62,15 @@ namespace obsffmpeg {
 
 		void register_encoder();
 
-		const char* get_name();
-
 		void get_defaults(obs_data_t* settings);
 
 		void get_properties(obs_properties_t* props);
 
 		const AVCodec* get_avcodec();
+
+		const encoder_info& get_info();
+
+		const encoder_info& get_fallback();
 	};
 
 	class encoder {
@@ -69,10 +80,6 @@ namespace obsffmpeg {
 		const AVCodec*  _codec;
 		AVCodecContext* _context;
 
-		bool                      _initialized;
-		std::pair<size_t, size_t> _resolution;
-		AVPixelFormat             _format;
-
 		ffmpeg::avframe_queue _frame_queue;
 		ffmpeg::avframe_queue _frame_queue_used;
 		ffmpeg::swscale       _swscale;
@@ -87,11 +94,9 @@ namespace obsffmpeg {
 		std::vector<uint8_t> _sei_data;
 
 		public:
-		encoder(obs_data_t* settings, obs_encoder_t* encoder);
+		encoder(obs_data_t* settings, obs_encoder_t* encoder, bool is_texture_encode = false);
 		virtual ~encoder();
 
-		bool initialize();
-
 		public: // OBS API
 		// Shared
 		void get_properties(obs_properties_t* props);

source/ffmpeg/tools.cpp

@@ -20,6 +20,8 @@
 // SOFTWARE.
 
 #include "tools.hpp"
+#include <list>
+#include <map>
 #include <sstream>
 #include <stdexcept>
 
@@ -159,59 +161,49 @@ const char* ffmpeg::tools::get_error_description(int error)
 	return "Not Translated Yet";
 }
 
+static std::map<video_format, AVPixelFormat> obs_to_av_format_map = {
+    {VIDEO_FORMAT_I420, AV_PIX_FMT_YUV420P},  // YUV 4:2:0
+    {VIDEO_FORMAT_NV12, AV_PIX_FMT_NV12},     // NV12 Packed YUV
+    {VIDEO_FORMAT_YVYU, AV_PIX_FMT_YVYU422},  // YVYU Packed YUV
+    {VIDEO_FORMAT_YUY2, AV_PIX_FMT_YUYV422},  // YUYV Packed YUV
+    {VIDEO_FORMAT_UYVY, AV_PIX_FMT_UYVY422},  // UYVY Packed YUV
+    {VIDEO_FORMAT_RGBA, AV_PIX_FMT_RGBA},     //
+    {VIDEO_FORMAT_BGRA, AV_PIX_FMT_BGRA},     //
+    {VIDEO_FORMAT_BGRX, AV_PIX_FMT_BGR0},     //
+    {VIDEO_FORMAT_Y800, AV_PIX_FMT_GRAY8},    //
+    {VIDEO_FORMAT_I444, AV_PIX_FMT_YUV444P},  //
+    {VIDEO_FORMAT_BGR3, AV_PIX_FMT_BGR24},    //
+    {VIDEO_FORMAT_I422, AV_PIX_FMT_YUV422P},  //
+    {VIDEO_FORMAT_I40A, AV_PIX_FMT_YUVA420P}, //
+    {VIDEO_FORMAT_I42A, AV_PIX_FMT_YUVA422P}, //
+    {VIDEO_FORMAT_YUVA, AV_PIX_FMT_YUVA444P}, //
+                                              //{VIDEO_FORMAT_AYUV, AV_PIX_FMT_AYUV444P}, //
+};
+
 AVPixelFormat ffmpeg::tools::obs_videoformat_to_avpixelformat(video_format v)
 {
-	switch (v) {
+	auto found = obs_to_av_format_map.find(v);
-		// 32-Bits
+	if (found != obs_to_av_format_map.end()) {
-	case VIDEO_FORMAT_BGRX:
+		return found->second;
-		return AV_PIX_FMT_BGR0;
-	case VIDEO_FORMAT_BGRA:
-		return AV_PIX_FMT_BGRA;
-	case VIDEO_FORMAT_RGBA:
-		return AV_PIX_FMT_RGBA;
-	case VIDEO_FORMAT_I444:
-		return AV_PIX_FMT_YUV444P;
-	case VIDEO_FORMAT_YUY2:
-		return AV_PIX_FMT_YUYV422;
-	case VIDEO_FORMAT_YVYU:
-		return AV_PIX_FMT_YVYU422;
-	case VIDEO_FORMAT_UYVY:
-		return AV_PIX_FMT_UYVY422;
-	case VIDEO_FORMAT_I420:
-		return AV_PIX_FMT_YUV420P;
-	case VIDEO_FORMAT_NV12:
-		return AV_PIX_FMT_NV12;
 	}
-	throw std::invalid_argument("unknown format");
+	return AV_PIX_FMT_NONE;
 }
 
 video_format ffmpeg::tools::avpixelformat_to_obs_videoformat(AVPixelFormat v)
 {
-	switch (v) {
+	for (const auto& kv : obs_to_av_format_map) {
-	case AV_PIX_FMT_YUV420P:
+		if (kv.second == v)
-		return VIDEO_FORMAT_I420;
+			return kv.first;
-	case AV_PIX_FMT_NV12:
-		return VIDEO_FORMAT_NV12;
-	case AV_PIX_FMT_YVYU422:
-		return VIDEO_FORMAT_YVYU;
-	case AV_PIX_FMT_YUYV422:
-		return VIDEO_FORMAT_YUY2;
-	case AV_PIX_FMT_UYVY422:
-		return VIDEO_FORMAT_UYVY;
-	case AV_PIX_FMT_RGBA:
-		return VIDEO_FORMAT_RGBA;
-	case AV_PIX_FMT_BGRA:
-		return VIDEO_FORMAT_BGRA;
-	case AV_PIX_FMT_BGR0:
-		return VIDEO_FORMAT_BGRX;
-	case AV_PIX_FMT_GRAY8:
-		return VIDEO_FORMAT_Y800;
-	case AV_PIX_FMT_YUV444P:
-		return VIDEO_FORMAT_I444;
 	}
 	return VIDEO_FORMAT_NONE;
 }
 
+AVPixelFormat ffmpeg::tools::get_least_lossy_format(const AVPixelFormat* haystack, AVPixelFormat needle)
+{
+	int data_loss = 0;
+	return avcodec_find_best_pix_fmt_of_list(haystack, needle, 0, &data_loss);
+}
+
 AVColorSpace ffmpeg::tools::obs_videocolorspace_to_avcolorspace(video_colorspace v)
 {
 	switch (v) {
@@ -235,3 +227,154 @@ AVColorRange ffmpeg::tools::obs_videorangetype_to_avcolorrange(video_range_type
 	}
 	throw std::invalid_argument("unknown range");
 }
+
+bool ffmpeg::tools::can_hardware_encode(const AVCodec* codec)
+{
+	AVPixelFormat hardware_formats[] = {AV_PIX_FMT_D3D11};
+
+	for (const AVPixelFormat* fmt = codec->pix_fmts; (fmt != nullptr) && (*fmt != AV_PIX_FMT_NONE); fmt++) {
+		for (auto cmp : hardware_formats) {
+			if (*fmt == cmp) {
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+std::vector<AVPixelFormat> ffmpeg::tools::get_software_formats(const AVPixelFormat* list)
+{
+	AVPixelFormat hardware_formats[] = {
+#if FF_API_VAAPI
+		AV_PIX_FMT_VAAPI_MOCO,
+		AV_PIX_FMT_VAAPI_IDCT,
+#endif
+		AV_PIX_FMT_VAAPI,
+		AV_PIX_FMT_DXVA2_VLD,
+		AV_PIX_FMT_VDPAU,
+		AV_PIX_FMT_QSV,
+		AV_PIX_FMT_MMAL,
+		AV_PIX_FMT_D3D11VA_VLD,
+		AV_PIX_FMT_CUDA,
+		AV_PIX_FMT_XVMC,
+		AV_PIX_FMT_VIDEOTOOLBOX,
+		AV_PIX_FMT_MEDIACODEC,
+		AV_PIX_FMT_D3D11,
+		AV_PIX_FMT_OPENCL,
+	};
+
+	std::vector<AVPixelFormat> fmts;
+	for (auto fmt = list; fmt && (*fmt != AV_PIX_FMT_NONE); fmt++) {
+		bool is_blacklisted = false;
+		for (auto blacklisted : hardware_formats) {
+			if (*fmt == blacklisted)
+				is_blacklisted = true;
+		}
+		if (!is_blacklisted)
+			fmts.push_back(*fmt);
+	}
+
+	fmts.push_back(AV_PIX_FMT_NONE);
+
+	return std::move(fmts);
+}
+
+static std::map<std::pair<AVPixelFormat, AVPixelFormat>, double_t> format_compatibility = {
+    {{AV_PIX_FMT_NV12, AV_PIX_FMT_NV12}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_NV12, AV_PIX_FMT_NV21}, 65535.0},
+
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVA420P}, 65535.0},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P9}, 58981.5},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10}, 53083.35},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P12}, 47775.015},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P14}, 42997.5135},
+    {{AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P16}, 38697.76215},
+
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P9}, 65535.0},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P10}, 58981.5},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA420P16}, 53083.35},
+    {{AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUV420P}, 32767.0},
+
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUVA422P}, 65535.0},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P9}, 58981.5},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P10}, 53083.35},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P12}, 47775.015},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P14}, 42997.5135},
+    {{AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV422P16}, 38697.76215},
+
+    {{AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA422P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P9}, 65535.0},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P10}, 58981.5},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P16}, 53083.35},
+    {{AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUV422P}, 32767.0},
+
+    {{AV_PIX_FMT_YVYU422, AV_PIX_FMT_YVYU422}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YVYU422, AV_PIX_FMT_YUYV422}, 65535.0},
+
+    {{AV_PIX_FMT_UYVY422, AV_PIX_FMT_UYVY422}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_UYVY422, AV_PIX_FMT_YVYU422}, 65535.0},
+
+    {{AV_PIX_FMT_YUYV422, AV_PIX_FMT_YUYV422}, std::numeric_limits<double_t>::max()},
+
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVA444P}, 65535.0},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P9}, 58981.5},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P10}, 53083.35},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P12}, 47775.015},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P14}, 42997.5135},
+    {{AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV444P16}, 38697.76215},
+
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P9}, 65535.0},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P10}, 58981.5},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA444P16}, 53083.35},
+    {{AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUV444P}, 32767.0},
+
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_RGBA}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_RGB0}, 65535.0},
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_0RGB}, 32767.0},
+    {{AV_PIX_FMT_RGBA, AV_PIX_FMT_RGB24}, 16384.0},
+
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_BGRA}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_BGR0}, 65535.0},
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_0BGR}, 32767.0},
+    {{AV_PIX_FMT_BGRA, AV_PIX_FMT_BGR24}, 16384.0},
+
+    {{AV_PIX_FMT_BGR0, AV_PIX_FMT_BGR0}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_BGR0, AV_PIX_FMT_BGRA}, 65535.0},
+    {{AV_PIX_FMT_BGR0, AV_PIX_FMT_BGR24}, 32767.0},
+
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY8}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9}, 65535.0},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY10}, 58981.5},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY12}, 53083.35},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY14}, 47775.015},
+    {{AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY16}, 42997.5135},
+
+    {{AV_PIX_FMT_BGR24, AV_PIX_FMT_BGR24}, std::numeric_limits<double_t>::max()},
+    {{AV_PIX_FMT_BGR24, AV_PIX_FMT_RGB24}, 32767.0},
+};
+
+AVPixelFormat ffmpeg::tools::get_best_compatible_format(const AVPixelFormat* list, AVPixelFormat source)
+{
+	double_t      score = std::numeric_limits<double_t>::min();
+	AVPixelFormat best  = source;
+
+	for (auto fmt = list; fmt && (*fmt != AV_PIX_FMT_NONE); fmt++) {
+		auto found = format_compatibility.find(std::pair{source, *fmt});
+		if (found != format_compatibility.end()) {
+			score = found->second;
+			best  = *fmt;
+		}
+	}
+
+	if (score <= 0) {
+		int data_loss = 0;
+		return avcodec_find_best_pix_fmt_of_list(list, source, 0, &data_loss);
+	}
+
+	return best;
+}

source/ffmpeg/tools.hpp

@@ -25,9 +25,11 @@
 
 #include <obs.h>
 #include <string>
+#include <vector>
 
 extern "C" {
 #include <libavutil/pixfmt.h>
+#include <libavcodec/avcodec.h>
 }
 
 namespace ffmpeg {
@@ -44,9 +46,17 @@ namespace ffmpeg {
 
 		video_format avpixelformat_to_obs_videoformat(AVPixelFormat v);
 
+		AVPixelFormat get_least_lossy_format(const AVPixelFormat* haystack, AVPixelFormat needle);
+
 		AVColorSpace obs_videocolorspace_to_avcolorspace(video_colorspace v);
 
 		AVColorRange obs_videorangetype_to_avcolorrange(video_range_type v);
+
+		bool can_hardware_encode(const AVCodec* codec);
+
+		std::vector<AVPixelFormat> get_software_formats(const AVPixelFormat* list);
+
+		AVPixelFormat get_best_compatible_format(const AVPixelFormat* list, AVPixelFormat source);
 	} // namespace tools
 } // namespace ffmpeg
 

source/ui/handler.cpp

@@ -22,3 +22,21 @@
 #include "handler.hpp"
 
 void obsffmpeg::ui::handler::override_visible_name(const AVCodec*, std::string&) {}
+
+void obsffmpeg::ui::handler::override_info(obs_encoder_info* main, obs_encoder_info* fallback) {}
+
+void obsffmpeg::ui::handler::override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+                                                  const AVCodec* codec, AVCodecContext* context)
+{}
+
+void obsffmpeg::ui::handler::log_options(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context) {}
+
+void obsffmpeg::ui::handler::import_from_ffmpeg(const std::string ffmpeg, obs_data_t* settings, const AVCodec* codec,
+                                                AVCodecContext* context)
+{}
+
+std::string obsffmpeg::ui::handler::export_for_ffmpeg(obs_data_t* settings, const AVCodec* codec,
+                                                      AVCodecContext* context)
+{
+	return std::string();
+}

source/ui/handler.hpp

@@ -24,7 +24,10 @@
 #include <string>
 
 extern "C" {
+#include <obs.h>
+
 #include <obs-data.h>
+#include <obs-encoder.h>
 #include <obs-properties.h>
 #pragma warning(push)
 #pragma warning(disable : 4244)
@@ -38,6 +41,11 @@ namespace obsffmpeg {
 			public:
 			virtual void override_visible_name(const AVCodec* codec, std::string& name);
 
+			virtual void override_info(obs_encoder_info* main, obs_encoder_info* fallback);
+
+			virtual void override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+			                                  const AVCodec* codec, AVCodecContext* context);
+
 			virtual void get_defaults(obs_data_t* settings, const AVCodec* codec,
 			                          AVCodecContext* context) = 0;
 
@@ -45,6 +53,14 @@ namespace obsffmpeg {
 			                            AVCodecContext* context) = 0;
 
 			virtual void update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context) = 0;
+
+			virtual void log_options(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context);
+
+			virtual void import_from_ffmpeg(const std::string ffmpeg, obs_data_t* settings,
+			                                const AVCodec* codec, AVCodecContext* context);
+
+			virtual std::string export_for_ffmpeg(obs_data_t* settings, const AVCodec* codec,
+			                                      AVCodecContext* context);
 		};
 	} // namespace ui
 } // namespace obsffmpeg

source/ui/prores_aw_handler.cpp

@@ -20,6 +20,7 @@
 // SOFTWARE.
 
 #include "prores_aw_handler.hpp"
+#include "codecs/prores.hpp"
 #include "plugin.hpp"
 #include "utility.hpp"
 
@@ -27,13 +28,6 @@ extern "C" {
 #include <obs-module.h>
 }
 
-#define P_PROFILE "AppleProRes.Profile"
-#define P_PROFILE_APCO "AppleProRes.Profile.APCO"
-#define P_PROFILE_APCS "AppleProRes.Profile.APCS"
-#define P_PROFILE_APCN "AppleProRes.Profile.APCN"
-#define P_PROFILE_APCH "AppleProRes.Profile.APCH"
-#define P_PROFILE_AP4H "AppleProRes.Profile.AP4H"
-
 INITIALIZER(prores_aw_handler_init)
 {
 	obsffmpeg::initializers.push_back([]() {
@@ -41,44 +35,72 @@ INITIALIZER(prores_aw_handler_init)
 	});
 };
 
+void obsffmpeg::ui::prores_aw_handler::override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+                                                            const AVCodec* codec, AVCodecContext* context)
+{
+	std::string profile = "";
+
+	int profile_id = static_cast<int>(obs_data_get_int(settings, P_PRORES_PROFILE));
+	for (auto ptr = codec->profiles; ptr->profile != FF_PROFILE_UNKNOWN; ptr++) {
+		if (ptr->profile == profile_id) {
+			profile = ptr->name;
+			break;
+		}
+	}
+
+	std::unordered_map<AVPixelFormat, std::list<std::string>> valid_formats = {
+	    {AV_PIX_FMT_YUV422P10, {"apco", "apcs", "apcn", "apch"}}, {AV_PIX_FMT_YUV444P10, {"ap4h", "ap4x"}}};
+
+	for (auto kv : valid_formats) {
+		for (auto name : kv.second) {
+			if (profile == name) {
+				target_format = kv.first;
+			}
+		}
+	}
+}
+
 void obsffmpeg::ui::prores_aw_handler::get_defaults(obs_data_t* settings, const AVCodec*, AVCodecContext*)
 {
-	obs_data_set_default_int(settings, P_PROFILE, 0);
+	obs_data_set_default_int(settings, P_PRORES_PROFILE, 0);
 }
 
 void obsffmpeg::ui::prores_aw_handler::get_properties(obs_properties_t* props, const AVCodec* codec,
                                                       AVCodecContext* context)
 {
 	if (!context) {
-		auto p = obs_properties_add_list(props, P_PROFILE, TRANSLATE(P_PROFILE), OBS_COMBO_TYPE_LIST,
+		auto p = obs_properties_add_list(props, P_PRORES_PROFILE, TRANSLATE(P_PRORES_PROFILE),
-		                                 OBS_COMBO_FORMAT_INT);
+		                                 OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_INT);
-		obs_property_set_long_description(p, TRANSLATE(DESC(P_PROFILE)));
+		obs_property_set_long_description(p, TRANSLATE(DESC(P_PRORES_PROFILE)));
 		for (auto ptr = codec->profiles; ptr->profile != FF_PROFILE_UNKNOWN; ptr++) {
 			if (strcmp("apco", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCO),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCO),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("apcs", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCS),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCS),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("apcn", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCN),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCN),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("apch", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_APCH),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_APCH),
 				                          static_cast<int64_t>(ptr->profile));
 			} else if (strcmp("ap4h", ptr->name) == 0) {
-				obs_property_list_add_int(p, TRANSLATE(P_PROFILE_AP4H),
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_AP4H),
+				                          static_cast<int64_t>(ptr->profile));
+			} else if (strcmp("ap4x", ptr->name) == 0) {
+				obs_property_list_add_int(p, TRANSLATE(P_PRORES_PROFILE_AP4X),
 				                          static_cast<int64_t>(ptr->profile));
 			} else {
 				obs_property_list_add_int(p, ptr->name, ptr->profile);
 			}
 		}
 	} else {
-		obs_property_set_enabled(obs_properties_get(props, P_PROFILE), false);
+		obs_property_set_enabled(obs_properties_get(props, P_PRORES_PROFILE), false);
 	}
 }
 
 void obsffmpeg::ui::prores_aw_handler::update(obs_data_t* settings, const AVCodec*, AVCodecContext* context)
 {
-	context->profile = static_cast<int>(obs_data_get_int(settings, P_PROFILE));
+	context->profile = static_cast<int>(obs_data_get_int(settings, P_PRORES_PROFILE));
 }

source/ui/prores_aw_handler.hpp

@@ -34,6 +34,9 @@ namespace obsffmpeg {
 	namespace ui {
 		class prores_aw_handler : public handler {
 			public:
+			virtual void override_colorformat(AVPixelFormat& target_format, obs_data_t* settings,
+			                                  const AVCodec* codec, AVCodecContext* context) override;
+
 			virtual void get_defaults(obs_data_t* settings, const AVCodec* codec,
 			                          AVCodecContext* context) override;