encoder: Add support for true hardware encoding

This is the last step towards truly efficient encoding on AMD, Nvidia and Intel GPUs. With this we have no software overhead and can directly encode the content that OBS gives us, without going through any intermediate CPU layer. This is effectively what @jp9000 did for the OBS-integrated nvenc, but thanks to FFmpeg it works on all encoders that support D3D11VA acceleration.

With the change, the encoding should now work flawlessly even in very constrained situations (unless OBS itself is being starved of resources). Especially people streaming and recording Ubisoft games will likely see a drastic increase in encoding capability, and thanks to the new options will also be able to get a much higher quality stream and recording with the same hardware.

CMakeLists.txt

@@ -280,6 +280,8 @@ set(PROJECT_PRIVATE
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/swscale.cpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/tools.hpp"
 	"${PROJECT_SOURCE_DIR}/source/ffmpeg/tools.cpp"
+	"${PROJECT_SOURCE_DIR}/source/hwapi/base.hpp"
+	"${PROJECT_SOURCE_DIR}/source/hwapi/base.cpp"
 	"${PROJECT_SOURCE_DIR}/source/ui/handler.hpp"
 	"${PROJECT_SOURCE_DIR}/source/ui/handler.cpp"
 	"${PROJECT_SOURCE_DIR}/source/ui/debug_handler.hpp"
@@ -293,6 +295,12 @@ set(PROJECT_PRIVATE
 	"${PROJECT_SOURCE_DIR}/source/ui/nvenc_hevc_handler.hpp"
 	"${PROJECT_SOURCE_DIR}/source/ui/nvenc_hevc_handler.cpp"
 )
+if(WIN32)
+	list(APPEND PROJECT_PRIVATE 
+		"${PROJECT_SOURCE_DIR}/source/hwapi/d3d11.hpp"
+		"${PROJECT_SOURCE_DIR}/source/hwapi/d3d11.cpp"
+	)
+endif()
 
 # Source Grouping
 source_group(TREE "${PROJECT_SOURCE_DIR}" PREFIX "Data Files" FILES ${PROJECT_DATA})
@@ -357,7 +365,7 @@ endif()
 
 # Link Libraries
 target_link_libraries(${PROJECT_NAME}
-	"${PROJECT_LIBRARIES}"
+	${PROJECT_LIBRARIES}
 	${FFMPEG_LIBRARIES}
 )
 

source/encoder.cpp

@@ -52,6 +52,11 @@ extern "C" {
 // - encode_texture/encode
 // I don't understand what get_video_info is actually for in this order, as this postpones initialization to encode...
 
+#ifdef WIN32
+#define HARDWARE_ENCODING
+#include "hwapi/d3d11.hpp"
+#endif
+
 // FFmpeg
 #define ST_FFMPEG "FFmpeg"
 #define ST_FFMPEG_CUSTOMSETTINGS "FFmpeg.CustomSettings"
@@ -328,6 +333,7 @@ obsffmpeg::encoder_factory::encoder_factory(const AVCodec* codec) : avcodec_ptr(
 #endif
 
 	// Hardware encoder?
+#ifdef HARDWARE_ENCODING
 	if (ffmpeg::tools::can_hardware_encode(avcodec_ptr)) {
 		info_fallback.uid           = info.uid + "_sw";
 		info_fallback.codec         = info.codec;
@@ -341,6 +347,7 @@ obsffmpeg::encoder_factory::encoder_factory(const AVCodec* codec) : avcodec_ptr(
 
 		info.oei.caps |= OBS_ENCODER_CAP_PASS_TEXTURE;
 	}
+#endif
 }
 
 obsffmpeg::encoder_factory::~encoder_factory() {}
@@ -541,59 +548,8 @@ const obsffmpeg::encoder_info& obsffmpeg::encoder_factory::get_fallback()
 	return info_fallback;
 }
 
-obsffmpeg::encoder::encoder(obs_data_t* settings, obs_encoder_t* encoder, bool is_texture_encode)
+void obsffmpeg::encoder::initialize_sw(obs_data_t* settings)
-    : _self(encoder), _lag_in_frames(0), _count_send_frames(0), _have_first_frame(false)
 {
-	if (is_texture_encode) {
-		throw obsffmpeg::unsupported_gpu_exception("not implemented yet");
-	}
-
-	_factory = reinterpret_cast<encoder_factory*>(obs_encoder_get_type_data(_self));
-
-	// Verify that the codec actually still exists.
-	_codec = avcodec_find_encoder_by_name(_factory->get_avcodec()->name);
-	if (!_codec) {
-		PLOG_ERROR("Failed to find encoder for codec '%s'.", _factory->get_avcodec()->name);
-		throw std::runtime_error("failed to find codec");
-	}
-
-	// Find Codec UI handler.
-	_handler = obsffmpeg::find_codec_handler(_codec->name);
-
-	// Initialize context.
-	_context = avcodec_alloc_context3(_codec);
-	if (!_context) {
-		PLOG_ERROR("Failed to create context for encoder '%s'.", _codec->name);
-		throw std::runtime_error("failed to create context");
-	}
-
-	// Settings
-	/// Rate Control
-	_context->strict_std_compliance = static_cast<int>(obs_data_get_int(settings, ST_FFMPEG_STANDARDCOMPLIANCE));
-	_context->debug                 = 0;
-	/// Threading
-	if (_codec->capabilities
-	    & (AV_CODEC_CAP_AUTO_THREADS | AV_CODEC_CAP_FRAME_THREADS | AV_CODEC_CAP_SLICE_THREADS)) {
-		if (_codec->capabilities & AV_CODEC_CAP_FRAME_THREADS) {
-			_context->thread_type |= FF_THREAD_FRAME;
-		}
-		if (_codec->capabilities & AV_CODEC_CAP_SLICE_THREADS) {
-			_context->thread_type |= FF_THREAD_SLICE;
-		}
-		int64_t threads = obs_data_get_int(settings, ST_FFMPEG_THREADS);
-		if (threads > 0) {
-			_context->thread_count = static_cast<int>(threads);
-			_lag_in_frames         = _context->thread_count;
-		} else {
-			_context->thread_count = std::thread::hardware_concurrency();
-			_lag_in_frames         = _context->thread_count;
-		}
-	}
-
-	// Create 8MB of precached Packet data for use later on.
-	av_init_packet(&_current_packet);
-	av_new_packet(&_current_packet, 8 * 1024 * 1024); // 8 MB precached Packet size.
-
 	if (_codec->type == AVMEDIA_TYPE_VIDEO) {
 		// Initialize Video Encoding
 		auto voi = video_output_get_info(obs_encoder_video(_self));
@@ -656,27 +612,105 @@ obsffmpeg::encoder::encoder(obs_data_t* settings, obs_encoder_t* encoder, bool i
 			     << (_swscale.is_source_full_range() ? "full" : "partial") << " range.";
 			throw std::runtime_error(sstr.str());
 		}
+
+		// Create Frame queue
+		_frame_queue.set_pixel_format(_context->pix_fmt);
+		_frame_queue.set_resolution(_context->width, _context->height);
+		_frame_queue.precache(2);
+	}
+}
+
+void obsffmpeg::encoder::initialize_hw(obs_data_t* settings)
+{
+	// Initialize Video Encoding
+	auto voi = video_output_get_info(obs_encoder_video(_self));
+
+	_context->width                   = voi->width;
+	_context->height                  = voi->height;
+	_context->field_order             = AV_FIELD_PROGRESSIVE;
+	_context->ticks_per_frame         = 1;
+	_context->sample_aspect_ratio.num = _context->sample_aspect_ratio.den = 1;
+	_context->framerate.num = _context->time_base.den = voi->fps_num;
+	_context->framerate.den = _context->time_base.num = voi->fps_den;
+	ffmpeg::tools::setup_obs_color(voi->colorspace, voi->range, _context);
+	_context->sw_pix_fmt = ffmpeg::tools::obs_videoformat_to_avpixelformat(voi->format);
+
+#ifdef WIN32
+	_context->pix_fmt = AV_PIX_FMT_D3D11;
+#endif
+
+	_context->hw_device_ctx = _hwinst->create_device_context();
+
+	_context->hw_frames_ctx = av_hwframe_ctx_alloc(_context->hw_device_ctx);
+	if (!_context->hw_frames_ctx)
+		throw std::runtime_error("Failed to allocate AVHWFramesContext.");
+
+	AVHWFramesContext* ctx = reinterpret_cast<AVHWFramesContext*>(_context->hw_frames_ctx->data);
+	ctx->width             = _context->width;
+	ctx->height            = _context->height;
+	ctx->format            = _context->pix_fmt;
+	ctx->sw_format         = _context->sw_pix_fmt;
+
+	if (av_hwframe_ctx_init(_context->hw_frames_ctx) < 0)
+		throw std::runtime_error("Failed to initialize AVHWFramesContext.");
+}
+
+obsffmpeg::encoder::encoder(obs_data_t* settings, obs_encoder_t* encoder, bool is_texture_encode)
+    : _self(encoder), _lag_in_frames(0), _count_send_frames(0), _have_first_frame(false)
+{
+	// Initial set up.
+	_factory = reinterpret_cast<encoder_factory*>(obs_encoder_get_type_data(_self));
+	_codec   = _factory->get_avcodec();
+	_handler = obsffmpeg::find_codec_handler(_codec->name);
+
+	if (is_texture_encode) {
+#ifdef WIN32
+		_hwapi = std::make_shared<obsffmpeg::hwapi::d3d11>();
+#endif
+		obsffmpeg::hwapi::device dev;
+		if (_handler)
+			dev = _handler->find_hw_device(_hwapi, _codec, _context);
+		try {
+			_hwinst = _hwapi->create(dev);
+		} catch (...) {
+			throw obsffmpeg::unsupported_gpu_exception("Creating GPU context failed.");
+		}
 	}
 
-	{ // Log Encoder info
+	// Initialize context.
+	_context = avcodec_alloc_context3(_codec);
+	if (!_context) {
+		PLOG_ERROR("Failed to create context for encoder '%s'.", _codec->name);
+		throw std::runtime_error("failed to create context");
+	}
+
+	// Create 8MB of precached Packet data for use later on.
+	av_init_packet(&_current_packet);
+	av_new_packet(&_current_packet, 8 * 1024 * 1024); // 8 MB precached Packet size.
+
+	if (!is_texture_encode) {
+		initialize_sw(settings);
+	} else {
+		try {
+			initialize_hw(settings);
+		} catch (...) {
+			throw obsffmpeg::unsupported_gpu_exception("Initializing hardware context failed.");
+		}
+	}
+
+	// Log Encoder info
 	PLOG_INFO("[%s] Initializing...", _codec->name);
 	PLOG_INFO("[%s]   Video Input: %ldx%ld %s %s %s", _codec->name, _swscale.get_source_width(),
-		          _swscale.get_source_height(),
+	          _swscale.get_source_height(), ffmpeg::tools::get_pixel_format_name(_swscale.get_source_format()),
-		          ffmpeg::tools::get_pixel_format_name(_swscale.get_source_format()),
 	          ffmpeg::tools::get_color_space_name(_swscale.get_source_colorspace()),
 	          _swscale.is_source_full_range() ? "Full" : "Partial");
 	PLOG_INFO("[%s]   Video Output: %ldx%ld %s %s %s", _codec->name, _swscale.get_target_width(),
-		          _swscale.get_target_height(),
+	          _swscale.get_target_height(), ffmpeg::tools::get_pixel_format_name(_swscale.get_target_format()),
-		          ffmpeg::tools::get_pixel_format_name(_swscale.get_target_format()),
 	          ffmpeg::tools::get_color_space_name(_swscale.get_target_colorspace()),
 	          _swscale.is_target_full_range() ? "Full" : "Partial");
-		PLOG_INFO("[%s]   Framerate: %ld/%ld (%f FPS)", _codec->name, _context->time_base.den,
+	PLOG_INFO("[%s]   Framerate: %ld/%ld (%f FPS)", _codec->name, _context->time_base.den, _context->time_base.num,
-		          _context->time_base.num,
+	          static_cast<double_t>(_context->time_base.den) / static_cast<double_t>(_context->time_base.num));
-		          static_cast<double_t>(_context->time_base.den)
+	PLOG_INFO("[%s]   Custom Settings: %s", _codec->name, obs_data_get_string(settings, ST_FFMPEG_CUSTOMSETTINGS));
-		              / static_cast<double_t>(_context->time_base.num));
-		PLOG_INFO("[%s]   Custom Settings: %s", _codec->name,
-		          obs_data_get_string(settings, ST_FFMPEG_CUSTOMSETTINGS));
-	}
 
 	// Update settings
 	update(settings);
@@ -689,11 +723,6 @@ obsffmpeg::encoder::encoder(obs_data_t* settings, obs_encoder_t* encoder, bool i
 		     << "' failed with error: " << ffmpeg::tools::get_error_description(res) << " (code " << res << ")";
 		throw std::runtime_error(sstr.str());
 	}
-
-	// Create Frame queue
-	_frame_queue.set_pixel_format(_context->pix_fmt);
-	_frame_queue.set_resolution(_context->width, _context->height);
-	_frame_queue.precache(2);
 }
 
 obsffmpeg::encoder::~encoder()
@@ -737,6 +766,29 @@ void obsffmpeg::encoder::get_properties(obs_properties_t* props)
 
 bool obsffmpeg::encoder::update(obs_data_t* settings)
 {
+	// Settings
+	/// Rate Control
+	_context->strict_std_compliance = static_cast<int>(obs_data_get_int(settings, ST_FFMPEG_STANDARDCOMPLIANCE));
+	_context->debug                 = 0;
+	/// Threading
+	if (_codec->capabilities
+	    & (AV_CODEC_CAP_AUTO_THREADS | AV_CODEC_CAP_FRAME_THREADS | AV_CODEC_CAP_SLICE_THREADS)) {
+		if (_codec->capabilities & AV_CODEC_CAP_FRAME_THREADS) {
+			_context->thread_type |= FF_THREAD_FRAME;
+		}
+		if (_codec->capabilities & AV_CODEC_CAP_SLICE_THREADS) {
+			_context->thread_type |= FF_THREAD_SLICE;
+		}
+		int64_t threads = obs_data_get_int(settings, ST_FFMPEG_THREADS);
+		if (threads > 0) {
+			_context->thread_count = static_cast<int>(threads);
+			_lag_in_frames         = _context->thread_count;
+		} else {
+			_context->thread_count = std::thread::hardware_concurrency();
+			_lag_in_frames         = _context->thread_count;
+		}
+	}
+
 	if (_handler)
 		_handler->update(settings, _codec, _context);
 
@@ -874,88 +926,8 @@ bool obsffmpeg::encoder::video_encode(encoder_frame* frame, encoder_packet* pack
 		}
 	}
 
-	// Send and receive frames.
+	if (!encode_avframe(vframe, packet, received_packet))
-	{
-#ifdef _DEBUG
-		ScopeProfiler profile("loop");
-#endif
-
-		bool sent_frame  = false;
-		bool recv_packet = false;
-		bool should_lag  = (_lag_in_frames - _count_send_frames) <= 0;
-
-		auto loop_begin = std::chrono::high_resolution_clock::now();
-		auto loop_end   = loop_begin + std::chrono::milliseconds(50);
-
-		while ((!sent_frame || (should_lag && !recv_packet))
-		       && !(std::chrono::high_resolution_clock::now() > loop_end)) {
-			bool eagain_is_stupid = false;
-
-			if (!sent_frame) {
-#ifdef _DEBUG
-				ScopeProfiler profile_inner("send");
-#endif
-				int res = send_frame(vframe);
-				switch (res) {
-				case 0:
-					sent_frame = true;
-					vframe     = nullptr;
-					break;
-				case AVERROR(EAGAIN):
-					// This means we should call receive_packet again, but what do we do with that data?
-					// Why can't we queue on both? Do I really have to implement threading for this stuff?
-					if (*received_packet == true) {
-						PLOG_WARNING(
-						    "Skipped frame due to EAGAIN when a packet was already returned.");
-						sent_frame = true;
-					}
-					eagain_is_stupid = true;
-					break;
-				case AVERROR(EOF):
-					PLOG_ERROR("Skipped frame due to end of stream.");
-					sent_frame = true;
-					break;
-				default:
-					PLOG_ERROR("Failed to encode frame: %s (%ld).",
-					           ffmpeg::tools::get_error_description(res), res);
 		return false;
-				}
-			}
-
-			if (!recv_packet) {
-#ifdef _DEBUG
-				ScopeProfiler profile_inner("recieve");
-#endif
-				int res = receive_packet(received_packet, packet);
-				switch (res) {
-				case 0:
-					recv_packet = true;
-					break;
-				case AVERROR(EOF):
-					PLOG_ERROR("Received end of file.");
-					recv_packet = true;
-					break;
-				case AVERROR(EAGAIN):
-					if (sent_frame) {
-						recv_packet = true;
-					}
-					if (eagain_is_stupid) {
-						PLOG_ERROR("Both send and recieve returned EAGAIN, encoder is broken.");
-						return false;
-					}
-					break;
-				default:
-					PLOG_ERROR("Failed to receive packet: %s (%ld).",
-					           ffmpeg::tools::get_error_description(res), res);
-					return false;
-				}
-			}
-
-			if (!sent_frame || !recv_packet) {
-				std::this_thread::sleep_for(std::chrono::milliseconds(1));
-			}
-		}
-	}
 
 	if (vframe != nullptr) {
 		_frame_queue.push(vframe);
@@ -964,9 +936,34 @@ bool obsffmpeg::encoder::video_encode(encoder_frame* frame, encoder_packet* pack
 	return true;
 }
 
-bool obsffmpeg::encoder::video_encode_texture(uint32_t, int64_t, uint64_t, uint64_t*, encoder_packet*, bool*)
+bool obsffmpeg::encoder::video_encode_texture(uint32_t handle, int64_t pts, uint64_t lock_key, uint64_t* next_lock_key,
+                                              encoder_packet* packet, bool* received_packet)
 {
+	if (handle == GS_INVALID_HANDLE) {
+		PLOG_ERROR("Received invalid handle.");
+		*next_lock_key = lock_key;
 		return false;
+	}
+
+	std::shared_ptr<AVFrame> frame{av_frame_alloc(), [](void* ptr) {
+		                               av_frame_unref(reinterpret_cast<AVFrame*>(ptr));
+		                               av_frame_free(reinterpret_cast<AVFrame**>(&ptr));
+	                               }};
+		
+	std::shared_ptr<AVFrame> vframe = _hwinst->avframe_from_obs(_context->hw_frames_ctx, handle, lock_key, next_lock_key);
+
+	vframe->color_range     = _context->color_range;
+	vframe->colorspace      = _context->colorspace;
+	vframe->color_primaries = _context->color_primaries;
+	vframe->color_trc       = _context->color_trc;
+	vframe->pts             = pts;
+
+	if (!encode_avframe(vframe, packet, received_packet))
+		return false;
+
+	*next_lock_key = lock_key;
+
+	return true;
 }
 
 int obsffmpeg::encoder::receive_packet(bool* received_packet, struct encoder_packet* packet)
@@ -1039,6 +1036,7 @@ int obsffmpeg::encoder::send_frame(std::shared_ptr<AVFrame> const frame)
 	int res = avcodec_send_frame(_context, frame.get());
 	switch (res) {
 	case 0:
+		if (!_hwapi)
 			_frame_queue_used.push(frame);
 		_count_send_frames++;
 	case AVERROR(EAGAIN):
@@ -1047,3 +1045,87 @@ int obsffmpeg::encoder::send_frame(std::shared_ptr<AVFrame> const frame)
 	}
 	return res;
 }
+
+bool obsffmpeg::encoder::encode_avframe(std::shared_ptr<AVFrame>& frame, encoder_packet* packet, bool* received_packet)
+{
+#ifdef _DEBUG
+	ScopeProfiler profile("loop");
+#endif
+
+	bool sent_frame  = false;
+	bool recv_packet = false;
+	bool should_lag  = (_lag_in_frames - _count_send_frames) <= 0;
+
+	auto loop_begin = std::chrono::high_resolution_clock::now();
+	auto loop_end   = loop_begin + std::chrono::milliseconds(50);
+
+	while ((!sent_frame || (should_lag && !recv_packet))
+	       && !(std::chrono::high_resolution_clock::now() > loop_end)) {
+		bool eagain_is_stupid = false;
+
+		if (!sent_frame) {
+#ifdef _DEBUG
+			ScopeProfiler profile_inner("send");
+#endif
+			int res = send_frame(frame);
+			switch (res) {
+			case 0:
+				sent_frame = true;
+				frame      = nullptr;
+				break;
+			case AVERROR(EAGAIN):
+				// This means we should call receive_packet again, but what do we do with that data?
+				// Why can't we queue on both? Do I really have to implement threading for this stuff?
+				if (*received_packet == true) {
+					PLOG_WARNING("Skipped frame due to EAGAIN when a packet was already returned.");
+					sent_frame = true;
+				}
+				eagain_is_stupid = true;
+				break;
+			case AVERROR(EOF):
+				PLOG_ERROR("Skipped frame due to end of stream.");
+				sent_frame = true;
+				break;
+			default:
+				PLOG_ERROR("Failed to encode frame: %s (%ld).",
+				           ffmpeg::tools::get_error_description(res), res);
+				return false;
+			}
+		}
+
+		if (!recv_packet) {
+#ifdef _DEBUG
+			ScopeProfiler profile_inner("recieve");
+#endif
+			int res = receive_packet(received_packet, packet);
+			switch (res) {
+			case 0:
+				recv_packet = true;
+				break;
+			case AVERROR(EOF):
+				PLOG_ERROR("Received end of file.");
+				recv_packet = true;
+				break;
+			case AVERROR(EAGAIN):
+				if (sent_frame) {
+					recv_packet = true;
+				}
+				if (eagain_is_stupid) {
+					PLOG_ERROR("Both send and recieve returned EAGAIN, encoder is broken.");
+					return false;
+				}
+				break;
+			default:
+				PLOG_ERROR("Failed to receive packet: %s (%ld).",
+				           ffmpeg::tools::get_error_description(res), res);
+				return false;
+			}
+		}
+
+		if (!sent_frame || !recv_packet) {
+			std::this_thread::sleep_for(std::chrono::milliseconds(1));
+		}
+	}
+
+	return true;
+}

source/encoder.hpp

@@ -27,6 +27,7 @@
 #include <vector>
 #include "ffmpeg/avframe-queue.hpp"
 #include "ffmpeg/swscale.hpp"
+#include "hwapi/base.hpp"
 #include "ui/handler.hpp"
 
 extern "C" {
@@ -82,9 +83,13 @@ namespace obsffmpeg {
 
 		const AVCodec*     _codec;
 		AVCodecContext*    _context;
+		AVHWFramesContext* _hwcontext;
 
 		std::shared_ptr<obsffmpeg::ui::handler> _handler;
 
+		std::shared_ptr<obsffmpeg::hwapi::base>     _hwapi;
+		std::shared_ptr<obsffmpeg::hwapi::instance> _hwinst;
+
 		ffmpeg::avframe_queue _frame_queue;
 		ffmpeg::avframe_queue _frame_queue_used;
 		ffmpeg::swscale       _swscale;
@@ -98,6 +103,9 @@ namespace obsffmpeg {
 		std::vector<uint8_t> _extra_data;
 		std::vector<uint8_t> _sei_data;
 
+		void initialize_sw(obs_data_t* settings);
+		void initialize_hw(obs_data_t* settings);
+
 		public:
 		encoder(obs_data_t* settings, obs_encoder_t* encoder, bool is_texture_encode = false);
 		virtual ~encoder();
@@ -130,5 +138,8 @@ namespace obsffmpeg {
 		int receive_packet(bool* received_packet, struct encoder_packet* packet);
 
 		int send_frame(std::shared_ptr<AVFrame> frame);
+
+		bool encode_avframe(std::shared_ptr<AVFrame>& frame, struct encoder_packet* packet,
+		                    bool* received_packet);
 	};
 } // namespace obsffmpeg

source/hwapi/base.cpp

@@ -0,0 +1,22 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "base.hpp"

source/hwapi/base.hpp

@@ -0,0 +1,61 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <cinttypes>
+#include <list>
+#include <string>
+#include <utility>
+
+extern "C" {
+#pragma warning(push)
+#pragma warning(disable : 4244)
+#include <libavutil/frame.h>
+#include <libavutil/hwcontext.h>
+#pragma warning(pop)
+}
+
+namespace obsffmpeg {
+	namespace hwapi {
+		struct device {
+			std::pair<int64_t, int64_t> id;
+			std::string                 name;
+		};
+
+		class instance;
+
+		class base {
+			public:
+			virtual std::list<obsffmpeg::hwapi::device> enumerate_adapters() = 0;
+
+			virtual std::shared_ptr<obsffmpeg::hwapi::instance> create(obsffmpeg::hwapi::device target) = 0;
+		};
+
+		class instance {
+			public:
+			virtual AVBufferRef* create_device_context() = 0;
+
+			virtual std::shared_ptr<AVFrame> avframe_from_obs(AVBufferRef* frames, uint32_t handle, uint64_t lock_key,
+			                              uint64_t* next_lock_key) = 0;
+		};
+	} // namespace hwapi
+} // namespace obsffmpeg

source/hwapi/d3d11.cpp

@@ -0,0 +1,210 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "d3d11.hpp"
+#include <sstream>
+#include <vector>
+
+extern "C" {
+#pragma warning(push)
+#pragma warning(disable : 4244)
+#include <libavutil/hwcontext_d3d11va.h>
+#pragma warning(pop)
+}
+
+obsffmpeg::hwapi::d3d11::d3d11() : _dxgi_module(0), _d3d11_module(0)
+{
+	_dxgi_module = LoadLibraryW(L"dxgi.dll");
+	if (!_dxgi_module)
+		throw std::runtime_error("Unable to load DXGI");
+
+	_d3d11_module = LoadLibraryW(L"d3d11.dll");
+	if (!_d3d11_module)
+		throw std::runtime_error("Unable to load D3D11");
+
+	_CreateDXGIFactory = reinterpret_cast<CreateDXGIFactory_t>(GetProcAddress(_dxgi_module, "CreateDXGIFactory"));
+	_CreateDXGIFactory1 =
+	    reinterpret_cast<CreateDXGIFactory1_t>(GetProcAddress(_dxgi_module, "CreateDXGIFactory1"));
+	_D3D11CreateDevice = reinterpret_cast<D3D11CreateDevice_t>(GetProcAddress(_d3d11_module, "D3D11CreateDevice"));
+
+	if (!_CreateDXGIFactory && !_CreateDXGIFactory1)
+		throw std::runtime_error("DXGI not supported");
+
+	if (!_D3D11CreateDevice)
+		throw std::runtime_error("D3D11 not supported");
+
+	HRESULT hr = _CreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)&_dxgifactory);
+	if (FAILED(hr)) {
+		std::stringstream sstr;
+		sstr << "Failed to create DXGI Factory (" << hr << ")";
+		throw std::runtime_error(sstr.str());
+	}
+}
+
+obsffmpeg::hwapi::d3d11::~d3d11()
+{
+	FreeLibrary(_dxgi_module);
+	FreeLibrary(_d3d11_module);
+}
+
+std::list<obsffmpeg::hwapi::device> obsffmpeg::hwapi::d3d11::enumerate_adapters()
+{
+	std::list<device> adapters;
+
+	// Enumerate Adapters
+	IDXGIAdapter1* dxgi_adapter = nullptr;
+	for (UINT idx = 0; !FAILED(_dxgifactory->EnumAdapters1(idx, &dxgi_adapter)); idx++) {
+		DXGI_ADAPTER_DESC1 desc = DXGI_ADAPTER_DESC1();
+		dxgi_adapter->GetDesc1(&desc);
+
+		std::vector<char> buf(1024);
+		size_t            len = snprintf(buf.data(), buf.size(), "%ls (VEN_%04x/DEV_%04x/SUB_%04x/REV_%04x)",
+                                      desc.Description, desc.VendorId, desc.DeviceId, desc.SubSysId, desc.Revision);
+
+		device dev;
+		dev.name      = std::string(buf.data(), buf.data() + len);
+		dev.id.first  = desc.AdapterLuid.HighPart;
+		dev.id.second = desc.AdapterLuid.LowPart;
+
+		adapters.push_back(dev);
+	}
+
+	return std::move(adapters);
+}
+
+std::shared_ptr<obsffmpeg::hwapi::instance> obsffmpeg::hwapi::d3d11::create(obsffmpeg::hwapi::device target)
+{
+	std::shared_ptr<d3d11_instance>   inst;
+	ATL::CComPtr<ID3D11Device>        device;
+	ATL::CComPtr<ID3D11DeviceContext> context;
+	IDXGIAdapter1*                    adapter = nullptr;
+
+	// Find the correct "Adapter" (device).
+	IDXGIAdapter1* dxgi_adapter = nullptr;
+	for (UINT idx = 0; !FAILED(_dxgifactory->EnumAdapters1(idx, &dxgi_adapter)); idx++) {
+		DXGI_ADAPTER_DESC1 desc = DXGI_ADAPTER_DESC1();
+		dxgi_adapter->GetDesc1(&desc);
+
+		if ((desc.AdapterLuid.LowPart == target.id.second) && (desc.AdapterLuid.HighPart == target.id.first)) {
+			adapter = dxgi_adapter;
+			break;
+		}
+	}
+
+	// Create a D3D11 Device
+	UINT                           device_flags   = D3D11_CREATE_DEVICE_VIDEO_SUPPORT;
+	std::vector<D3D_FEATURE_LEVEL> feature_levels = {D3D_FEATURE_LEVEL_12_1, D3D_FEATURE_LEVEL_12_0,
+	                                                 D3D_FEATURE_LEVEL_11_1};
+
+	if (FAILED(_D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_HARDWARE, NULL, device_flags, feature_levels.data(),
+	                              static_cast<UINT>(feature_levels.size()), D3D11_SDK_VERSION, &device, NULL,
+	                              &context))) {
+		throw std::runtime_error("Failed to create D3D11 device for target.");
+	}
+
+	return std::make_shared<d3d11_instance>(device, context);
+}
+
+struct D3D11AVFrame {
+	ATL::CComPtr<ID3D11Texture2D> handle;
+};
+
+obsffmpeg::hwapi::d3d11_instance::d3d11_instance(ATL::CComPtr<ID3D11Device>        device,
+                                                 ATL::CComPtr<ID3D11DeviceContext> context)
+{
+	_device  = device;
+	_context = context;
+}
+
+obsffmpeg::hwapi::d3d11_instance::~d3d11_instance() {}
+
+AVBufferRef* obsffmpeg::hwapi::d3d11_instance::create_device_context()
+{
+	AVBufferRef* dctx_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_D3D11VA);
+	if (!dctx_ref)
+		throw std::runtime_error("Failed to allocate AVHWDeviceContext.");
+
+	AVHWDeviceContext*      dctx    = reinterpret_cast<AVHWDeviceContext*>(dctx_ref->data);
+	AVD3D11VADeviceContext* d3d11va = reinterpret_cast<AVD3D11VADeviceContext*>(dctx->hwctx);
+
+	// TODO: Determine if these need an additional reference.
+	d3d11va->device         = _device;
+	d3d11va->device->AddRef();
+	d3d11va->device_context = _context;
+	d3d11va->device_context->AddRef();
+
+	if (av_hwdevice_ctx_init(dctx_ref) < 0)
+		throw std::runtime_error("Failed to initialize AVHWDeviceContext.");
+
+	return dctx_ref;
+}
+
+std::shared_ptr<AVFrame> obsffmpeg::hwapi::d3d11_instance::avframe_from_obs(AVBufferRef* frames, uint32_t handle,
+                                                                            uint64_t lock_key, uint64_t* next_lock_key)
+{
+	AVFrame*                      frame = av_frame_alloc();
+	ATL::CComPtr<IDXGIKeyedMutex> mutex;
+	ATL::CComPtr<ID3D11Texture2D> input;
+	D3D11_TEXTURE2D_DESC          input_desc;
+	D3D11_TEXTURE2D_DESC          output_desc;
+	//ATL::CComPtr<ID3D11Texture2D> output;
+
+	if (FAILED(_device->OpenSharedResource(reinterpret_cast<HANDLE>(static_cast<uintptr_t>(handle)),
+	                                       __uuidof(ID3D11Texture2D), reinterpret_cast<void**>(&input)))) {
+		throw std::runtime_error("Failed to open shared texture resource.");
+	}
+	if (FAILED(input->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast<void**>(&mutex)))) {
+		throw std::runtime_error("Failed to retrieve mutex for texture resource.");
+	}
+
+	if (FAILED(mutex->AcquireSync(lock_key, 1000))) {
+		throw std::runtime_error("Failed to acquire lock on input texture.");
+	}
+
+	// Set some parameters on the input texture, and get its description.
+	UINT evict = input->GetEvictionPriority();
+	input->SetEvictionPriority(DXGI_RESOURCE_PRIORITY_MAXIMUM);
+	input->GetDesc(&input_desc);
+
+	if (av_hwframe_get_buffer(frames, frame, 0) < 0) {
+		throw std::runtime_error("Failed to create AVFrame.");
+	}
+
+	reinterpret_cast<ID3D11Texture2D*>(frame->data[0])->GetDesc(&output_desc);
+
+	// Clone the content of the input texture.
+	_context->CopyResource(reinterpret_cast<ID3D11Texture2D*>(frame->data[0]), input);
+
+	// Restore original parameters on input.
+	input->SetEvictionPriority(evict);
+
+	if (FAILED(mutex->ReleaseSync(lock_key))) {
+		throw std::runtime_error("Failed to release lock on input texture.");
+	}
+
+	// TODO: Determine if this is necessary.
+	mutex->ReleaseSync(*next_lock_key);
+
+	return std::shared_ptr<AVFrame>(frame, [](AVFrame* frame) {
+		av_frame_unref(frame);
+		av_frame_free(&frame);
+	});
+}

source/hwapi/d3d11.hpp

@@ -0,0 +1,73 @@
+// FFMPEG Video Encoder Integration for OBS Studio
+// Copyright (c) 2019 Michael Fabian Dirks <info@xaymar.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include <atlutil.h>
+#include <d3d11.h>
+#include <dxgi.h>
+#include <windows.h>
+#include "base.hpp"
+
+namespace obsffmpeg {
+	namespace hwapi {
+		class d3d11 : public base {
+			typedef HRESULT(__stdcall* CreateDXGIFactory_t)(REFIID, void**);
+			typedef HRESULT(__stdcall* CreateDXGIFactory1_t)(REFIID, void**);
+			typedef HRESULT(__stdcall* D3D11CreateDevice_t)(_In_opt_ IDXGIAdapter*, D3D_DRIVER_TYPE,
+			                                                HMODULE, UINT, CONST D3D_FEATURE_LEVEL*, UINT,
+			                                                UINT, _Out_opt_ ID3D11Device**,
+			                                                _Out_opt_ D3D_FEATURE_LEVEL*,
+			                                                _Out_opt_ ID3D11DeviceContext**);
+
+			HMODULE              _dxgi_module;
+			CreateDXGIFactory_t  _CreateDXGIFactory;
+			CreateDXGIFactory1_t _CreateDXGIFactory1;
+
+			HMODULE             _d3d11_module;
+			D3D11CreateDevice_t _D3D11CreateDevice;
+
+			ATL::CComPtr<IDXGIFactory1> _dxgifactory;
+
+			public:
+			d3d11();
+			virtual ~d3d11();
+
+			virtual std::list<obsffmpeg::hwapi::device> enumerate_adapters() override;
+
+			virtual std::shared_ptr<obsffmpeg::hwapi::instance>
+			    create(obsffmpeg::hwapi::device target) override;
+		};
+
+		class d3d11_instance : public instance {
+			ATL::CComPtr<ID3D11Device>        _device;
+			ATL::CComPtr<ID3D11DeviceContext> _context;
+
+			public:
+			d3d11_instance(ATL::CComPtr<ID3D11Device> device, ATL::CComPtr<ID3D11DeviceContext> context);
+			virtual ~d3d11_instance();
+
+			virtual AVBufferRef* create_device_context() override;
+
+			virtual std::shared_ptr<AVFrame> avframe_from_obs(AVBufferRef* frames, uint32_t handle,
+			                                                  uint64_t  lock_key,
+			                              uint64_t* next_lock_key) override;
+		};
+	} // namespace hwapi
+} // namespace obsffmpeg

source/ui/handler.cpp

@@ -33,6 +33,12 @@ void obsffmpeg::ui::handler::get_defaults(obs_data_t* settings, const AVCodec* c
 
 void obsffmpeg::ui::handler::get_properties(obs_properties_t* props, const AVCodec* codec, AVCodecContext* context) {}
 
+obsffmpeg::hwapi::device obsffmpeg::ui::handler::find_hw_device(std::shared_ptr<obsffmpeg::hwapi::base> api,
+                                                                const AVCodec* codec, AVCodecContext* context)
+{
+	return obsffmpeg::hwapi::device();
+}
+
 void obsffmpeg::ui::handler::update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context) {}
 
 void obsffmpeg::ui::handler::log_options(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context) {}

source/ui/handler.hpp

@@ -22,6 +22,7 @@
 #pragma once
 
 #include <string>
+#include "hwapi/base.hpp"
 
 extern "C" {
 #include <obs.h>
@@ -51,6 +52,9 @@ namespace obsffmpeg {
 			virtual void get_properties(obs_properties_t* props, const AVCodec* codec,
 			                            AVCodecContext* context);
 
+			virtual obsffmpeg::hwapi::device find_hw_device(std::shared_ptr<obsffmpeg::hwapi::base> api,
+			                                                const AVCodec* codec, AVCodecContext* context);
+
 			virtual void update(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context);
 
 			virtual void log_options(obs_data_t* settings, const AVCodec* codec, AVCodecContext* context);