encoder: Add support for true hardware encoding

This is the last step towards truly efficient encoding on AMD, Nvidia and Intel GPUs. With this we have no software overhead and can directly encode the content that OBS gives us, without going through any intermediate CPU layer. This is effectively what @jp9000 did for the OBS-integrated nvenc, but thanks to FFmpeg it works on all encoders that support D3D11VA acceleration.

With the change, the encoding should now work flawlessly even in very constrained situations (unless OBS itself is being starved of resources). Especially people streaming and recording Ubisoft games will likely see a drastic increase in encoding capability, and thanks to the new options will also be able to get a much higher quality stream and recording with the same hardware.

source/encoder.hpp

@@ -27,6 +27,7 @@
 #include <vector>
 #include "ffmpeg/avframe-queue.hpp"
 #include "ffmpeg/swscale.hpp"
+#include "hwapi/base.hpp"
 #include "ui/handler.hpp"
 
 extern "C" {
@@ -80,11 +81,15 @@ namespace obsffmpeg {
 		obs_encoder_t*   _self;
 		encoder_factory* _factory;
 
-		const AVCodec*  _codec;
-		AVCodecContext* _context;
+		const AVCodec*     _codec;
+		AVCodecContext*    _context;
+		AVHWFramesContext* _hwcontext;
 
 		std::shared_ptr<obsffmpeg::ui::handler> _handler;
 
+		std::shared_ptr<obsffmpeg::hwapi::base>     _hwapi;
+		std::shared_ptr<obsffmpeg::hwapi::instance> _hwinst;
+
 		ffmpeg::avframe_queue _frame_queue;
 		ffmpeg::avframe_queue _frame_queue_used;
 		ffmpeg::swscale       _swscale;
@@ -98,6 +103,9 @@ namespace obsffmpeg {
 		std::vector<uint8_t> _extra_data;
 		std::vector<uint8_t> _sei_data;
 
+		void initialize_sw(obs_data_t* settings);
+		void initialize_hw(obs_data_t* settings);
+
 		public:
 		encoder(obs_data_t* settings, obs_encoder_t* encoder, bool is_texture_encode = false);
 		virtual ~encoder();
@@ -130,5 +138,8 @@ namespace obsffmpeg {
 		int receive_packet(bool* received_packet, struct encoder_packet* packet);
 
 		int send_frame(std::shared_ptr<AVFrame> frame);
+
+		bool encode_avframe(std::shared_ptr<AVFrame>& frame, struct encoder_packet* packet,
+		                    bool* received_packet);
 	};
 } // namespace obsffmpeg