FreeRDP
Loading...
Searching...
No Matches
h264_ffmpeg.c
1
21#include <freerdp/config.h>
22
23#include <winpr/wlog.h>
24#include <freerdp/log.h>
25#include <freerdp/codec/h264.h>
26#include <libavcodec/avcodec.h>
27#include <libavutil/opt.h>
28
29#include "h264.h"
30
31#ifdef WITH_VIDEOTOOLBOX
32#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
33#include <libavutil/hwcontext.h>
34#else
35#pragma warning You have asked for VideoToolbox decoding, \
36 but your version of libavutil is too old !Disabling.
37#undef WITH_VIDEOTOOLBOX
38#endif
39#endif
40
41#ifdef WITH_VAAPI
42#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
43#include <libavutil/hwcontext.h>
44#else
45#pragma warning You have asked for VA - API decoding, \
46 but your version of libavutil is too old !Disabling.
47#undef WITH_VAAPI
48#endif
49#endif
50
51/* Fallback support for older libavcodec versions */
52#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
53#define AV_CODEC_ID_H264 CODEC_ID_H264
54#endif
55
56#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
57#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
58#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
59#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
60#endif
61
62#if LIBAVUTIL_VERSION_MAJOR < 52
63#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
64#endif
65
66/* Ubuntu 14.04 ships without the functions provided by avutil,
67 * so define error to string methods here. */
68#if !defined(av_err2str)
69static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
70{
71 av_strerror(errnum, errbuf, errbuf_size);
72 return errbuf;
73}
74
75#define av_err2str(errnum) error_string((char[64])WINPR_C_ARRAY_INIT, 64, errnum)
76#endif
77
78#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING)
79static const char* get_vaapi_device(void)
80{
81 static char device[MAX_PATH] = WINPR_C_ARRAY_INIT;
82 static bool initialized = false;
83 if (!initialized)
84 {
85 initialized = true;
86 // NOLINTNEXTLINE(concurrency-mt-unsafe)
87 const char* env = getenv("FREERDP_VAAPI_DEVICE");
88 if (env)
89 (void)_snprintf(device, sizeof(device), "%s", env);
90 else
91 (void)_snprintf(device, sizeof(device), "/dev/dri/renderD128");
92 }
93 return device;
94}
95#endif
96
97typedef struct
98{
99 const AVCodec* codecDecoder;
100 AVCodecContext* codecDecoderContext;
101 const AVCodec* codecEncoder;
102 AVCodecContext* codecEncoderContext;
103 AVCodecParserContext* codecParser;
104 AVFrame* videoFrame;
105#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
106 AVPacket bufferpacket;
107#endif
108 AVPacket* packet;
109#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
110 AVBufferRef* hwctx;
111 AVFrame* hwVideoFrame;
112 enum AVPixelFormat hw_pix_fmt;
113#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
114 AVBufferRef* hw_frames_ctx;
115#endif
116
117#endif
118} H264_CONTEXT_LIBAVCODEC;
119
120static void libavcodec_destroy_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
121{
122 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
123
124 if (!h264 || !h264->subsystem)
125 return;
126
127 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
128
129 if (sys->codecEncoderContext)
130 {
131#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
132 avcodec_free_context(&sys->codecEncoderContext);
133#else
134 avcodec_close(sys->codecEncoderContext);
135 av_free(sys->codecEncoderContext);
136#endif
137 }
138
139 sys->codecEncoderContext = nullptr;
140}
141
142#ifdef WITH_VAAPI_H264_ENCODING
143static int set_hw_frames_ctx(H264_CONTEXT* WINPR_RESTRICT h264)
144{
145 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
146 AVBufferRef* hw_frames_ref = nullptr;
147 AVHWFramesContext* frames_ctx = nullptr;
148 int err = 0;
149
150 if (!(hw_frames_ref = av_hwframe_ctx_alloc(sys->hwctx)))
151 {
152 WLog_Print(h264->log, WLOG_ERROR, "Failed to create VAAPI frame context");
153 return -1;
154 }
155 frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data);
156 frames_ctx->format = AV_PIX_FMT_VAAPI;
157 frames_ctx->sw_format = AV_PIX_FMT_NV12;
158 frames_ctx->width = sys->codecEncoderContext->width;
159 frames_ctx->height = sys->codecEncoderContext->height;
160 frames_ctx->initial_pool_size = 20;
161 if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0)
162 {
163 WLog_Print(h264->log, WLOG_ERROR,
164 "Failed to initialize VAAPI frame context."
165 "Error code: %s",
166 av_err2str(err));
167 av_buffer_unref(&hw_frames_ref);
168 return err;
169 }
170 sys->codecEncoderContext->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
171 if (!sys->codecEncoderContext->hw_frames_ctx)
172 err = AVERROR(ENOMEM);
173
174 av_buffer_unref(&hw_frames_ref);
175 return err;
176}
177#endif
178
179static BOOL libavcodec_create_encoder_context(H264_CONTEXT* WINPR_RESTRICT h264)
180{
181 BOOL recreate = FALSE;
182 H264_CONTEXT_LIBAVCODEC* sys = nullptr;
183
184 if (!h264 || !h264->subsystem)
185 return FALSE;
186
187 if ((h264->width > INT_MAX) || (h264->height > INT_MAX))
188 return FALSE;
189
190 sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
191 if (!sys || !sys->codecEncoder)
192 return FALSE;
193
194 recreate = !sys->codecEncoderContext;
195
196 if (sys->codecEncoderContext)
197 {
198 if ((sys->codecEncoderContext->width != (int)h264->width) ||
199 (sys->codecEncoderContext->height != (int)h264->height))
200 recreate = TRUE;
201 }
202
203 if (!recreate)
204 return TRUE;
205
206 libavcodec_destroy_encoder_context(h264);
207
208 sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
209
210 if (!sys->codecEncoderContext)
211 goto EXCEPTION;
212
213 switch (h264->RateControlMode)
214 {
215 case H264_RATECONTROL_VBR:
216 sys->codecEncoderContext->bit_rate = h264->BitRate;
217 break;
218
219 case H264_RATECONTROL_CQP:
220 if (av_opt_set_int(sys->codecEncoderContext, "qp", h264->QP, AV_OPT_SEARCH_CHILDREN) <
221 0)
222 {
223 WLog_Print(h264->log, WLOG_ERROR, "av_opt_set_int failed");
224 }
225 break;
226
227 default:
228 break;
229 }
230
231 sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
232 sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
233 sys->codecEncoderContext->delay = 0;
234#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
235 sys->codecEncoderContext->framerate =
236 (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 };
237#endif
238 sys->codecEncoderContext->time_base =
239 (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) };
240 av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
241
242 sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
243
244#ifdef WITH_VAAPI_H264_ENCODING
245 if (sys->hwctx)
246 {
247 av_opt_set(sys->codecEncoderContext, "preset", "veryslow", AV_OPT_SEARCH_CHILDREN);
248
249 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_VAAPI;
250 /* set hw_frames_ctx for encoder's AVCodecContext */
251 if (set_hw_frames_ctx(h264) < 0)
252 goto EXCEPTION;
253 }
254 else
255#endif
256 {
257 av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
258 sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
259 }
260
261 if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, nullptr) < 0)
262 goto EXCEPTION;
263
264 return TRUE;
265EXCEPTION:
266 libavcodec_destroy_encoder_context(h264);
267 return FALSE;
268}
269
270static int libavcodec_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
271 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize)
272{
273 union
274 {
275 const BYTE* cpv;
276 BYTE* pv;
277 } cnv;
278 int rc = -1;
279 int status = 0;
280 int gotFrame = 0;
281 AVPacket* packet = nullptr;
282
283 WINPR_ASSERT(h264);
284 WINPR_ASSERT(pSrcData || (SrcSize == 0));
285
286 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
287 BYTE** pYUVData = h264->pYUVData;
288 UINT32* iStride = h264->iStride;
289
290 WINPR_ASSERT(sys);
291
292#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
293 packet = &sys->bufferpacket;
294 WINPR_ASSERT(packet);
295 av_init_packet(packet);
296#else
297 packet = av_packet_alloc();
298#endif
299 if (!packet)
300 {
301 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
302 goto fail;
303 }
304
305 cnv.cpv = pSrcData;
306 packet->data = cnv.pv;
307 packet->size = (int)MIN(SrcSize, INT32_MAX);
308
309 WINPR_ASSERT(sys->codecDecoderContext);
310 /* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
311#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
312 status = avcodec_send_packet(sys->codecDecoderContext, packet);
313
314 if (status < 0)
315 {
316 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
317 goto fail;
318 }
319
320 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
321
322#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
323 status = avcodec_receive_frame(sys->codecDecoderContext,
324 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
325#else
326 status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
327#endif
328 if (status == AVERROR(EAGAIN))
329 {
330 rc = 0;
331 goto fail;
332 }
333
334 if (status == 0)
335 gotFrame = 1;
336#else
337#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
338 status =
339 avcodec_decode_video2(sys->codecDecoderContext,
340 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame, packet);
341#else
342 status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, packet);
343#endif
344#endif
345 if (status < 0)
346 {
347 WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
348 goto fail;
349 }
350
351#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
352
353 if (sys->hwctx)
354 {
355 if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
356 {
357 sys->videoFrame->width = sys->hwVideoFrame->width;
358 sys->videoFrame->height = sys->hwVideoFrame->height;
359 status = av_hwframe_transfer_data(sys->videoFrame, sys->hwVideoFrame, 0);
360 }
361 else
362 {
363 status = av_frame_copy(sys->videoFrame, sys->hwVideoFrame);
364 }
365 gotFrame = (status == 0);
366
367 if (status < 0)
368 {
369 WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)",
370 status, av_err2str(status));
371 goto fail;
372 }
373 }
374
375#endif
376
377 if (gotFrame)
378 {
379 WINPR_ASSERT(sys->videoFrame);
380
381 pYUVData[0] = sys->videoFrame->data[0];
382 pYUVData[1] = sys->videoFrame->data[1];
383 pYUVData[2] = sys->videoFrame->data[2];
384 iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
385 iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
386 iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
387
388 if (sys->videoFrame->width < WINPR_ASSERTING_INT_CAST(int64_t, h264->width))
389 goto fail;
390 if (sys->videoFrame->height < WINPR_ASSERTING_INT_CAST(int64_t, h264->height))
391 goto fail;
392 rc = 1;
393 }
394 else
395 rc = -2;
396
397fail:
398#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
399 av_packet_unref(packet);
400#else
401 av_packet_free(&packet);
402#endif
403
404 return rc;
405}
406
407static int libavcodec_compress(H264_CONTEXT* WINPR_RESTRICT h264,
408 const BYTE** WINPR_RESTRICT pSrcYuv,
409 const UINT32* WINPR_RESTRICT pStride,
410 BYTE** WINPR_RESTRICT ppDstData, UINT32* WINPR_RESTRICT pDstSize)
411{
412 union
413 {
414 const BYTE* cpv;
415 uint8_t* pv;
416 } cnv;
417 int rc = -1;
418 int status = 0;
419 int gotFrame = 0;
420
421 WINPR_ASSERT(h264);
422
423 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
424 WINPR_ASSERT(sys);
425
426 if (!libavcodec_create_encoder_context(h264))
427 return -1;
428
429#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
430 sys->packet = &sys->bufferpacket;
431 av_packet_unref(sys->packet);
432 av_init_packet(sys->packet);
433#else
434 av_packet_free(&sys->packet);
435 sys->packet = av_packet_alloc();
436#endif
437 if (!sys->packet)
438 {
439 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate AVPacket");
440 goto fail;
441 }
442
443 WINPR_ASSERT(sys->packet);
444 sys->packet->data = nullptr;
445 sys->packet->size = 0;
446
447 WINPR_ASSERT(sys->videoFrame);
448 WINPR_ASSERT(sys->codecEncoderContext);
449 sys->videoFrame->format = AV_PIX_FMT_YUV420P;
450 sys->videoFrame->width = sys->codecEncoderContext->width;
451 sys->videoFrame->height = sys->codecEncoderContext->height;
452#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
453 sys->videoFrame->colorspace = AVCOL_SPC_BT709;
454#endif
455#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
456 sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
457#endif
458 cnv.cpv = pSrcYuv[0];
459 sys->videoFrame->data[0] = cnv.pv;
460
461 cnv.cpv = pSrcYuv[1];
462 sys->videoFrame->data[1] = cnv.pv;
463
464 cnv.cpv = pSrcYuv[2];
465 sys->videoFrame->data[2] = cnv.pv;
466
467 sys->videoFrame->linesize[0] = (int)pStride[0];
468 sys->videoFrame->linesize[1] = (int)pStride[1];
469 sys->videoFrame->linesize[2] = (int)pStride[2];
470 sys->videoFrame->pts++;
471
472#ifdef WITH_VAAPI_H264_ENCODING
473 if (sys->hwctx)
474 {
475 av_frame_unref(sys->hwVideoFrame);
476 if ((status = av_hwframe_get_buffer(sys->codecEncoderContext->hw_frames_ctx,
477 sys->hwVideoFrame, 0)) < 0 ||
478 !sys->hwVideoFrame->hw_frames_ctx)
479 {
480 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_get_buffer failed (%s [%d])",
481 av_err2str(status), status);
482 goto fail;
483 }
484 sys->videoFrame->format = AV_PIX_FMT_NV12;
485 if ((status = av_hwframe_transfer_data(sys->hwVideoFrame, sys->videoFrame, 0)) < 0)
486 {
487 WLog_Print(h264->log, WLOG_ERROR, "av_hwframe_transfer_data failed (%s [%d])",
488 av_err2str(status), status);
489 goto fail;
490 }
491 }
492#endif
493
494 /* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
495#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
496#ifdef WITH_VAAPI_H264_ENCODING
497 status = avcodec_send_frame(sys->codecEncoderContext,
498 sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
499#else
500 status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
501#endif
502
503 if (status < 0)
504 {
505 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
506 av_err2str(status), status);
507 goto fail;
508 }
509
510 status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
511
512 if (status < 0)
513 {
514 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
515 av_err2str(status), status);
516 goto fail;
517 }
518
519 gotFrame = (status == 0);
520#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
521
522 do
523 {
524 status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
525 &gotFrame);
526 } while ((status >= 0) && (gotFrame == 0));
527
528#else
529 sys->packet->size =
530 avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
531 sys->codecDecoderContext->height);
532 sys->packet->data = av_malloc(sys->packet->size);
533
534 if (!sys->packet->data)
535 status = -1;
536 else
537 {
538 status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
539 sys->packet->size, sys->videoFrame);
540 }
541
542#endif
543
544 if (status < 0)
545 {
546 WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
547 av_err2str(status), status);
548 goto fail;
549 }
550
551 WINPR_ASSERT(sys->packet);
552 *ppDstData = sys->packet->data;
553 *pDstSize = (UINT32)MAX(0, sys->packet->size);
554
555 if (!gotFrame)
556 {
557 WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status),
558 status);
559 rc = -2;
560 }
561 else
562 rc = 1;
563fail:
564 return rc;
565}
566
567static void libavcodec_uninit(H264_CONTEXT* h264)
568{
569 WINPR_ASSERT(h264);
570
571 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
572
573 if (!sys)
574 return;
575
576 if (sys->packet)
577 {
578#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
579 av_packet_unref(sys->packet);
580#else
581 av_packet_free(&sys->packet);
582#endif
583 }
584
585 if (sys->videoFrame)
586 {
587#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
588 av_frame_free(&sys->videoFrame);
589#else
590 av_free(sys->videoFrame);
591#endif
592 }
593
594#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
595 if (sys->hwVideoFrame)
596 {
597#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
598 av_frame_free(&sys->hwVideoFrame);
599#else
600 av_free(sys->hwVideoFrame);
601#endif
602 }
603
604 if (sys->hwctx)
605 av_buffer_unref(&sys->hwctx);
606
607#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
608
609 if (sys->hw_frames_ctx)
610 av_buffer_unref(&sys->hw_frames_ctx);
611
612#endif
613
614#endif
615
616 if (sys->codecParser)
617 av_parser_close(sys->codecParser);
618
619 if (sys->codecDecoderContext)
620 {
621#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
622 avcodec_free_context(&sys->codecDecoderContext);
623#else
624 avcodec_close(sys->codecDecoderContext);
625 av_free(sys->codecDecoderContext);
626#endif
627 }
628
629 libavcodec_destroy_encoder_context(h264);
630 free(sys);
631 h264->pSystemData = nullptr;
632}
633
634#if defined(WITH_VAAPI) || defined(WITH_VIDEOTOOLBOX)
635static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
636 const enum AVPixelFormat* fmts)
637{
638 WINPR_ASSERT(ctx);
639
640 H264_CONTEXT* h264 = (H264_CONTEXT*)ctx->opaque;
641 WINPR_ASSERT(h264);
642
643 H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
644 WINPR_ASSERT(sys);
645
646 for (const enum AVPixelFormat* p = fmts; *p != AV_PIX_FMT_NONE; p++)
647 {
648 if (*p == sys->hw_pix_fmt)
649 {
650#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100) || defined(WITH_VIDEOTOOLBOX)
651 if (sys->hw_frames_ctx)
652 av_buffer_unref(&sys->hw_frames_ctx);
653
654 sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
655
656 if (!sys->hw_frames_ctx)
657 {
658 return AV_PIX_FMT_NONE;
659 }
660
661 sys->codecDecoderContext->pix_fmt = *p;
662 AVHWFramesContext* frames = (AVHWFramesContext*)sys->hw_frames_ctx->data;
663 frames->format = *p;
664 frames->height = sys->codecDecoderContext->coded_height;
665 frames->width = sys->codecDecoderContext->coded_width;
666#ifdef WITH_VIDEOTOOLBOX
667 frames->sw_format = AV_PIX_FMT_YUV420P;
668#else
669 frames->sw_format =
670 (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010
671 : AV_PIX_FMT_NV12);
672#endif
673 frames->initial_pool_size = 20;
674
675 if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
676 frames->initial_pool_size += sys->codecDecoderContext->thread_count;
677
678 int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
679
680 if (err < 0)
681 {
682 WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s",
683 av_err2str(err));
684 return AV_PIX_FMT_NONE;
685 }
686
687 sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
688#endif
689 return *p;
690 }
691 }
692
693 return AV_PIX_FMT_NONE;
694}
695#endif
696
697static BOOL libavcodec_init(H264_CONTEXT* h264)
698{
699 WINPR_ASSERT(h264);
700 H264_CONTEXT_LIBAVCODEC* sys =
701 (H264_CONTEXT_LIBAVCODEC*)calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
702
703 if (!sys)
704 {
705 goto EXCEPTION;
706 }
707
708 h264->pSystemData = (void*)sys;
709
710#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
711 avcodec_register_all();
712#endif
713
714 if (!h264->Compressor)
715 {
716 sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
717
718 if (!sys->codecDecoder)
719 {
720 WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
721 goto EXCEPTION;
722 }
723
724 sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
725
726 if (!sys->codecDecoderContext)
727 {
728 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
729 goto EXCEPTION;
730 }
731
732#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 18, 100)
733 if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
734 {
735 sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
736 }
737#endif
738
739#ifdef WITH_VAAPI
740
741 if (!sys->hwctx)
742 {
743 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI,
744 get_vaapi_device(), nullptr, 0);
745
746 if (ret < 0)
747 {
748 WLog_Print(h264->log, WLOG_ERROR,
749 "Could not initialize hardware decoder, falling back to software: %s",
750 av_err2str(ret));
751 sys->hwctx = nullptr;
752 goto fail_hwdevice_create;
753 }
754 }
755 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 decoding");
756
757 sys->codecDecoderContext->get_format = libavcodec_get_format;
758 sys->hw_pix_fmt = AV_PIX_FMT_VAAPI;
759#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
760 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
761#endif
762 sys->codecDecoderContext->opaque = (void*)h264;
763 fail_hwdevice_create:
764#endif
765
766#ifdef WITH_VIDEOTOOLBOX
767
768 if (!sys->hwctx)
769 {
770 int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr,
771 nullptr, 0);
772
773 if (ret < 0)
774 {
775 WLog_Print(
776 h264->log, WLOG_ERROR,
777 "Could not initialize VideoToolbox decoder, falling back to software: %s",
778 av_err2str(ret));
779 sys->hwctx = nullptr;
780 goto fail_vt_create;
781 }
782 }
783 WLog_Print(h264->log, WLOG_INFO, "Using VideoToolbox for accelerated H264 decoding");
784
785 sys->codecDecoderContext->get_format = libavcodec_get_format;
786 sys->hw_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
787 sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
788 sys->codecDecoderContext->opaque = (void*)h264;
789 fail_vt_create:
790#endif
791
792 if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, nullptr) < 0)
793 {
794 WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
795 goto EXCEPTION;
796 }
797
798 sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
799
800 if (!sys->codecParser)
801 {
802 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
803 goto EXCEPTION;
804 }
805 }
806 else
807 {
808#ifdef WITH_VAAPI_H264_ENCODING
809 if (h264->hwAccel) /* user requested hw accel */
810 {
811 sys->codecEncoder = avcodec_find_encoder_by_name("h264_vaapi");
812 if (!sys->codecEncoder)
813 {
814 WLog_Print(h264->log, WLOG_ERROR, "H264 VAAPI encoder not found");
815 }
816 else if (av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI, get_vaapi_device(),
817 nullptr, 0) < 0)
818 {
819 WLog_Print(h264->log, WLOG_ERROR, "av_hwdevice_ctx_create failed");
820 sys->codecEncoder = nullptr;
821 sys->hwctx = nullptr;
822 }
823 else
824 {
825 WLog_Print(h264->log, WLOG_INFO, "Using VAAPI for accelerated H264 encoding");
826 }
827 }
828#endif
829 if (!sys->codecEncoder)
830 {
831 sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
832 h264->hwAccel = FALSE; /* not supported */
833 }
834
835 if (!sys->codecEncoder)
836 {
837 WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize H264 encoder");
838 goto EXCEPTION;
839 }
840 }
841
842#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
843 sys->videoFrame = av_frame_alloc();
844#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
845 sys->hwVideoFrame = av_frame_alloc();
846#endif
847#else
848 sys->videoFrame = avcodec_alloc_frame();
849#endif
850
851 if (!sys->videoFrame)
852 {
853 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
854 goto EXCEPTION;
855 }
856
857#if defined(WITH_VAAPI) || defined(WITH_VAAPI_H264_ENCODING) || defined(WITH_VIDEOTOOLBOX)
858 if (!sys->hwVideoFrame)
859 {
860 WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
861 goto EXCEPTION;
862 }
863
864#endif
865 sys->videoFrame->pts = 0;
866 return TRUE;
867EXCEPTION:
868 libavcodec_uninit(h264);
869 return FALSE;
870}
871
872const H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec = { "libavcodec", libavcodec_init,
873 libavcodec_uninit, libavcodec_decompress,
874 libavcodec_compress };