20#include <freerdp/config.h>
22#include <winpr/assert.h>
23#include <winpr/wlog.h>
24#include <winpr/stream.h>
25#include <freerdp/log.h>
27#include <freerdp/codec/video.h>
28#include <freerdp/codec/h264.h>
30#define TAG FREERDP_TAG("codec.video")
32#if defined(WITH_SWSCALE)
35static BOOL freerdp_video_fill_plane_info(BYTE* data[4],
int lineSize[4],
36 FREERDP_VIDEO_FORMAT format, UINT32 width, UINT32 height,
39#include "image_ffmpeg.h"
42#if defined(WITH_VIDEO_FFMPEG) && !defined(WITH_SWSCALE_LOADING)
43#define WITH_MJPEG_DECODER
44#include <libavcodec/avcodec.h>
47struct s_FREERDP_VIDEO_CONTEXT
51 struct SwsContext* sws;
53#if defined(WITH_MJPEG_DECODER)
54 AVCodecContext* mjpegDecoder;
55 AVPacket* mjpegPacket;
69static enum AVPixelFormat video_format_to_av(FREERDP_VIDEO_FORMAT format)
74 case FREERDP_VIDEO_FORMAT_MJPEG:
75 case FREERDP_VIDEO_FORMAT_H264:
76 return AV_PIX_FMT_NONE;
79 case FREERDP_VIDEO_FORMAT_YUV420P:
80 return AV_PIX_FMT_YUV420P;
81 case FREERDP_VIDEO_FORMAT_NV12:
82 return AV_PIX_FMT_NV12;
85 case FREERDP_VIDEO_FORMAT_YUYV422:
86 return AV_PIX_FMT_YUYV422;
89 case FREERDP_VIDEO_FORMAT_RGB24:
90 return AV_PIX_FMT_RGB24;
91 case FREERDP_VIDEO_FORMAT_RGB32:
92 return AV_PIX_FMT_RGB32;
94 case FREERDP_VIDEO_FORMAT_NONE:
96 return AV_PIX_FMT_NONE;
100#if defined(WITH_MJPEG_DECODER)
104static FREERDP_VIDEO_FORMAT av_format_to_video(
enum AVPixelFormat format)
109 case AV_PIX_FMT_YUV420P:
110 return FREERDP_VIDEO_FORMAT_YUV420P;
112 case AV_PIX_FMT_NV12:
113 return FREERDP_VIDEO_FORMAT_NV12;
116 case AV_PIX_FMT_YUYV422:
117 return FREERDP_VIDEO_FORMAT_YUYV422;
120 case AV_PIX_FMT_RGB24:
121 return FREERDP_VIDEO_FORMAT_RGB24;
123 case AV_PIX_FMT_RGB32:
124 return FREERDP_VIDEO_FORMAT_RGB32;
127 return FREERDP_VIDEO_FORMAT_NONE;
132BOOL freerdp_video_available(
void)
134 return freerdp_swscale_available() && freerdp_avutil_available();
137FREERDP_VIDEO_CONTEXT* freerdp_video_context_new(UINT32 width, UINT32 height)
139 FREERDP_VIDEO_CONTEXT* context =
nullptr;
141 if (!freerdp_video_available())
143 WLog_ERR(TAG,
"Video codecs not available - FFmpeg not loaded");
147 context = (FREERDP_VIDEO_CONTEXT*)calloc(1,
sizeof(FREERDP_VIDEO_CONTEXT));
151 context->width = width;
152 context->height = height;
154#if defined(WITH_MJPEG_DECODER)
156 const AVCodec* codec = avcodec_find_decoder(AV_CODEC_ID_MJPEG);
159 WLog_ERR(TAG,
"avcodec_find_decoder failed to find MJPEG codec");
163 context->mjpegDecoder = avcodec_alloc_context3(codec);
164 if (!context->mjpegDecoder)
166 WLog_ERR(TAG,
"avcodec_alloc_context3 failed");
170 context->mjpegDecoder->width = (int)width;
171 context->mjpegDecoder->height = (int)height;
173 context->mjpegDecoder->err_recognition |= AV_EF_EXPLODE;
175 if (avcodec_open2(context->mjpegDecoder, codec,
nullptr) < 0)
177 WLog_ERR(TAG,
"avcodec_open2 failed");
181 context->mjpegPacket = av_packet_alloc();
182 if (!context->mjpegPacket)
184 WLog_ERR(TAG,
"av_packet_alloc failed");
188 context->mjpegFrame = av_frame_alloc();
189 if (!context->mjpegFrame)
191 WLog_ERR(TAG,
"av_frame_alloc failed");
198#if defined(WITH_MJPEG_DECODER)
200 freerdp_video_context_free(context);
205void freerdp_video_context_free(FREERDP_VIDEO_CONTEXT* context)
212 freerdp_sws_freeContext(context->sws);
213 context->sws =
nullptr;
216#if defined(WITH_MJPEG_DECODER)
217 if (context->mjpegFrame)
218 av_frame_free(&context->mjpegFrame);
220 if (context->mjpegPacket)
222 context->mjpegPacket->data =
nullptr;
223 context->mjpegPacket->size = 0;
224 av_packet_free(&context->mjpegPacket);
227 if (context->mjpegDecoder)
228 avcodec_free_context(&context->mjpegDecoder);
233 h264_context_free(context->h264);
234 context->h264 =
nullptr;
240static UINT32 video_get_h264_bitrate(UINT32 height)
247 { 1080, 2700 }, { 720, 1250 }, { 480, 700 }, { 360, 400 },
248 { 240, 170 }, { 180, 140 }, { 0, 100 },
250 const size_t nBitrates = ARRAYSIZE(bitrates);
252 for (
size_t i = 0; i < nBitrates; i++)
254 if (height >= bitrates[i].height)
256 UINT32 bitrate = bitrates[i].bitrate;
257 WLog_DBG(TAG,
"Auto-calculated H.264 bitrate: %u kbps", bitrate);
258 return bitrate * 1000;
266BOOL freerdp_video_context_reconfigure(FREERDP_VIDEO_CONTEXT* context, UINT32 width, UINT32 height,
267 UINT32 framerate, UINT32 bitrate, UINT32 usageType)
269 WINPR_ASSERT(context);
271 if (width == 0 || height == 0)
273 WLog_ERR(TAG,
"Invalid dimensions: %ux%u", width, height);
278 bitrate = video_get_h264_bitrate(height);
282 context->h264 = h264_context_new(TRUE);
285 WLog_ERR(TAG,
"h264_context_new failed");
290 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_USAGETYPE, usageType))
293 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_FRAMERATE, framerate))
296 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_BITRATE, bitrate))
299 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_RATECONTROL,
300 H264_RATECONTROL_CQP))
303 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_QP, 26))
306 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_HW_ACCEL, FALSE))
309 if (!context->h264Configured || (context->width != width) || (context->height != height))
311 if (!h264_context_reset(context->h264, width, height))
313 WLog_ERR(TAG,
"h264_context_reset failed");
318 context->h264Framerate = framerate;
319 context->h264Bitrate = bitrate;
320 context->h264UsageType = usageType;
321 context->h264Configured = TRUE;
322 context->width = width;
323 context->height = height;
330 h264_context_free(context->h264);
331 context->h264 =
nullptr;
336static BOOL freerdp_video_decode_mjpeg(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData,
337 size_t srcSize, BYTE* dstData[4],
int dstLineSize[4],
338 FREERDP_VIDEO_FORMAT* dstFormat);
340static BOOL freerdp_video_convert_to_yuv(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData[4],
341 const int srcLineSize[4], FREERDP_VIDEO_FORMAT srcFormat,
342 BYTE* dstData[3],
const int dstLineSize[3],
343 FREERDP_VIDEO_FORMAT dstFormat, UINT32 width,
346BOOL freerdp_video_sample_convert(FREERDP_VIDEO_CONTEXT* context, FREERDP_VIDEO_FORMAT srcFormat,
347 const void* srcSampleData,
size_t srcSampleLength,
348 FREERDP_VIDEO_FORMAT dstFormat,
wStream* output)
350 WINPR_ASSERT(context);
351 WINPR_ASSERT(srcSampleData);
352 WINPR_ASSERT(output);
354 if (srcFormat == dstFormat)
356 if (!Stream_EnsureRemainingCapacity(output, srcSampleLength))
358 Stream_Write(output, srcSampleData, srcSampleLength);
362 if (!freerdp_video_available())
364 WLog_ERR(TAG,
"Video codecs not available");
368 if (srcSampleLength == 0)
370 WLog_ERR(TAG,
"Invalid source sample length: 0");
374 if (!freerdp_video_conversion_supported(srcFormat, dstFormat))
376 WLog_ERR(TAG,
"Conversion from format %u to %u not supported", srcFormat, dstFormat);
380 const BOOL srcCompressed =
381 (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG || srcFormat == FREERDP_VIDEO_FORMAT_H264);
382 const BOOL dstCompressed =
383 (dstFormat == FREERDP_VIDEO_FORMAT_MJPEG || dstFormat == FREERDP_VIDEO_FORMAT_H264);
385 BYTE* intermediate_data[4] = WINPR_C_ARRAY_INIT;
386 int intermediate_linesize[4] = WINPR_C_ARRAY_INIT;
387 FREERDP_VIDEO_FORMAT intermediate_format = FREERDP_VIDEO_FORMAT_NONE;
391 if (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG)
393 if (!freerdp_video_decode_mjpeg(context, (
const BYTE*)srcSampleData, srcSampleLength,
394 intermediate_data, intermediate_linesize,
395 &intermediate_format))
397 WLog_ERR(TAG,
"MJPEG decoding failed");
401 else if (srcFormat == FREERDP_VIDEO_FORMAT_H264)
403 WLog_ERR(TAG,
"H264 decoding not supported");
409 intermediate_format = srcFormat;
412 if (dstFormat == FREERDP_VIDEO_FORMAT_H264)
414 if (!context->h264Configured)
416 WLog_ERR(TAG,
"H264 encoder not configured");
420 const UINT32 hwAccel = h264_context_get_option(context->h264, H264_CONTEXT_OPTION_HW_ACCEL);
421 const FREERDP_VIDEO_FORMAT yuvFormat =
422 hwAccel ? FREERDP_VIDEO_FORMAT_NV12 : FREERDP_VIDEO_FORMAT_YUV420P;
424 BYTE* yuvData[3] = WINPR_C_ARRAY_INIT;
425 UINT32 yuvStrides[3] = WINPR_C_ARRAY_INIT;
427 if (h264_get_yuv_buffer(context->h264, 0, context->width, context->height, yuvData,
430 WLog_ERR(TAG,
"h264_get_yuv_buffer failed");
434 int yuvLineSizes[4] = { (int)yuvStrides[0], (
int)yuvStrides[1], (int)yuvStrides[2], 0 };
438 const BYTE* cIntermediateData[4] = { intermediate_data[0], intermediate_data[1],
439 intermediate_data[2], intermediate_data[3] };
440 if (!freerdp_video_convert_to_yuv(context, cIntermediateData, intermediate_linesize,
441 intermediate_format, yuvData, yuvLineSizes, yuvFormat,
442 context->width, context->height))
444 WLog_ERR(TAG,
"YUV conversion failed");
450 BYTE* srcPlanes[4] = WINPR_C_ARRAY_INIT;
451 int srcStrides[4] = WINPR_C_ARRAY_INIT;
453 if (!freerdp_video_fill_plane_info(srcPlanes, srcStrides, srcFormat, context->width,
454 context->height, (
const BYTE*)srcSampleData))
456 WLog_ERR(TAG,
"Failed to fill plane info");
460 const BYTE* cSrcPlanes[4] = { srcPlanes[0], srcPlanes[1], srcPlanes[2], srcPlanes[3] };
461 if (!freerdp_video_convert_to_yuv(context, cSrcPlanes, srcStrides, srcFormat, yuvData,
462 yuvLineSizes, yuvFormat, context->width,
465 WLog_ERR(TAG,
"YUV conversion failed");
470 BYTE* h264Data =
nullptr;
473 if (h264_compress(context->h264, &h264Data, &h264Size) < 0)
475 WLog_ERR(TAG,
"H264 compression failed");
479 if (!Stream_EnsureRemainingCapacity(output, h264Size))
481 WLog_ERR(TAG,
"Failed to ensure stream capacity");
485 Stream_Write(output, h264Data, h264Size);
488 else if (dstCompressed)
490 WLog_ERR(TAG,
"Only H264 encoding is supported");
494 WLog_ERR(TAG,
"Raw format output not yet implemented");
498static BOOL freerdp_video_decode_mjpeg(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData,
499 size_t srcSize, BYTE* dstData[4],
int dstLineSize[4],
500 FREERDP_VIDEO_FORMAT* dstFormat)
502#if defined(WITH_MJPEG_DECODER)
503 WINPR_ASSERT(context);
504 WINPR_ASSERT(srcData);
505 WINPR_ASSERT(dstData);
506 WINPR_ASSERT(dstLineSize);
507 WINPR_ASSERT(dstFormat);
509 if (!context->mjpegDecoder)
511 WLog_ERR(TAG,
"MJPEG decoder not initialized");
515 context->mjpegPacket->data = WINPR_CAST_CONST_PTR_AWAY(srcData, uint8_t*);
516 WINPR_ASSERT(srcSize <= INT32_MAX);
517 context->mjpegPacket->size = (int)srcSize;
519 if (avcodec_send_packet(context->mjpegDecoder, context->mjpegPacket) < 0)
521 WLog_ERR(TAG,
"avcodec_send_packet failed");
525 if (avcodec_receive_frame(context->mjpegDecoder, context->mjpegFrame) < 0)
527 WLog_ERR(TAG,
"avcodec_receive_frame failed");
532 for (
size_t i = 0; i < 4; i++)
534 dstData[i] = context->mjpegFrame->data[i];
535 dstLineSize[i] = context->mjpegFrame->linesize[i];
539 *dstFormat = av_format_to_video(context->mjpegDecoder->pix_fmt);
543 WINPR_UNUSED(context);
544 WINPR_UNUSED(srcData);
545 WINPR_UNUSED(srcSize);
546 WINPR_UNUSED(dstData);
547 WINPR_UNUSED(dstLineSize);
548 WINPR_UNUSED(dstFormat);
549 WLog_ERR(TAG,
"MJPEG decoder not available (requires direct FFmpeg linking)");
554static BOOL freerdp_video_convert_to_yuv(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData[4],
555 const int srcLineSize[4], FREERDP_VIDEO_FORMAT srcFormat,
556 BYTE* dstData[3],
const int dstLineSize[3],
557 FREERDP_VIDEO_FORMAT dstFormat, UINT32 width,
560 WINPR_ASSERT(srcData);
561 WINPR_ASSERT(srcLineSize);
562 WINPR_ASSERT(dstData);
563 WINPR_ASSERT(dstLineSize);
565 if (!freerdp_swscale_available())
567 WLog_ERR(TAG,
"swscale not available - install FFmpeg to enable video processing");
571 enum AVPixelFormat srcPixFmt = video_format_to_av(srcFormat);
572 enum AVPixelFormat dstPixFmt = video_format_to_av(dstFormat);
574 if (srcPixFmt == AV_PIX_FMT_NONE || dstPixFmt == AV_PIX_FMT_NONE)
576 WLog_ERR(TAG,
"Unsupported pixel format");
581 struct SwsContext* sws =
nullptr;
582 BOOL needFree = FALSE;
584 if (context && context->sws)
590 sws = freerdp_sws_getContext((
int)width, (
int)height, srcPixFmt, (
int)width, (
int)height,
591 dstPixFmt, 0,
nullptr,
nullptr,
nullptr);
594 WLog_ERR(TAG,
"sws_getContext failed");
605 const BYTE* cSrcData[4] = { srcData[0], srcData[1], srcData[2], srcData[3] };
608 uint8_t* localDstData[4] = { dstData[0], dstData[1], dstData[2],
nullptr };
609 int localDstLineSize[4] = { dstLineSize[0], dstLineSize[1], dstLineSize[2], 0 };
611 const int result = freerdp_sws_scale(sws, cSrcData, srcLineSize, 0, (
int)height, localDstData,
616 freerdp_sws_freeContext(sws);
622static BOOL freerdp_video_fill_plane_info(BYTE* data[4],
int lineSize[4],
623 FREERDP_VIDEO_FORMAT format, UINT32 width, UINT32 height,
627 WINPR_ASSERT(lineSize);
629 enum AVPixelFormat pixFmt = video_format_to_av(format);
630 if (pixFmt == AV_PIX_FMT_NONE)
632 WLog_ERR(TAG,
"Unsupported pixel format");
636 if (!freerdp_avutil_available())
638 WLog_ERR(TAG,
"avutil not available");
642 if (freerdp_av_image_fill_linesizes(lineSize, pixFmt, (
int)width) < 0)
644 WLog_ERR(TAG,
"av_image_fill_linesizes failed");
648 if (freerdp_av_image_fill_pointers(data, pixFmt, (
int)height,
649 WINPR_CAST_CONST_PTR_AWAY(buffer, BYTE*), lineSize) < 0)
651 WLog_ERR(TAG,
"av_image_fill_pointers failed");
658BOOL freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT srcFormat,
659 FREERDP_VIDEO_FORMAT dstFormat)
661 if (srcFormat == dstFormat)
664 if (!freerdp_video_available())
667 if (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG)
669#if !defined(WITH_MJPEG_DECODER)
673 else if (srcFormat == FREERDP_VIDEO_FORMAT_H264)
678 if (dstFormat == FREERDP_VIDEO_FORMAT_MJPEG)
682 else if (dstFormat == FREERDP_VIDEO_FORMAT_H264)
684#if !defined(WITH_OPENH264) && !defined(WITH_VIDEO_FFMPEG) && !defined(WITH_MEDIA_FOUNDATION) && \
685 !defined(WITH_MEDIACODEC)
697BOOL freerdp_video_available(
void)
702FREERDP_VIDEO_CONTEXT* freerdp_video_context_new(WINPR_ATTR_UNUSED UINT32 width,
703 WINPR_ATTR_UNUSED UINT32 height)
706 WINPR_UNUSED(height);
707 return calloc(1,
sizeof(
char));
710void freerdp_video_context_free(FREERDP_VIDEO_CONTEXT* context)
715BOOL freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT srcFormat,
716 FREERDP_VIDEO_FORMAT dstFormat)
718 if (srcFormat == dstFormat)
723BOOL freerdp_video_context_reconfigure(WINPR_ATTR_UNUSED FREERDP_VIDEO_CONTEXT* context,
724 WINPR_ATTR_UNUSED UINT32 width,
725 WINPR_ATTR_UNUSED UINT32 height,
726 WINPR_ATTR_UNUSED UINT32 framerate,
727 WINPR_ATTR_UNUSED UINT32 bitrate,
728 WINPR_ATTR_UNUSED UINT32 usageType)
733BOOL freerdp_video_sample_convert(WINPR_ATTR_UNUSED FREERDP_VIDEO_CONTEXT* context,
734 FREERDP_VIDEO_FORMAT srcFormat,
const void* srcSampleData,
735 size_t srcSampleLength, FREERDP_VIDEO_FORMAT dstFormat,
738 if (srcFormat == dstFormat)
740 if (!Stream_EnsureRemainingCapacity(output, srcSampleLength))
743 Stream_Write(output, srcSampleData, srcSampleLength);