FreeRDP
Loading...
Searching...
No Matches
h264.c
1
21#include <freerdp/config.h>
22
23#include <winpr/crt.h>
24#include <winpr/print.h>
25#include <winpr/library.h>
26#include <winpr/bitstream.h>
27#include <winpr/synch.h>
28
29#include <freerdp/primitives.h>
30#include <freerdp/codec/h264.h>
31#include <freerdp/codec/yuv.h>
32#include <freerdp/log.h>
33#include <freerdp/codec/region.h>
34
35#include "h264.h"
36
37#define TAG FREERDP_TAG("codec")
38
39WINPR_ATTR_NODISCARD
40static BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight);
41
42WINPR_ATTR_NODISCARD
43static BOOL yuv_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height)
44{
45 BOOL isNull = FALSE;
46 UINT32 pheight = height;
47
48 if (!h264)
49 return FALSE;
50
51 if (stride == 0)
52 {
53 const UINT32 pad = width % 16;
54 stride = width;
55 if (pad > 0)
56 width += 16 - pad;
57 }
58
59 /* Add padding lines. Allows relaxing bounds checks in decoder functions */
60 stride += 32 - stride % 16;
61 pheight += 32 - pheight % 16;
62
63 const size_t nPlanes = h264->hwAccel ? 2 : 3;
64
65 for (size_t x = 0; x < nPlanes; x++)
66 {
67 if (!h264->pYUVData[x] || !h264->pOldYUVData[x])
68 isNull = TRUE;
69 }
70
71 if (pheight == 0)
72 return FALSE;
73 if (stride == 0)
74 return FALSE;
75
76 if (isNull || (width != h264->width) || (height != h264->height) ||
77 (stride != h264->iStride[0]))
78 {
79 UINT32 iStride[3] = WINPR_C_ARRAY_INIT;
80 if (h264->hwAccel) /* NV12 */
81 {
82 iStride[0] = stride;
83 iStride[1] = stride;
84 iStride[2] = 0;
85 }
86 else /* I420 */
87 {
88 iStride[0] = stride;
89 iStride[1] = (stride + 1) / 2;
90 iStride[2] = (stride + 1) / 2;
91 }
92
93 for (size_t x = 0; x < nPlanes; x++)
94 {
95 BYTE* tmp1 = winpr_aligned_recalloc(h264->pYUVData[x], iStride[x], pheight, 16);
96 BYTE* tmp2 = winpr_aligned_recalloc(h264->pOldYUVData[x], iStride[x], pheight, 16);
97 if (tmp1)
98 h264->pYUVData[x] = tmp1;
99 if (tmp2)
100 h264->pOldYUVData[x] = tmp2;
101 if (!tmp1 || !tmp2)
102 return FALSE;
103 }
104 /* Only update the strides after the buffers were successfully allocated */
105 memcpy(h264->iStride, iStride, sizeof(h264->iStride));
106 h264->width = width;
107 h264->height = height;
108 }
109
110 return TRUE;
111}
112
113BOOL avc420_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height)
114{
115 return yuv_ensure_buffer(h264, stride, width, height);
116}
117
118WINPR_ATTR_NODISCARD
119static BOOL isRectValid(wLog* log, size_t pos, UINT32 width, UINT32 height,
120 const RECTANGLE_16* rect)
121{
122 WINPR_ASSERT(rect);
123 if ((rect->left > width) || (rect->right > width) || (rect->left >= rect->right) ||
124 (rect->top > height) || (rect->bottom > height) || (rect->top >= rect->bottom))
125 {
126 char buffer[64] = WINPR_C_ARRAY_INIT;
127 WLog_Print(log, WLOG_WARN,
128 "Rectangle %" PRIuz " %s outside of bounding frame %" PRIu32 "x%" PRIu32, pos,
129 rectangle_to_string(rect, buffer, sizeof(buffer)), width, height);
130 return FALSE;
131 }
132
133 return TRUE;
134}
135
136WINPR_ATTR_NODISCARD
137static BOOL areRectsValid(wLog* log, UINT32 width, UINT32 height, const RECTANGLE_16* rects,
138 UINT32 count)
139{
140 WINPR_ASSERT(rects || (count == 0));
141 for (size_t x = 0; x < count; x++)
142 {
143 const RECTANGLE_16* rect = &rects[x];
144 if (!isRectValid(log, x, width, height, rect))
145 return FALSE;
146 }
147 return TRUE;
148}
149
150WINPR_ATTR_NODISCARD
151static int log_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize,
152 const RECTANGLE_16* rects, UINT32 nrRects)
153{
154 WINPR_ASSERT(h264);
155 WINPR_ASSERT(h264->subsystem);
156 WINPR_ASSERT(h264->subsystem->Decompress);
157
158 const int status = h264->subsystem->Decompress(h264, pSrcData, SrcSize);
159 if (status < 0)
160 {
161 WLog_Print(h264->log, WLOG_WARN, "H264 decompress failed with %d", status);
162 return status;
163 }
164
165 /* no picture decoded, nothing to validate */
166 if (status == 0)
167 return 0;
168
169 /* some server implementations (krdc) use H264 frames smaller than the surface sizes,
170 * validate the regions against this size as well */
171 if (!areRectsValid(h264->log, h264->YUVWidth, h264->YUVHeight, rects, nrRects))
172 return -1015;
173 if (!areRectsValid(h264->log, h264->width, h264->height, rects, nrRects))
174 return -1016;
175 return status;
176}
177
178INT32 avc420_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize, BYTE* pDstData,
179 DWORD DstFormat, UINT32 nDstStep, WINPR_ATTR_UNUSED UINT32 nDstWidth,
180 WINPR_ATTR_UNUSED UINT32 nDstHeight, const RECTANGLE_16* regionRects,
181 UINT32 numRegionRects)
182{
183 int status = 0;
184 const BYTE* pYUVData[3];
185
186 if (!h264 || h264->Compressor)
187 return -1001;
188
189 if (!areRectsValid(h264->log, nDstWidth, nDstHeight, regionRects, numRegionRects))
190 return -1013;
191
192 status = log_decompress(h264, pSrcData, SrcSize, regionRects, numRegionRects);
193
194 if (status == 0)
195 return 1;
196
197 if (status < 0)
198 return status;
199
200 pYUVData[0] = h264->pYUVData[0];
201 pYUVData[1] = h264->pYUVData[1];
202 pYUVData[2] = h264->pYUVData[2];
203 if (!yuv420_context_decode(h264->yuv, pYUVData, h264->iStride, h264->YUVHeight, DstFormat,
204 pDstData, nDstStep, regionRects, numRegionRects))
205 return -1002;
206
207 return 1;
208}
209
210WINPR_ATTR_NODISCARD
211static BOOL allocate_h264_metablock(UINT32 QP, RECTANGLE_16* rectangles,
212 RDPGFX_H264_METABLOCK* meta, size_t count)
213{
214 /* [MS-RDPEGFX] 2.2.4.4.2 RDPGFX_AVC420_QUANT_QUALITY */
215 if (!meta || (QP > UINT8_MAX))
216 {
217 free(rectangles);
218 return FALSE;
219 }
220
221 meta->regionRects = rectangles;
222 if (count == 0)
223 return TRUE;
224
225 if (count > UINT32_MAX)
226 return FALSE;
227
228 meta->quantQualityVals = calloc(count, sizeof(RDPGFX_H264_QUANT_QUALITY));
229
230 if (!meta->quantQualityVals || !meta->regionRects)
231 return FALSE;
232 meta->numRegionRects = (UINT32)count;
233 for (size_t x = 0; x < count; x++)
234 {
235 RDPGFX_H264_QUANT_QUALITY* cur = &meta->quantQualityVals[x];
236 cur->qp = (UINT8)QP;
237
238 /* qpVal bit 6 and 7 are flags, so mask them out here.
239 * qualityVal is [0-100] so 100 - qpVal [0-64] is always in range */
240 cur->qualityVal = 100 - (QP & 0x3F);
241 }
242 return TRUE;
243}
244
245WINPR_ATTR_NODISCARD
246static inline BOOL diff_tile(const RECTANGLE_16* regionRect, BYTE* pYUVData[3],
247 BYTE* pOldYUVData[3], UINT32 const iStride[3])
248{
249 size_t size = 0;
250
251 if (!regionRect || !pYUVData || !pOldYUVData || !iStride)
252 return FALSE;
253 size = regionRect->right - regionRect->left;
254 if (regionRect->right > iStride[0])
255 return FALSE;
256 if (regionRect->right / 2u > iStride[1])
257 return FALSE;
258 if (regionRect->right / 2u > iStride[2])
259 return FALSE;
260
261 for (size_t y = regionRect->top; y < regionRect->bottom; y++)
262 {
263 const BYTE* cur0 = &pYUVData[0][y * iStride[0]];
264 const BYTE* old0 = &pOldYUVData[0][y * iStride[0]];
265
266 if (memcmp(&cur0[regionRect->left], &old0[regionRect->left], size) != 0)
267 return TRUE;
268 }
269
270 /* the U and V planes have half the rows: the tile's chroma is on rows top / 2 to
271 * (bottom + 1) / 2. Row y of them is screen row 2 * y, so reading row y here compared
272 * another part of the screen (and past the planes' content below half height). */
273 const size_t uvLeft = regionRect->left / 2u;
274 const size_t uvSize = (regionRect->right + 1u) / 2u - uvLeft;
275 for (size_t y = regionRect->top / 2u; y < (regionRect->bottom + 1u) / 2u; y++)
276 {
277 const BYTE* cur1 = &pYUVData[1][y * iStride[1]];
278 const BYTE* cur2 = &pYUVData[2][y * iStride[2]];
279 const BYTE* old1 = &pOldYUVData[1][y * iStride[1]];
280 const BYTE* old2 = &pOldYUVData[2][y * iStride[2]];
281
282 if (memcmp(&cur1[uvLeft], &old1[uvLeft], uvSize) != 0)
283 return TRUE;
284 if (memcmp(&cur2[uvLeft], &old2[uvLeft], uvSize) != 0)
285 return TRUE;
286 }
287 return FALSE;
288}
289
290/* The AVC444v2 auxiliary view keeps the chroma detail of screen columns [left, right) in two
291 * halves (MS-RDPEGFX 3.3.8.3.3): in its Y plane at [left / 2, right / 2) and, width / 2 further,
292 * at [width / 2 + left / 2, width / 2 + right / 2) on the same rows; in its U and V planes at a
293 * quarter of those columns on half the rows. Comparing the tile's own columns there, as
294 * diff_tile does, looks at other pixels, so a tile whose chroma changed could go unreported
295 * and the client kept its old colours. */
296WINPR_ATTR_NODISCARD
297static inline BOOL diff_tile_avc444v2_aux(const RECTANGLE_16* tile, UINT32 width, BYTE* pYUVData[3],
298 BYTE* pOldYUVData[3], UINT32 const iStride[3])
299{
300 if (!tile || !pYUVData || !pOldYUVData || !iStride || (tile->right <= tile->left))
301 return FALSE;
302
303 const size_t l2 = tile->left / 2u;
304 const size_t n2 = (tile->right + 1u) / 2u - l2;
305 const size_t h2 = width / 2u;
306 const size_t l4 = tile->left / 4u;
307 const size_t n4 = (tile->right + 3u) / 4u - l4;
308 const size_t h4 = width / 4u;
309
310 if ((h2 + l2 + n2 > iStride[0]) || (h4 + l4 + n4 > iStride[1]) || (h4 + l4 + n4 > iStride[2]))
311 return TRUE; /* can't tell: send it */
312
313 for (size_t y = tile->top; y < tile->bottom; y++)
314 {
315 const BYTE* cur = &pYUVData[0][y * iStride[0]];
316 const BYTE* old = &pOldYUVData[0][y * iStride[0]];
317 if ((memcmp(&cur[l2], &old[l2], n2) != 0) ||
318 (memcmp(&cur[h2 + l2], &old[h2 + l2], n2) != 0))
319 return TRUE;
320 }
321
322 for (size_t y = tile->top / 2u; y < (tile->bottom + 1u) / 2u; y++)
323 {
324 for (size_t p = 1; p < 3; p++)
325 {
326 const BYTE* cur = &pYUVData[p][y * iStride[p]];
327 const BYTE* old = &pOldYUVData[p][y * iStride[p]];
328 if ((memcmp(&cur[l4], &old[l4], n4) != 0) ||
329 (memcmp(&cur[h4 + l4], &old[h4 + l4], n4) != 0))
330 return TRUE;
331 }
332 }
333 return FALSE;
334}
335
336WINPR_ATTR_NODISCARD
337static BOOL detect_changes(BOOL firstFrameDone, const UINT32 QP, const RECTANGLE_16* regionRect,
338 BYTE* pYUVData[3], BYTE* pOldYUVData[3], UINT32 const iStride[3],
339 UINT32 v2AuxWidth, RDPGFX_H264_METABLOCK* meta)
340{
341 size_t count = 0;
342 size_t wc = 0;
343 size_t hc = 0;
344 RECTANGLE_16* rectangles = nullptr;
345
346 if (!regionRect || !pYUVData || !pOldYUVData || !iStride || !meta)
347 return FALSE;
348
349 wc = (regionRect->right - regionRect->left) / 64 + 1;
350 hc = (regionRect->bottom - regionRect->top) / 64 + 1;
351 rectangles = calloc(wc * hc, sizeof(RECTANGLE_16));
352 if (!rectangles)
353 return FALSE;
354 if (!firstFrameDone)
355 {
356 rectangles[0] = *regionRect;
357 count = 1;
358 }
359 else
360 {
361 for (size_t y = 0; y < regionRect->bottom - regionRect->top; y += 64)
362 {
363 for (size_t x = 0; x < regionRect->right - regionRect->left; x += 64)
364 {
365 RECTANGLE_16 rect;
366 rect.left = (UINT16)MIN(UINT16_MAX, regionRect->left + x);
367 rect.top = (UINT16)MIN(UINT16_MAX, regionRect->top + y);
368 rect.right =
369 (UINT16)MIN(UINT16_MAX, MIN(regionRect->left + x + 64, regionRect->right));
370 rect.bottom =
371 (UINT16)MIN(UINT16_MAX, MIN(regionRect->top + y + 64, regionRect->bottom));
372 const BOOL changed =
373 v2AuxWidth
374 ? diff_tile_avc444v2_aux(&rect, v2AuxWidth, pYUVData, pOldYUVData, iStride)
375 : diff_tile(&rect, pYUVData, pOldYUVData, iStride);
376 if (changed)
377 rectangles[count++] = rect;
378 }
379 }
380 }
381 if (!allocate_h264_metablock(QP, rectangles, meta, count))
382 return FALSE;
383 return TRUE;
384}
385
386INT32 h264_get_yuv_buffer(H264_CONTEXT* h264, UINT32 nSrcStride, UINT32 nSrcWidth,
387 UINT32 nSrcHeight, BYTE* YUVData[3], UINT32 stride[3])
388{
389 if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress)
390 return -1;
391
392 if (!yuv_ensure_buffer(h264, nSrcStride, nSrcWidth, nSrcHeight))
393 return -1;
394
395 for (size_t x = 0; x < 3; x++)
396 {
397 YUVData[x] = h264->pYUVData[x];
398 stride[x] = h264->iStride[x];
399 }
400
401 return 0;
402}
403
404INT32 h264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize)
405{
406 if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress)
407 return -1;
408
409 const BYTE* pcYUVData[3] = { h264->pYUVData[0], h264->pYUVData[1], h264->pYUVData[2] };
410
411 return h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize);
412}
413
414/* The region to encode, rounded out to the 16 pixel macroblock grid inside the frame.
415 * Clients rebuild AVC444 colours relative to each region's corner: the odd chroma columns and
416 * rows, and every fourth sample in AVC444v2 (MS-RDPEGFX 3.3.8.3.2, 3.3.8.3.3), are counted from
417 * left and top. A region starting at an odd or unaligned column or row (the shadow server sends
418 * the bounding box of what changed, at any pixel) put those samples one pixel off. Encoders work
419 * on 16x16 macroblocks anyway, so the wider region costs no bits. */
420static RECTANGLE_16 h264_align_region(const RECTANGLE_16* rect, UINT32 width, UINT32 height)
421{
422 WINPR_ASSERT(rect);
423 const UINT32 right = MIN(width, ((UINT32)rect->right + 15u) & ~15u);
424 const UINT32 bottom = MIN(height, ((UINT32)rect->bottom + 15u) & ~15u);
425 const RECTANGLE_16 aligned = { (UINT16)(rect->left & ~15u), (UINT16)(rect->top & ~15u),
426 (UINT16)MAX(rect->right, right),
427 (UINT16)MAX(rect->bottom, bottom) };
428 return aligned;
429}
430
431/* The encoder alternates between two sets of planes: the one it converts into this frame held
432 * the frame before last, and only the region is converted. Outside the region the encoder then
433 * saw that older picture, a change every frame where nothing changed, which costs bits (and
434 * takes them from what did change). Start from the last frame, unless the region is all of it. */
435static void h264_keep_last_frame(const RECTANGLE_16* region, UINT32 width, UINT32 height,
436 BYTE* dst[3], BYTE* const src[3], const size_t size[3])
437{
438 WINPR_ASSERT(region);
439 if ((region->left == 0) && (region->top == 0) && (region->right >= width) &&
440 (region->bottom >= height))
441 return;
442
443 for (size_t x = 0; x < 3; x++)
444 {
445 if (dst[x] && src[x] && (size[x] > 0))
446 memcpy(dst[x], src[x], size[x]);
447 }
448}
449
450/* the size of the 4:2:0 planes, as yuv_ensure_buffer allocates them */
451static void h264_yuv420_plane_sizes(const H264_CONTEXT* h264, size_t size[3])
452{
453 const size_t rows = h264->height + 32u - h264->height % 16u;
454 for (size_t x = 0; x < 3; x++)
455 size[x] = 1ull * h264->iStride[x] * rows;
456}
457
458INT32 avc420_compress(H264_CONTEXT* h264, const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep,
459 UINT32 nSrcWidth, UINT32 nSrcHeight, const RECTANGLE_16* regionRect,
460 BYTE** ppDstData, UINT32* pDstSize, RDPGFX_H264_METABLOCK* meta)
461{
462 INT32 rc = -1;
463 BYTE* pYUVData[3] = WINPR_C_ARRAY_INIT;
464 const BYTE* pcYUVData[3] = WINPR_C_ARRAY_INIT;
465 BYTE* pOldYUVData[3] = WINPR_C_ARRAY_INIT;
466
467 if (!h264 || !regionRect || !meta || !h264->Compressor)
468 return -1;
469
470 if (!h264->subsystem->Compress)
471 return -1;
472
473 if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight))
474 return -1;
475
476 if (h264->encodingBuffer)
477 {
478 for (size_t x = 0; x < 3; x++)
479 {
480 pYUVData[x] = h264->pYUVData[x];
481 pOldYUVData[x] = h264->pOldYUVData[x];
482 }
483 }
484 else
485 {
486 for (size_t x = 0; x < 3; x++)
487 {
488 pYUVData[x] = h264->pOldYUVData[x];
489 pOldYUVData[x] = h264->pYUVData[x];
490 }
491 }
492 h264->encodingBuffer = !h264->encodingBuffer;
493 {
494 size_t size420[3] = WINPR_C_ARRAY_INIT;
495 h264_yuv420_plane_sizes(h264, size420);
496 h264_keep_last_frame(regionRect, nSrcWidth, nSrcHeight, pYUVData, pOldYUVData, size420);
497 }
498
499 const RECTANGLE_16 aligned = h264_align_region(regionRect, nSrcWidth, nSrcHeight);
500 if (!yuv420_context_encode(h264->yuv, pSrcData, nSrcStep, SrcFormat, h264->iStride, pYUVData,
501 &aligned, 1))
502 goto fail;
503
504 if (!detect_changes(h264->firstLumaFrameDone, h264->QP, &aligned, pYUVData, pOldYUVData,
505 h264->iStride, 0, meta))
506 goto fail;
507
508 if (meta->numRegionRects == 0)
509 {
510 rc = 0;
511 goto fail;
512 }
513
514 for (size_t x = 0; x < 3; x++)
515 pcYUVData[x] = pYUVData[x];
516
517 rc = h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize);
518 if (rc >= 0)
519 h264->firstLumaFrameDone = TRUE;
520
521fail:
522 if (rc < 0)
523 free_h264_metablock(meta);
524 return rc;
525}
526
527INT32 avc444_compress(H264_CONTEXT* h264, const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep,
528 UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE version, const RECTANGLE_16* region,
529 BYTE* op, BYTE** ppDstData, UINT32* pDstSize, BYTE** ppAuxDstData,
530 UINT32* pAuxDstSize, RDPGFX_H264_METABLOCK* meta,
531 RDPGFX_H264_METABLOCK* auxMeta)
532{
533 int rc = -1;
534 BYTE* coded = nullptr;
535 UINT32 codedSize = 0;
536 BYTE** pYUV444Data = nullptr;
537 BYTE** pOldYUV444Data = nullptr;
538 BYTE** pYUVData = nullptr;
539 BYTE** pOldYUVData = nullptr;
540
541 if (!h264 || !h264->Compressor)
542 return -1;
543
544 if (!h264->subsystem->Compress)
545 return -1;
546
547 if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight))
548 return -1;
549
550 if (!avc444_ensure_buffer(h264, nSrcHeight))
551 return -1;
552
553 if (h264->encodingBuffer)
554 {
555 pYUV444Data = h264->pOldYUV444Data;
556 pOldYUV444Data = h264->pYUV444Data;
557 pYUVData = h264->pOldYUVData;
558 pOldYUVData = h264->pYUVData;
559 }
560 else
561 {
562 pYUV444Data = h264->pYUV444Data;
563 pOldYUV444Data = h264->pOldYUV444Data;
564 pYUVData = h264->pYUVData;
565 pOldYUVData = h264->pOldYUVData;
566 }
567 h264->encodingBuffer = !h264->encodingBuffer;
568 {
569 size_t size420[3] = WINPR_C_ARRAY_INIT;
570 size_t size444[3] = WINPR_C_ARRAY_INIT;
571 h264_yuv420_plane_sizes(h264, size420);
572 for (size_t x = 0; x < 3; x++)
573 size444[x] = h264->iYUV444Size[x];
574 h264_keep_last_frame(region, nSrcWidth, nSrcHeight, pYUVData, pOldYUVData, size420);
575 h264_keep_last_frame(region, nSrcWidth, nSrcHeight, pYUV444Data, pOldYUV444Data, size444);
576 }
577
578 const RECTANGLE_16 aligned = h264_align_region(region, nSrcWidth, nSrcHeight);
579 if (!yuv444_context_encode(h264->yuv, version, pSrcData, nSrcStep, SrcFormat, h264->iStride,
580 pYUV444Data, pYUVData, &aligned, 1))
581 goto fail;
582
583 if (!detect_changes(h264->firstLumaFrameDone, h264->QP, &aligned, pYUV444Data, pOldYUV444Data,
584 h264->iStride, 0, meta))
585 goto fail;
586 /* in AVC444v2 the auxiliary view is laid out by the frame width, not by screen position */
587 if (!detect_changes(h264->firstChromaFrameDone, h264->QP, &aligned, pYUVData, pOldYUVData,
588 h264->iStride, (version == 2) ? h264->width : 0, auxMeta))
589 goto fail;
590
591 /* [MS-RDPEGFX] 2.2.4.5 RFX_AVC444_BITMAP_STREAM
592 * LC:
593 * 0 ... Luma & Chroma
594 * 1 ... Luma
595 * 2 ... Chroma
596 */
597 if ((meta->numRegionRects > 0) && (auxMeta->numRegionRects > 0))
598 *op = 0;
599 else if (meta->numRegionRects > 0)
600 *op = 1;
601 else if (auxMeta->numRegionRects > 0)
602 *op = 2;
603 else
604 {
605 WLog_Print(h264->log, WLOG_TRACE, "no changes detected for luma or chroma frame");
606 rc = 0;
607 goto fail;
608 }
609
610 if ((*op == 0) || (*op == 1))
611 {
612 const BYTE* pcYUV444Data[3] = { pYUV444Data[0], pYUV444Data[1], pYUV444Data[2] };
613
614 if (h264->subsystem->Compress(h264, pcYUV444Data, h264->iStride, &coded, &codedSize) < 0)
615 goto fail;
616 h264->firstLumaFrameDone = TRUE;
617 memcpy(h264->lumaData, coded, codedSize);
618 *ppDstData = h264->lumaData;
619 *pDstSize = codedSize;
620 }
621
622 if ((*op == 0) || (*op == 2))
623 {
624 const BYTE* pcYUVData[3] = { pYUVData[0], pYUVData[1], pYUVData[2] };
625
626 if (h264->subsystem->Compress(h264, pcYUVData, h264->iStride, &coded, &codedSize) < 0)
627 goto fail;
628 h264->firstChromaFrameDone = TRUE;
629 *ppAuxDstData = coded;
630 *pAuxDstSize = codedSize;
631 }
632
633 rc = 1;
634fail:
635 if (rc < 0)
636 {
637 free_h264_metablock(meta);
638 free_h264_metablock(auxMeta);
639 }
640 return rc;
641}
642
643BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight)
644{
645 WINPR_ASSERT(h264);
646
647 const UINT32* piMainStride = h264->iStride;
648 UINT32* piDstSize = h264->iYUV444Size;
649 UINT32* piDstStride = h264->iYUV444Stride;
650 BYTE** ppYUVDstData = h264->pYUV444Data;
651 BYTE** ppOldYUVDstData = h264->pOldYUV444Data;
652
653 nDstHeight = MAX(h264->height, nDstHeight);
654 const UINT32 pad = nDstHeight % 16;
655 UINT32 padDstHeight = nDstHeight; /* Need alignment to 16x16 blocks */
656
657 if (pad != 0)
658 padDstHeight += 16 - pad;
659
660 /* Add padding lines. Allows relaxing bounds checks in decoder functions */
661 padDstHeight += 16;
662
663 if ((piMainStride[0] == 0) || (padDstHeight == 0))
664 return FALSE;
665
666 const uint64_t dstsize = 1ull * piMainStride[0] * padDstHeight;
667 if (dstsize > UINT32_MAX)
668 return FALSE;
669
670 if ((piMainStride[0] != piDstStride[0]) || (piDstSize[0] != dstsize))
671 {
672 const UINT32 size = WINPR_ASSERTING_INT_CAST(UINT32, dstsize);
673
674 for (UINT32 x = 0; x < 3; x++)
675 {
676 BYTE* tmp1 = winpr_aligned_recalloc(ppYUVDstData[x], size, 1, 16);
677 if (!tmp1)
678 return FALSE;
679 ppYUVDstData[x] = tmp1;
680 BYTE* tmp2 = winpr_aligned_recalloc(ppOldYUVDstData[x], size, 1, 16);
681 if (!tmp2)
682 return FALSE;
683 ppOldYUVDstData[x] = tmp2;
684 }
685
686 {
687 BYTE* tmp = winpr_aligned_recalloc(h264->lumaData, size, 4, 16);
688 if (!tmp)
689 goto fail;
690 h264->lumaData = tmp;
691 }
692
693 /* Only update stride and size after all buffers were successfully allocated,
694 * otherwise the next call would skip the reallocation and use too small planes */
695 for (UINT32 x = 0; x < 3; x++)
696 {
697 piDstStride[x] = piMainStride[0];
698 piDstSize[x] = size;
699 }
700 }
701
702 for (UINT32 x = 0; x < 3; x++)
703 {
704 if (!ppOldYUVDstData[x] || !ppYUVDstData[x] || (piDstSize[x] == 0) || (piDstStride[x] == 0))
705 {
706 WLog_Print(h264->log, WLOG_ERROR,
707 "YUV buffer not initialized! check your decoder settings");
708 goto fail;
709 }
710 }
711
712 if (!h264->lumaData)
713 goto fail;
714
715 return TRUE;
716fail:
717 return FALSE;
718}
719
720WINPR_ATTR_NODISCARD
721static BOOL avc444_process_rects(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize,
722 BYTE* pDstData, UINT32 DstFormat, UINT32 nDstStep,
723 WINPR_ATTR_UNUSED UINT32 nDstWidth, UINT32 nDstHeight,
724 const RECTANGLE_16* rects, UINT32 nrRects, avc444_frame_type type)
725{
726 const BYTE* pYUVData[3] = WINPR_C_ARRAY_INIT;
727 BYTE* pYUVDstData[3] = WINPR_C_ARRAY_INIT;
728 UINT32* piDstStride = h264->iYUV444Stride;
729 BYTE** ppYUVDstData = h264->pYUV444Data;
730 const UINT32* piStride = h264->iStride;
731
732 const int status = log_decompress(h264, pSrcData, SrcSize, rects, nrRects);
733 if (status < 0)
734 return FALSE;
735 if (status == 0)
736 return TRUE;
737
738 pYUVData[0] = h264->pYUVData[0];
739 pYUVData[1] = h264->pYUVData[1];
740 pYUVData[2] = h264->pYUVData[2];
741 if (!avc444_ensure_buffer(h264, nDstHeight))
742 return FALSE;
743
744 pYUVDstData[0] = ppYUVDstData[0];
745 pYUVDstData[1] = ppYUVDstData[1];
746 pYUVDstData[2] = ppYUVDstData[2];
747 return (yuv444_context_decode(h264->yuv, (BYTE)type, pYUVData, piStride, h264->YUVHeight,
748 pYUVDstData, piDstStride, DstFormat, pDstData, nDstStep, rects,
749 nrRects));
750}
751
752#if defined(AVC444_FRAME_STAT)
753static UINT64 op1 = 0;
754static double op1sum = 0;
755static UINT64 op2 = 0;
756static double op2sum = 0;
757static UINT64 op3 = 0;
758static double op3sum = 0;
759static double avg(UINT64* count, double old, double size)
760{
761 double tmp = size + *count * old;
762 (*count)++;
763 tmp = tmp / *count;
764 return tmp;
765}
766#endif
767
768INT32 avc444_decompress(H264_CONTEXT* h264, BYTE op, const RECTANGLE_16* regionRects,
769 UINT32 numRegionRects, const BYTE* pSrcData, UINT32 SrcSize,
770 const RECTANGLE_16* auxRegionRects, UINT32 numAuxRegionRect,
771 const BYTE* pAuxSrcData, UINT32 AuxSrcSize, BYTE* pDstData, DWORD DstFormat,
772 UINT32 nDstStep, UINT32 nDstWidth, UINT32 nDstHeight, UINT32 codecId)
773{
774 INT32 status = -1;
775 avc444_frame_type chroma =
776 (codecId == RDPGFX_CODECID_AVC444) ? AVC444_CHROMAv1 : AVC444_CHROMAv2;
777
778 if (!h264 || !regionRects || !pSrcData || !pDstData || h264->Compressor)
779 return -1001;
780
781 if (!areRectsValid(h264->log, nDstWidth, nDstHeight, regionRects, numRegionRects))
782 return -1013;
783 if (!areRectsValid(h264->log, nDstWidth, nDstHeight, auxRegionRects, numAuxRegionRect))
784 return -1014;
785
786 switch (op)
787 {
788 case 0: /* YUV420 in stream 1
789 * Chroma420 in stream 2 */
790 if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
791 nDstWidth, nDstHeight, regionRects, numRegionRects,
792 AVC444_LUMA))
793 status = -1;
794 else if (!avc444_process_rects(h264, pAuxSrcData, AuxSrcSize, pDstData, DstFormat,
795 nDstStep, nDstWidth, nDstHeight, auxRegionRects,
796 numAuxRegionRect, chroma))
797 status = -1;
798 else
799 status = 0;
800
801 break;
802
803 case 2: /* Chroma420 in stream 1 */
804 if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
805 nDstWidth, nDstHeight, regionRects, numRegionRects, chroma))
806 status = -1;
807 else
808 status = 0;
809
810 break;
811
812 case 1: /* YUV420 in stream 1 */
813 if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
814 nDstWidth, nDstHeight, regionRects, numRegionRects,
815 AVC444_LUMA))
816 status = -1;
817 else
818 status = 0;
819
820 break;
821
822 default: /* WTF? */
823 break;
824 }
825
826#if defined(AVC444_FRAME_STAT)
827
828 switch (op)
829 {
830 case 0:
831 op1sum = avg(&op1, op1sum, SrcSize + AuxSrcSize);
832 break;
833
834 case 1:
835 op2sum = avg(&op2, op2sum, SrcSize);
836 break;
837
838 case 2:
839 op3sum = avg(&op3, op3sum, SrcSize);
840 break;
841
842 default:
843 break;
844 }
845
846 WLog_Print(h264->log, WLOG_INFO,
847 "luma=%" PRIu64 " [avg=%lf] chroma=%" PRIu64 " [avg=%lf] combined=%" PRIu64
848 " [avg=%lf]",
849 op1, op1sum, op2, op2sum, op3, op3sum);
850#endif
851 return status;
852}
853
854#define MAX_SUBSYSTEMS 10
855static INIT_ONCE subsystems_once = INIT_ONCE_STATIC_INIT;
856static const H264_CONTEXT_SUBSYSTEM* subSystems[MAX_SUBSYSTEMS] = WINPR_C_ARRAY_INIT;
857
858WINPR_ATTR_NODISCARD
859static BOOL CALLBACK h264_register_subsystems(WINPR_ATTR_UNUSED PINIT_ONCE once,
860 WINPR_ATTR_UNUSED PVOID param,
861 WINPR_ATTR_UNUSED PVOID* context)
862{
863 int i = 0;
864
865#ifdef WITH_MEDIACODEC
866 {
867 subSystems[i] = &g_Subsystem_mediacodec;
868 i++;
869 }
870#endif
871#if defined(_WIN32) && defined(WITH_MEDIA_FOUNDATION)
872 {
873 subSystems[i] = &g_Subsystem_MF;
874 i++;
875 }
876#endif
877#ifdef WITH_APPLE_VIDEOTOOLBOX
878 {
879 subSystems[i] = &g_Subsystem_VideoToolbox;
880 i++;
881 }
882#endif
883#ifdef WITH_OPENH264
884 {
885 subSystems[i] = &g_Subsystem_OpenH264;
886 i++;
887 }
888#endif
889#ifdef WITH_VIDEO_FFMPEG
890 {
891 subSystems[i] = &g_Subsystem_libavcodec;
892 i++;
893 }
894#endif
895#ifdef WITH_H264_WEBCODECS
896 {
897 subSystems[i] = &g_Subsystem_WebCodecs;
898 i++;
899 }
900#endif
901 return i > 0;
902}
903
904WINPR_ATTR_NODISCARD
905static BOOL h264_context_init(H264_CONTEXT* h264)
906{
907 if (!h264)
908 return FALSE;
909
910 h264->subsystem = nullptr;
911 if (!InitOnceExecuteOnce(&subsystems_once, h264_register_subsystems, nullptr, nullptr))
912 return FALSE;
913
914 for (size_t i = 0; i < MAX_SUBSYSTEMS; i++)
915 {
916 const H264_CONTEXT_SUBSYSTEM* subsystem = subSystems[i];
917
918 if (!subsystem || !subsystem->Init)
919 break;
920
921 if (subsystem->Init(h264))
922 {
923 h264->subsystem = subsystem;
924 return TRUE;
925 }
926 }
927
928 return FALSE;
929}
930
931static void h264_buffers_free(H264_CONTEXT* h264)
932{
933 WINPR_ASSERT(h264);
934 for (size_t x = 0; x < 3; x++)
935 {
936 if (h264->Compressor)
937 {
938 winpr_aligned_free(h264->pYUVData[x]);
939 winpr_aligned_free(h264->pOldYUVData[x]);
940 }
941 winpr_aligned_free(h264->pYUV444Data[x]);
942 winpr_aligned_free(h264->pOldYUV444Data[x]);
943
944 h264->pYUVData[x] = nullptr;
945 h264->pOldYUVData[x] = nullptr;
946 h264->pYUV444Data[x] = nullptr;
947 h264->pOldYUV444Data[x] = nullptr;
948 }
949 winpr_aligned_free(h264->lumaData);
950 h264->lumaData = nullptr;
951}
952
953BOOL h264_context_reset(H264_CONTEXT* h264, UINT32 width, UINT32 height)
954{
955 if (!h264)
956 return FALSE;
957
958 if (width > UINT16_MAX)
959 {
960 WLog_Print(h264->log, WLOG_ERROR, "h264::width %" PRIu32 " > %d, aborting", width,
961 UINT16_MAX);
962 return FALSE;
963 }
964 if (height > UINT16_MAX)
965 {
966 WLog_Print(h264->log, WLOG_ERROR, "h264::height %" PRIu32 " > %d, aborting", height,
967 UINT16_MAX);
968 return FALSE;
969 }
970
971 if (h264->subsystem && h264->subsystem->Uninit)
972 h264->subsystem->Uninit(h264);
973
974 h264_buffers_free(h264);
975 h264->width = width;
976 h264->height = height;
977
978 if (!h264_context_init(h264))
979 return FALSE;
980
981 return yuv_context_reset(h264->yuv, width, height);
982}
983
984H264_CONTEXT* h264_context_new(BOOL Compressor)
985{
986 H264_CONTEXT* h264 = (H264_CONTEXT*)calloc(1, sizeof(H264_CONTEXT));
987 if (!h264)
988 return nullptr;
989
990 h264->log = WLog_Get(TAG);
991
992 if (!h264->log)
993 goto fail;
994
995 h264->Compressor = Compressor;
996 if (Compressor)
997 {
998 /* Default compressor settings, may be changed by caller */
999 h264->BitRate = 1000000;
1000 h264->FrameRate = 30;
1001 }
1002
1003 if (!h264_context_init(h264))
1004 goto fail;
1005
1006 h264->yuv = yuv_context_new(Compressor, 0);
1007 if (!h264->yuv)
1008 goto fail;
1009
1010 return h264;
1011
1012fail:
1013 WINPR_PRAGMA_DIAG_PUSH
1014 WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1015 h264_context_free(h264);
1016 WINPR_PRAGMA_DIAG_POP
1017 return nullptr;
1018}
1019
1020void h264_context_free(H264_CONTEXT* h264)
1021{
1022 if (!h264)
1023 return;
1024
1025 if (h264->subsystem)
1026 {
1027 WINPR_ASSERT(h264->subsystem->Uninit);
1028 h264->subsystem->Uninit(h264);
1029 }
1030
1031 h264_buffers_free(h264);
1032
1033 yuv_context_free(h264->yuv);
1034 free(h264);
1035}
1036
1037void free_h264_metablock(RDPGFX_H264_METABLOCK* meta)
1038{
1039 RDPGFX_H264_METABLOCK m = WINPR_C_ARRAY_INIT;
1040 if (!meta)
1041 return;
1042 free(meta->quantQualityVals);
1043 free(meta->regionRects);
1044 *meta = m;
1045}
1046
1047BOOL h264_context_set_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option, UINT32 value)
1048{
1049 WINPR_ASSERT(h264);
1050 switch (option)
1051 {
1052 case H264_CONTEXT_OPTION_BITRATE:
1053 h264->BitRate = value;
1054 return TRUE;
1055 case H264_CONTEXT_OPTION_FRAMERATE:
1056 h264->FrameRate = value;
1057 return TRUE;
1058 case H264_CONTEXT_OPTION_RATECONTROL:
1059 {
1060 switch (value)
1061 {
1062 case H264_RATECONTROL_VBR:
1063 h264->RateControlMode = H264_RATECONTROL_VBR;
1064 return TRUE;
1065 case H264_RATECONTROL_CQP:
1066 h264->RateControlMode = H264_RATECONTROL_CQP;
1067 return TRUE;
1068 default:
1069 WLog_Print(h264->log, WLOG_WARN,
1070 "Unknown H264_CONTEXT_OPTION_RATECONTROL value [0x%08" PRIx32 "]",
1071 value);
1072 return FALSE;
1073 }
1074 return TRUE;
1075 }
1076 case H264_CONTEXT_OPTION_QP:
1077 h264->QP = value;
1078 return TRUE;
1079 case H264_CONTEXT_OPTION_USAGETYPE:
1080 h264->UsageType = value;
1081 return TRUE;
1082 case H264_CONTEXT_OPTION_HW_ACCEL:
1083#ifdef WITH_APPLE_VIDEOTOOLBOX
1084 /* VideoToolbox follows Apple's decoder-selection policy;
1085 * ignore this option without reinitializing the decoder. */
1086 if (h264->subsystem == &g_Subsystem_VideoToolbox)
1087 return TRUE;
1088#endif
1089 if (h264->hwAccel == value)
1090 return TRUE;
1091 h264->hwAccel = (value);
1092 IFCALL(h264->subsystem->Uninit, h264);
1093 return IFCALLRESULT(TRUE, h264->subsystem->Init, h264);
1094 default:
1095 WLog_Print(h264->log, WLOG_WARN, "Unknown H264_CONTEXT_OPTION[0x%08" PRIx32 "]",
1096 option);
1097 return FALSE;
1098 }
1099}
1100
1101UINT32 h264_context_get_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option)
1102{
1103 WINPR_ASSERT(h264);
1104 switch (option)
1105 {
1106 case H264_CONTEXT_OPTION_BITRATE:
1107 return h264->BitRate;
1108 case H264_CONTEXT_OPTION_FRAMERATE:
1109 return h264->FrameRate;
1110 case H264_CONTEXT_OPTION_RATECONTROL:
1111 return h264->RateControlMode;
1112 case H264_CONTEXT_OPTION_QP:
1113 return h264->QP;
1114 case H264_CONTEXT_OPTION_USAGETYPE:
1115 return h264->UsageType;
1116 case H264_CONTEXT_OPTION_HW_ACCEL:
1117 return h264->hwAccel;
1118 default:
1119 WLog_Print(h264->log, WLOG_WARN, "Unknown H264_CONTEXT_OPTION[0x%08" PRIx32 "]",
1120 option);
1121 return 0;
1122 }
1123}
RDPGFX_H264_METABLOCK
Definition channels/rdpgfx.h:434
RDPGFX_H264_QUANT_QUALITY
Definition channels/rdpgfx.h:424
RECTANGLE_16
Definition types.h:109
RTL_RUN_ONCE
Definition include/winpr/synch.h:388