FreeRDP
Loading...
Searching...
No Matches
planar.c
1
22#include <freerdp/config.h>
23
24#include <winpr/crt.h>
25#include <winpr/wtypes.h>
26#include <winpr/assert.h>
27#include <winpr/cast.h>
28#include <winpr/print.h>
29
30#include <freerdp/primitives.h>
31#include <freerdp/log.h>
32#include <freerdp/codec/bitmap.h>
33#include <freerdp/codec/planar.h>
34
35#define TAG FREERDP_TAG("codec")
36
37#define PLANAR_ALIGN(val, align) \
38 ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
39
40typedef struct
41{
47 BYTE controlByte;
48 BYTE* rawValues;
49} RDP6_RLE_SEGMENT;
50
51typedef struct
52{
53 UINT32 cSegments;
54 RDP6_RLE_SEGMENT* segments;
55} RDP6_RLE_SEGMENTS;
56
57typedef struct
58{
67 BYTE formatHeader;
68} RDP6_BITMAP_STREAM;
69
70struct S_BITMAP_PLANAR_CONTEXT
71{
72 UINT32 maxWidth;
73 UINT32 maxHeight;
74 UINT32 maxPlaneSize;
75
76 BOOL AllowSkipAlpha;
77 BOOL AllowRunLengthEncoding;
78 BOOL AllowColorSubsampling;
79 BOOL AllowDynamicColorFidelity;
80
81 UINT32 ColorLossLevel;
82
83 BYTE* planes[4];
84 BYTE* planesBuffer;
85
86 BYTE* deltaPlanes[4];
87 BYTE* deltaPlanesBuffer;
88
89 BYTE* rlePlanes[4];
90 BYTE* rlePlanesBuffer;
91
92 BYTE* pTempData;
93 UINT32 nTempStep;
94
95 BOOL bgr;
96 BOOL topdown;
97};
98
99static inline BYTE PLANAR_CONTROL_BYTE(UINT32 nRunLength, UINT32 cRawBytes)
100{
101 return WINPR_ASSERTING_INT_CAST(UINT8, ((nRunLength & 0x0F) | ((cRawBytes & 0x0F) << 4)));
102}
103
104static inline BYTE PLANAR_CONTROL_BYTE_RUN_LENGTH(UINT32 controlByte)
105{
106 return (controlByte & 0x0F);
107}
108static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte)
109{
110 return ((controlByte >> 4) & 0x0F);
111}
112
113static inline UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
114 UINT32 DstFormat)
115{
116 WINPR_ASSERT(planar);
117 if (planar->bgr && alpha)
118 {
119 switch (DstFormat)
120 {
121 case PIXEL_FORMAT_ARGB32:
122 DstFormat = PIXEL_FORMAT_ABGR32;
123 break;
124 case PIXEL_FORMAT_XRGB32:
125 DstFormat = PIXEL_FORMAT_XBGR32;
126 break;
127 case PIXEL_FORMAT_ABGR32:
128 DstFormat = PIXEL_FORMAT_ARGB32;
129 break;
130 case PIXEL_FORMAT_XBGR32:
131 DstFormat = PIXEL_FORMAT_XRGB32;
132 break;
133 case PIXEL_FORMAT_BGRA32:
134 DstFormat = PIXEL_FORMAT_RGBA32;
135 break;
136 case PIXEL_FORMAT_BGRX32:
137 DstFormat = PIXEL_FORMAT_RGBX32;
138 break;
139 case PIXEL_FORMAT_RGBA32:
140 DstFormat = PIXEL_FORMAT_BGRA32;
141 break;
142 case PIXEL_FORMAT_RGBX32:
143 DstFormat = PIXEL_FORMAT_BGRX32;
144 break;
145 case PIXEL_FORMAT_RGB24:
146 DstFormat = PIXEL_FORMAT_BGR24;
147 break;
148 case PIXEL_FORMAT_BGR24:
149 DstFormat = PIXEL_FORMAT_RGB24;
150 break;
151 case PIXEL_FORMAT_RGB16:
152 DstFormat = PIXEL_FORMAT_BGR16;
153 break;
154 case PIXEL_FORMAT_BGR16:
155 DstFormat = PIXEL_FORMAT_RGB16;
156 break;
157 case PIXEL_FORMAT_ARGB15:
158 DstFormat = PIXEL_FORMAT_ABGR15;
159 break;
160 case PIXEL_FORMAT_RGB15:
161 DstFormat = PIXEL_FORMAT_BGR15;
162 break;
163 case PIXEL_FORMAT_ABGR15:
164 DstFormat = PIXEL_FORMAT_ARGB15;
165 break;
166 case PIXEL_FORMAT_BGR15:
167 DstFormat = PIXEL_FORMAT_RGB15;
168 break;
169 default:
170 break;
171 }
172 }
173 return DstFormat;
174}
175
176static inline BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
177 UINT32 width, UINT32 height,
178 BYTE* WINPR_RESTRICT outPlane,
179 UINT32* WINPR_RESTRICT dstSize);
180static inline BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
181 UINT32 width, UINT32 height,
182 BYTE* WINPR_RESTRICT outPlane);
183
184static inline INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
185 UINT32 nWidth, UINT32 nHeight)
186{
187 UINT32 used = 0;
188
189 WINPR_ASSERT(pSrcData);
190
191 for (UINT32 y = 0; y < nHeight; y++)
192 {
193 for (UINT32 x = 0; x < nWidth;)
194 {
195 if (used >= SrcSize)
196 {
197 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
198 SrcSize);
199 return -1;
200 }
201
202 const uint8_t controlByte = pSrcData[used++];
203 uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
204 uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
205
206 if (nRunLength == 1)
207 {
208 nRunLength = cRawBytes + 16;
209 cRawBytes = 0;
210 }
211 else if (nRunLength == 2)
212 {
213 nRunLength = cRawBytes + 32;
214 cRawBytes = 0;
215 }
216
217 used += cRawBytes;
218 x += cRawBytes;
219 x += nRunLength;
220
221 if (x > nWidth)
222 {
223 WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
224 return -1;
225 }
226
227 if (used > SrcSize)
228 {
229 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRId32, used,
230 INT32_MAX);
231 return -1;
232 }
233 }
234 }
235
236 if (used > INT32_MAX)
237 {
238 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
239 return -1;
240 }
241 return (INT32)used;
242}
243
244static inline UINT8 clamp(INT16 val)
245{
246 return (UINT8)val;
247}
248
249static inline INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
250 UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
251 UINT32 nWidth, UINT32 nHeight)
252{
253 BYTE* previousScanline = nullptr;
254 const BYTE* srcp = pSrcData;
255
256 WINPR_ASSERT(nHeight <= INT32_MAX);
257 WINPR_ASSERT(nWidth <= INT32_MAX);
258
259 for (UINT32 y = 0; y < nHeight; y++)
260 {
261 BYTE* dstp = &pDstData[1ULL * (y)*nWidth];
262 INT16 pixel = 0;
263 BYTE* currentScanline = dstp;
264
265 for (UINT32 x = 0; x < nWidth;)
266 {
267 BYTE controlByte = *srcp;
268 srcp++;
269
270 if ((srcp - pSrcData) > SrcSize * 1ll)
271 {
272 WLog_ERR(TAG, "error reading input buffer");
273 return -1;
274 }
275
276 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
277 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
278
279 if (nRunLength == 1)
280 {
281 nRunLength = cRawBytes + 16;
282 cRawBytes = 0;
283 }
284 else if (nRunLength == 2)
285 {
286 nRunLength = cRawBytes + 32;
287 cRawBytes = 0;
288 }
289
290 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
291 {
292 WLog_ERR(TAG, "too many pixels in scanline");
293 return -1;
294 }
295
296 if (!previousScanline)
297 {
298 /* first scanline, absolute values */
299 while (cRawBytes > 0)
300 {
301 pixel = *srcp;
302 srcp++;
303 *dstp = clamp(pixel);
304 dstp++;
305 x++;
306 cRawBytes--;
307 }
308
309 while (nRunLength > 0)
310 {
311 *dstp = clamp(pixel);
312 dstp++;
313 x++;
314 nRunLength--;
315 }
316 }
317 else
318 {
319 /* delta values relative to previous scanline */
320 while (cRawBytes > 0)
321 {
322 UINT8 deltaValue = *srcp;
323 srcp++;
324
325 if (deltaValue & 1)
326 {
327 deltaValue = deltaValue >> 1;
328 deltaValue = deltaValue + 1;
329 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue);
330 }
331 else
332 {
333 deltaValue = deltaValue >> 1;
334 pixel = WINPR_ASSERTING_INT_CAST(INT16, deltaValue);
335 }
336
337 const INT16 delta =
338 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
339 *dstp = clamp(delta);
340 dstp++;
341 x++;
342 cRawBytes--;
343 }
344
345 while (nRunLength > 0)
346 {
347 const INT16 deltaValue =
348 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
349 *dstp = clamp(deltaValue);
350 dstp++;
351 x++;
352 nRunLength--;
353 }
354 }
355 }
356
357 previousScanline = currentScanline;
358 }
359
360 return (INT32)(srcp - pSrcData);
361}
362
363static inline INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
364 BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
365 UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
366 UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
367{
368 INT32 beg = 0;
369 INT32 end = 0;
370 INT32 inc = 0;
371 BYTE* previousScanline = nullptr;
372 const BYTE* srcp = pSrcData;
373
374 WINPR_ASSERT(nHeight <= INT32_MAX);
375 WINPR_ASSERT(nWidth <= INT32_MAX);
376 WINPR_ASSERT(nDstStep <= INT32_MAX);
377
378 if (vFlip)
379 {
380 beg = (INT32)nHeight - 1;
381 end = -1;
382 inc = -1;
383 }
384 else
385 {
386 beg = 0;
387 end = (INT32)nHeight;
388 inc = 1;
389 }
390
391 for (INT32 y = beg; y != end; y += inc)
392 {
393 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
394 BYTE* dstp = &pDstData[off];
395 BYTE* currentScanline = dstp;
396 INT16 pixel = 0;
397
398 for (INT32 x = 0; x < (INT32)nWidth;)
399 {
400 const BYTE controlByte = *srcp;
401 srcp++;
402
403 if ((srcp - pSrcData) > SrcSize * 1ll)
404 {
405 WLog_ERR(TAG, "error reading input buffer");
406 return -1;
407 }
408
409 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
410 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
411
412 if (nRunLength == 1)
413 {
414 nRunLength = cRawBytes + 16;
415 cRawBytes = 0;
416 }
417 else if (nRunLength == 2)
418 {
419 nRunLength = cRawBytes + 32;
420 cRawBytes = 0;
421 }
422
423 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
424 {
425 WLog_ERR(TAG, "too many pixels in scanline");
426 return -1;
427 }
428
429 if (!previousScanline)
430 {
431 /* first scanline, absolute values */
432 while (cRawBytes > 0)
433 {
434 pixel = *srcp;
435 srcp++;
436 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
437 dstp += 4;
438 x++;
439 cRawBytes--;
440 }
441
442 while (nRunLength > 0)
443 {
444 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
445 dstp += 4;
446 x++;
447 nRunLength--;
448 }
449 }
450 else
451 {
452 /* delta values relative to previous scanline */
453 while (cRawBytes > 0)
454 {
455 BYTE deltaValue = *srcp;
456 srcp++;
457
458 if (deltaValue & 1)
459 {
460 deltaValue = deltaValue >> 1;
461 deltaValue = deltaValue + 1;
462 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue);
463 }
464 else
465 {
466 deltaValue = deltaValue >> 1;
467 pixel = deltaValue;
468 }
469
470 const INT16 delta =
471 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel);
472 *dstp = clamp(delta);
473 dstp += 4;
474 x++;
475 cRawBytes--;
476 }
477
478 while (nRunLength > 0)
479 {
480 const INT16 deltaValue =
481 WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]);
482 *dstp = clamp(deltaValue);
483 dstp += 4;
484 x++;
485 nRunLength--;
486 }
487 }
488 }
489
490 previousScanline = currentScanline;
491 }
492
493 return (INT32)(srcp - pSrcData);
494}
495
496static inline INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
497 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
498 BOOL vFlip)
499{
500 INT32 beg = 0;
501 INT32 end = (INT32)nHeight;
502 INT32 inc = 1;
503
504 WINPR_ASSERT(nHeight <= INT32_MAX);
505 WINPR_ASSERT(nWidth <= INT32_MAX);
506 WINPR_ASSERT(nDstStep <= INT32_MAX);
507
508 if (vFlip)
509 {
510 beg = (INT32)nHeight - 1;
511 end = -1;
512 inc = -1;
513 }
514
515 for (INT32 y = beg; y != end; y += inc)
516 {
517 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
518 BYTE* dstp = &pDstData[off];
519
520 for (INT32 x = 0; x < (INT32)nWidth; ++x)
521 {
522 *dstp = bValue;
523 dstp += 4;
524 }
525 }
526
527 return 0;
528}
529
530static inline BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
531 const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
532 const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
533{
534 WINPR_ASSERT(ppRgba);
535 WINPR_ASSERT(ppR);
536 WINPR_ASSERT(ppG);
537 WINPR_ASSERT(ppB);
538
539 switch (DstFormat)
540 {
541 case PIXEL_FORMAT_BGRA32:
542 for (UINT32 x = 0; x < width; x++)
543 {
544 *(*ppRgba)++ = *(*ppB)++;
545 *(*ppRgba)++ = *(*ppG)++;
546 *(*ppRgba)++ = *(*ppR)++;
547 *(*ppRgba)++ = *(*ppA)++;
548 }
549
550 return TRUE;
551
552 case PIXEL_FORMAT_BGRX32:
553 for (UINT32 x = 0; x < width; x++)
554 {
555 *(*ppRgba)++ = *(*ppB)++;
556 *(*ppRgba)++ = *(*ppG)++;
557 *(*ppRgba)++ = *(*ppR)++;
558 *(*ppRgba)++ = 0xFF;
559 }
560
561 return TRUE;
562
563 default:
564 if (ppA)
565 {
566 for (UINT32 x = 0; x < width; x++)
567 {
568 BYTE alpha = *(*ppA)++;
569 UINT32 color =
570 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
571 FreeRDPWriteColor(*ppRgba, DstFormat, color);
572 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
573 }
574 }
575 else
576 {
577 const BYTE alpha = 0xFF;
578
579 for (UINT32 x = 0; x < width; x++)
580 {
581 UINT32 color =
582 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
583 FreeRDPWriteColor(*ppRgba, DstFormat, color);
584 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
585 }
586 }
587
588 return TRUE;
589 }
590}
591
592static inline BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
593 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
594 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
595 UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
596 UINT32 totalHeight)
597{
598 INT32 beg = 0;
599 INT32 end = 0;
600 INT32 inc = 0;
601 const BYTE* pR = pSrcData[0];
602 const BYTE* pG = pSrcData[1];
603 const BYTE* pB = pSrcData[2];
604 const BYTE* pA = pSrcData[3];
605 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
606
607 if (vFlip)
608 {
609 beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1);
610 end = -1;
611 inc = -1;
612 }
613 else
614 {
615 beg = 0;
616 end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight);
617 inc = 1;
618 }
619
620 if (nYDst + nHeight > totalHeight)
621 {
622 WLog_ERR(TAG,
623 "planar plane destination Y %" PRIu32 " + height %" PRIu32
624 " exceeds totalHeight %" PRIu32,
625 nYDst, nHeight, totalHeight);
626 return FALSE;
627 }
628
629 if ((nXDst + nWidth) * bpp > nDstStep)
630 {
631 WLog_ERR(TAG,
632 "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
633 " exceeds stride %" PRIu32,
634 nXDst, nWidth, bpp, nDstStep);
635 return FALSE;
636 }
637
638 for (INT32 y = beg; y != end; y += inc)
639 {
640 BYTE* pRGB = nullptr;
641
642 if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight))
643 {
644 WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
645 nHeight);
646 return FALSE;
647 }
648
649 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp);
650 pRGB = &pDstData[off];
651
652 if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
653 return FALSE;
654 }
655
656 return TRUE;
657}
658
659static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
660 UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
661 UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
662{
663 size_t pos = 0;
664 WINPR_UNUSED(planeLength);
665
666 WINPR_ASSERT(plane);
667 WINPR_ASSERT(deltaPlane);
668
669 if (nWidth > nPlaneWidth * 2)
670 {
671 WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
672 nPlaneWidth);
673 return FALSE;
674 }
675
676 if (nHeight > nPlaneHeight * 2)
677 {
678 WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
679 nPlaneHeight);
680 return FALSE;
681 }
682
683 for (size_t y = 0; y < nHeight; y++)
684 {
685 const BYTE* src = plane + y / 2 * nPlaneWidth;
686
687 for (UINT32 x = 0; x < nWidth; x++)
688 {
689 deltaPlane[pos++] = src[x / 2];
690 }
691 }
692
693 return TRUE;
694}
695
696#if !defined(WITHOUT_FREERDP_3x_DEPRECATED)
697BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
698 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
699 UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
700 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
701 UINT32 nDstHeight, BOOL vFlip)
702{
703 return freerdp_bitmap_decompress_planar(planar, pSrcData, SrcSize, nSrcWidth, nSrcHeight,
704 pDstData, DstFormat, nDstStep, nXDst, nYDst, nDstWidth,
705 nDstHeight, vFlip);
706}
707#endif
708
709BOOL freerdp_bitmap_decompress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
710 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
711 UINT32 nSrcWidth, UINT32 nSrcHeight,
712 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
713 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
714 UINT32 nDstHeight, BOOL vFlip)
715{
716 BOOL useAlpha = FALSE;
717 INT32 status = 0;
718 INT32 rleSizes[4] = { 0, 0, 0, 0 };
719 UINT32 rawSizes[4] = WINPR_C_ARRAY_INIT;
720 UINT32 rawWidths[4] = WINPR_C_ARRAY_INIT;
721 UINT32 rawHeights[4] = WINPR_C_ARRAY_INIT;
722 const BYTE* planes[4] = WINPR_C_ARRAY_INIT;
723 const UINT32 w = MIN(nSrcWidth, nDstWidth);
724 const UINT32 h = MIN(nSrcHeight, nDstHeight);
725 const primitives_t* prims = primitives_get();
726
727 WINPR_ASSERT(planar);
728 WINPR_ASSERT(prims);
729
730 if (planar->maxWidth < nSrcWidth)
731 return FALSE;
732 if (planar->maxHeight < nSrcHeight)
733 return FALSE;
734
735 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
736 if (nDstStep <= 0)
737 nDstStep = nDstWidth * bpp;
738
739 const BYTE* srcp = pSrcData;
740
741 if (!pSrcData || (SrcSize < 1))
742 {
743 WLog_ERR(TAG, "Invalid argument pSrcData=%p [size=%" PRIu32 "]",
744 WINPR_CXX_COMPAT_CAST(const void*, pSrcData), SrcSize);
745 return FALSE;
746 }
747
748 if (!pDstData)
749 {
750 WLog_ERR(TAG, "Invalid argument pDstData=nullptr");
751 return FALSE;
752 }
753
754 const BYTE FormatHeader = *srcp++;
755 const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
756 const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) != 0;
757 const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) != 0;
758 const BYTE alpha = !(FormatHeader & PLANAR_FORMAT_HEADER_NA);
759
760 DstFormat = planar_invert_format(planar, alpha, DstFormat);
761
762 if (alpha)
763 useAlpha = FreeRDPColorHasAlpha(DstFormat);
764
765 // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
766 // alpha);
767
768 if (!cll && cs)
769 {
770 WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
771 return FALSE; /* Chroma subsampling requires YCoCg */
772 }
773
774 const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
775 const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
776 const UINT32 planeSize = nSrcWidth * nSrcHeight;
777 const UINT32 subSize = subWidth * subHeight;
778
779 if (!cs)
780 {
781 rawSizes[0] = planeSize; /* LumaOrRedPlane */
782 rawWidths[0] = nSrcWidth;
783 rawHeights[0] = nSrcHeight;
784 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
785 rawWidths[1] = nSrcWidth;
786 rawHeights[1] = nSrcHeight;
787 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
788 rawWidths[2] = nSrcWidth;
789 rawHeights[2] = nSrcHeight;
790 rawSizes[3] = planeSize; /* AlphaPlane */
791 rawWidths[3] = nSrcWidth;
792 rawHeights[3] = nSrcHeight;
793 }
794 else /* Chroma Subsampling */
795 {
796 rawSizes[0] = planeSize; /* LumaOrRedPlane */
797 rawWidths[0] = nSrcWidth;
798 rawHeights[0] = nSrcHeight;
799 rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
800 rawWidths[1] = subWidth;
801 rawHeights[1] = subHeight;
802 rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
803 rawWidths[2] = subWidth;
804 rawHeights[2] = subHeight;
805 rawSizes[3] = planeSize; /* AlphaPlane */
806 rawWidths[3] = nSrcWidth;
807 rawHeights[3] = nSrcHeight;
808 }
809
810 const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData));
811 if (SrcSize < diff)
812 {
813 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
814 return FALSE;
815 }
816
817 if (!rle) /* RAW */
818 {
819
820 UINT32 base = planeSize * 3;
821 if (cs)
822 base = planeSize + planeSize / 2;
823
824 if (alpha)
825 {
826 if ((SrcSize - diff) < (planeSize + base))
827 {
828 WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
829 (planeSize + base));
830 return FALSE;
831 }
832
833 planes[3] = srcp; /* AlphaPlane */
834 planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
835 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
836 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
837
838 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
839 {
840 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
841 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
842 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
843 return FALSE;
844 }
845 }
846 else
847 {
848 if ((SrcSize - diff) < base)
849 {
850 WLog_ERR(TAG, "plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff, base);
851 return FALSE;
852 }
853
854 planes[0] = srcp; /* LumaOrRedPlane */
855 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
856 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
857
858 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
859 {
860 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
861 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
862 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
863 return FALSE;
864 }
865 }
866 }
867 else /* RLE */
868 {
869 if (alpha)
870 {
871 planes[3] = srcp;
872 rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
873 rawHeights[3]); /* AlphaPlane */
874
875 if (rleSizes[3] < 0)
876 return FALSE;
877
878 planes[0] = planes[3] + rleSizes[3];
879 }
880 else
881 planes[0] = srcp;
882
883 const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData));
884 if (SrcSize < diff0)
885 {
886 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
887 return FALSE;
888 }
889 rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
890 rawHeights[0]); /* RedPlane */
891
892 if (rleSizes[0] < 0)
893 return FALSE;
894
895 planes[1] = planes[0] + rleSizes[0];
896
897 const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData));
898 if (SrcSize < diff1)
899 {
900 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
901 return FALSE;
902 }
903 rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
904 rawHeights[1]); /* GreenPlane */
905
906 if (rleSizes[1] < 1)
907 return FALSE;
908
909 planes[2] = planes[1] + rleSizes[1];
910 const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData));
911 if (SrcSize < diff2)
912 {
913 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
914 return FALSE;
915 }
916 rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
917 rawHeights[2]); /* BluePlane */
918
919 if (rleSizes[2] < 1)
920 return FALSE;
921 }
922
923 if (!cll) /* RGB */
924 {
925 UINT32 TempFormat = 0;
926 BYTE* pTempData = pDstData;
927 UINT32 nTempStep = nDstStep;
928 UINT32 nTotalHeight = nYDst + nDstHeight;
929
930 if (useAlpha)
931 TempFormat = PIXEL_FORMAT_BGRA32;
932 else
933 TempFormat = PIXEL_FORMAT_BGRX32;
934
935 TempFormat = planar_invert_format(planar, alpha, TempFormat);
936
937 if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
938 {
939 pTempData = planar->pTempData;
940 nTempStep = planar->nTempStep;
941 nTotalHeight = planar->maxHeight;
942 }
943
944 if (!rle) /* RAW */
945 {
946 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
947 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
948 return FALSE;
949
950 if (alpha)
951 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
952 else /* NoAlpha */
953 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
954
955 if ((SrcSize - (srcp - pSrcData)) == 1)
956 srcp++; /* pad */
957 }
958 else /* RLE */
959 {
960 if (nYDst + nSrcHeight > nTotalHeight)
961 {
962 WLog_ERR(TAG,
963 "planar plane destination Y %" PRIu32 " + height %" PRIu32
964 " exceeds totalHeight %" PRIu32,
965 nYDst, nSrcHeight, nTotalHeight);
966 return FALSE;
967 }
968
969 if ((nXDst + nSrcWidth) * bpp > nDstStep)
970 {
971 WLog_ERR(TAG,
972 "planar plane destination (X %" PRIu32 " + width %" PRIu32
973 ") * bpp %" PRIu32 " exceeds stride %" PRIu32,
974 nXDst, nSrcWidth, bpp, nDstStep);
975 return FALSE;
976 }
977
978 status = planar_decompress_plane_rle(
979 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep,
980 nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
981
982 if (status < 0)
983 return FALSE;
984
985 status = planar_decompress_plane_rle(
986 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep,
987 nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */
988
989 if (status < 0)
990 return FALSE;
991
992 status = planar_decompress_plane_rle(
993 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep,
994 nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
995
996 if (status < 0)
997 return FALSE;
998
999 srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
1000
1001 if (useAlpha)
1002 {
1003 status = planar_decompress_plane_rle(
1004 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData,
1005 nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */
1006 }
1007 else
1008 status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
1009 nSrcHeight, 3, vFlip);
1010
1011 if (status < 0)
1012 return FALSE;
1013
1014 if (alpha)
1015 srcp += rleSizes[3];
1016 }
1017
1018 if (pTempData != pDstData)
1019 {
1020 if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
1021 pTempData, TempFormat, nTempStep, nXDst, nYDst,
1022 nullptr, FREERDP_FLIP_NONE))
1023 {
1024 WLog_ERR(TAG, "planar image copy failed");
1025 return FALSE;
1026 }
1027 }
1028 }
1029 else /* YCoCg */
1030 {
1031 UINT32 TempFormat = 0;
1032 BYTE* pTempData = planar->pTempData;
1033 UINT32 nTempStep = planar->nTempStep;
1034 UINT32 nTotalHeight = planar->maxHeight;
1035 BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];
1036
1037 if (useAlpha)
1038 TempFormat = PIXEL_FORMAT_BGRA32;
1039 else
1040 TempFormat = PIXEL_FORMAT_BGRX32;
1041
1042 if (!pTempData)
1043 return FALSE;
1044
1045 if (rle) /* RLE encoded data. Decode and handle it like raw data. */
1046 {
1047 BYTE* rleBuffer[4] = WINPR_C_ARRAY_INIT;
1048
1049 if (!planar->rlePlanesBuffer)
1050 return FALSE;
1051
1052 rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
1053 rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
1054 rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
1055 rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
1056 if (useAlpha)
1057 {
1058 status = planar_decompress_plane_rle_only(
1059 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3],
1060 rawWidths[3], rawHeights[3]); /* AlphaPlane */
1061
1062 if (status < 0)
1063 return FALSE;
1064 }
1065
1066 if (alpha)
1067 srcp += rleSizes[3];
1068
1069 status = planar_decompress_plane_rle_only(
1070 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0],
1071 rawWidths[0], rawHeights[0]); /* LumaPlane */
1072
1073 if (status < 0)
1074 return FALSE;
1075
1076 status = planar_decompress_plane_rle_only(
1077 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1],
1078 rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */
1079
1080 if (status < 0)
1081 return FALSE;
1082
1083 status = planar_decompress_plane_rle_only(
1084 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2],
1085 rawWidths[2], rawHeights[2]); /* GreenChromaPlane */
1086
1087 if (status < 0)
1088 return FALSE;
1089
1090 planes[0] = rleBuffer[0];
1091 planes[1] = rleBuffer[1];
1092 planes[2] = rleBuffer[2];
1093 planes[3] = rleBuffer[3];
1094 }
1095
1096 /* RAW */
1097 {
1098 if (cs)
1099 { /* Chroma subsampling for Co and Cg:
1100 * Each pixel contains the value that should be expanded to
1101 * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
1102 if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
1103 rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
1104 return FALSE;
1105
1106 planes[1] = planar->deltaPlanes[0];
1107 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
1108 rawWidths[1] = nSrcWidth;
1109 rawHeights[1] = nSrcHeight;
1110
1111 if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
1112 rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
1113 return FALSE;
1114
1115 planes[2] = planar->deltaPlanes[1];
1116 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
1117 rawWidths[2] = nSrcWidth;
1118 rawHeights[2] = nSrcHeight;
1119 }
1120
1121 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
1122 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
1123 return FALSE;
1124
1125 if (alpha)
1126 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
1127 else /* NoAlpha */
1128 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
1129
1130 if ((SrcSize - (srcp - pSrcData)) == 1)
1131 srcp++; /* pad */
1132 }
1133
1134 WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
1135 int rc = prims->YCoCgToRGB_8u_AC4R(
1136 pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat,
1137 WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha);
1138 if (rc != PRIMITIVES_SUCCESS)
1139 {
1140 WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
1141 return FALSE;
1142 }
1143 }
1144
1145 WINPR_UNUSED(srcp);
1146 return TRUE;
1147}
1148
1149static inline BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
1150 const BYTE* WINPR_RESTRICT data, UINT32 format,
1151 UINT32 width, UINT32 height, UINT32 scanline,
1152 BYTE* WINPR_RESTRICT planes[4])
1153{
1154 WINPR_ASSERT(planar);
1155
1156 if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
1157 return FALSE;
1158
1159 if (scanline == 0)
1160 scanline = width * FreeRDPGetBytesPerPixel(format);
1161
1162 if (planar->topdown)
1163 {
1164 UINT32 k = 0;
1165 for (UINT32 i = 0; i < height; i++)
1166 {
1167 const BYTE* pixel = &data[1ULL * scanline * i];
1168
1169 for (UINT32 j = 0; j < width; j++)
1170 {
1171 const UINT32 color = FreeRDPReadColor(pixel, format);
1172 pixel += FreeRDPGetBytesPerPixel(format);
1173 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1174 &planes[0][k], nullptr);
1175 k++;
1176 }
1177 }
1178 }
1179 else
1180 {
1181 UINT32 k = 0;
1182
1183 for (INT64 i = (INT64)height - 1; i >= 0; i--)
1184 {
1185 const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];
1186
1187 for (UINT32 j = 0; j < width; j++)
1188 {
1189 const UINT32 color = FreeRDPReadColor(pixel, format);
1190 pixel += FreeRDPGetBytesPerPixel(format);
1191 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1192 &planes[0][k], nullptr);
1193 k++;
1194 }
1195 }
1196 }
1197 return TRUE;
1198}
1199
1200static inline UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1201 UINT32 cRawBytes, UINT32 nRunLength,
1202 BYTE* WINPR_RESTRICT pOutBuffer,
1203 UINT32 outBufferSize)
1204{
1205 const BYTE* pInput = pInBuffer;
1206 BYTE* pOutput = pOutBuffer;
1207 BYTE controlByte = 0;
1208 UINT32 nBytesToWrite = 0;
1209
1210 if (!cRawBytes && !nRunLength)
1211 return 0;
1212
1213 if (nRunLength < 3)
1214 {
1215 cRawBytes += nRunLength;
1216 nRunLength = 0;
1217 }
1218
1219 while (cRawBytes)
1220 {
1221 if (cRawBytes < 16)
1222 {
1223 if (nRunLength > 15)
1224 {
1225 if (nRunLength < 18)
1226 {
1227 controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
1228 nRunLength -= 13;
1229 cRawBytes = 0;
1230 }
1231 else
1232 {
1233 controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1234 nRunLength -= 15;
1235 cRawBytes = 0;
1236 }
1237 }
1238 else
1239 {
1240 controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1241 nRunLength = 0;
1242 cRawBytes = 0;
1243 }
1244 }
1245 else
1246 {
1247 controlByte = PLANAR_CONTROL_BYTE(0, 15);
1248 cRawBytes -= 15;
1249 }
1250
1251 if (outBufferSize < 1)
1252 return 0;
1253
1254 outBufferSize--;
1255 *pOutput = controlByte;
1256 pOutput++;
1257 nBytesToWrite = (controlByte >> 4);
1258
1259 if (nBytesToWrite)
1260 {
1261 if (outBufferSize < nBytesToWrite)
1262 return 0;
1263
1264 outBufferSize -= nBytesToWrite;
1265 CopyMemory(pOutput, pInput, nBytesToWrite);
1266 pOutput += nBytesToWrite;
1267 pInput += nBytesToWrite;
1268 }
1269 }
1270
1271 while (nRunLength)
1272 {
1273 if (nRunLength > 47)
1274 {
1275 if (nRunLength < 50)
1276 {
1277 controlByte = PLANAR_CONTROL_BYTE(2, 13);
1278 nRunLength -= 45;
1279 }
1280 else
1281 {
1282 controlByte = PLANAR_CONTROL_BYTE(2, 15);
1283 nRunLength -= 47;
1284 }
1285 }
1286 else if (nRunLength > 31)
1287 {
1288 controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1289 nRunLength = 0;
1290 }
1291 else if (nRunLength > 15)
1292 {
1293 controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1294 nRunLength = 0;
1295 }
1296 else
1297 {
1298 controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1299 nRunLength = 0;
1300 }
1301
1302 if (outBufferSize < 1)
1303 return 0;
1304
1305 --outBufferSize;
1306 *pOutput = controlByte;
1307 pOutput++;
1308 }
1309
1310 const intptr_t diff = (pOutput - pOutBuffer);
1311 if ((diff < 0) || (diff > UINT32_MAX))
1312 return 0;
1313 return (UINT32)diff;
1314}
1315
1316static inline UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1317 UINT32 inBufferSize,
1318 BYTE* WINPR_RESTRICT pOutBuffer,
1319 UINT32 outBufferSize)
1320{
1321 BYTE symbol = 0;
1322 const BYTE* pInput = pInBuffer;
1323 BYTE* pOutput = pOutBuffer;
1324 const BYTE* pBytes = nullptr;
1325 UINT32 cRawBytes = 0;
1326 UINT32 nRunLength = 0;
1327 UINT32 nBytesWritten = 0;
1328 UINT32 nTotalBytesWritten = 0;
1329
1330 if (!outBufferSize)
1331 return 0;
1332
1333 do
1334 {
1335 if (!inBufferSize)
1336 break;
1337
1338 const UINT32 bSymbolMatch = (symbol == *pInput) != 0;
1339 symbol = *pInput;
1340 pInput++;
1341 inBufferSize--;
1342
1343 if (nRunLength && !bSymbolMatch)
1344 {
1345 if (nRunLength < 3)
1346 {
1347 cRawBytes += nRunLength;
1348 nRunLength = 0;
1349 }
1350 else
1351 {
1352 pBytes = pInput - (cRawBytes + nRunLength + 1);
1353 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1354 pOutput, outBufferSize);
1355 nRunLength = 0;
1356
1357 if (!nBytesWritten || (nBytesWritten > outBufferSize))
1358 return nRunLength;
1359
1360 nTotalBytesWritten += nBytesWritten;
1361 outBufferSize -= nBytesWritten;
1362 pOutput += nBytesWritten;
1363 cRawBytes = 0;
1364 }
1365 }
1366
1367 nRunLength += bSymbolMatch;
1368 cRawBytes += (!bSymbolMatch) != 0;
1369 } while (outBufferSize);
1370
1371 if (cRawBytes || nRunLength)
1372 {
1373 pBytes = pInput - (cRawBytes + nRunLength);
1374 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1375 pOutput, outBufferSize);
1376
1377 if (!nBytesWritten)
1378 return 0;
1379
1380 nTotalBytesWritten += nBytesWritten;
1381 }
1382
1383 if (inBufferSize)
1384 return 0;
1385
1386 return nTotalBytesWritten;
1387}
1388
1389BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1390 UINT32 height, BYTE* WINPR_RESTRICT outPlane,
1391 UINT32* WINPR_RESTRICT dstSize)
1392{
1393 if (!outPlane)
1394 return FALSE;
1395
1396 UINT32 index = 0;
1397 const BYTE* pInput = inPlane;
1398 BYTE* pOutput = outPlane;
1399 UINT32 outBufferSize = *dstSize;
1400 UINT32 nTotalBytesWritten = 0;
1401
1402 while (outBufferSize > 0)
1403 {
1404 const UINT32 nBytesWritten =
1405 freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1406
1407 if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1408 return FALSE;
1409
1410 outBufferSize -= nBytesWritten;
1411 nTotalBytesWritten += nBytesWritten;
1412 pOutput += nBytesWritten;
1413 pInput += width;
1414 index++;
1415
1416 if (index >= height)
1417 break;
1418 }
1419
1420 *dstSize = nTotalBytesWritten;
1421 return TRUE;
1422}
1423
1424static inline BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
1425 UINT32 width, UINT32 height,
1426 BYTE* WINPR_RESTRICT outPlanes,
1427 UINT32* WINPR_RESTRICT dstSizes,
1428 BOOL skipAlpha)
1429{
1430 UINT32 outPlanesSize = width * height * 4;
1431
1432 /* AlphaPlane */
1433 if (skipAlpha)
1434 {
1435 dstSizes[0] = 0;
1436 }
1437 else
1438 {
1439 dstSizes[0] = outPlanesSize;
1440
1441 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1442 &dstSizes[0]))
1443 return FALSE;
1444
1445 outPlanes += dstSizes[0];
1446 outPlanesSize -= dstSizes[0];
1447 }
1448
1449 /* LumaOrRedPlane */
1450 dstSizes[1] = outPlanesSize;
1451
1452 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1453 &dstSizes[1]))
1454 return FALSE;
1455
1456 outPlanes += dstSizes[1];
1457 outPlanesSize -= dstSizes[1];
1458 /* OrangeChromaOrGreenPlane */
1459 dstSizes[2] = outPlanesSize;
1460
1461 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1462 &dstSizes[2]))
1463 return FALSE;
1464
1465 outPlanes += dstSizes[2];
1466 outPlanesSize -= dstSizes[2];
1467 /* GreenChromeOrBluePlane */
1468 dstSizes[3] = outPlanesSize;
1469
1470 return (freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1471 &dstSizes[3]));
1472}
1473
1474BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1475 UINT32 height, BYTE* WINPR_RESTRICT outPlane)
1476{
1477 if (!outPlane)
1478 {
1479 if (width * height == 0)
1480 return nullptr;
1481
1482 outPlane = (BYTE*)calloc(height, width);
1483 if (!outPlane)
1484 return nullptr;
1485 }
1486
1487 // first line is copied as is
1488 CopyMemory(outPlane, inPlane, width);
1489
1490 for (UINT32 y = 1; y < height; y++)
1491 {
1492 const size_t off = 1ull * width * y;
1493 BYTE* outPtr = &outPlane[off];
1494 const BYTE* srcPtr = &inPlane[off];
1495 const BYTE* prevLinePtr = &inPlane[off - width];
1496 for (UINT32 x = 0; x < width; x++)
1497 {
1498 const int delta = (int)srcPtr[x] - (int)prevLinePtr[x];
1499 const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1;
1500 const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i);
1501 const uint32_t s2c =
1502 (s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1);
1503 outPtr[x] = (BYTE)s2c;
1504 }
1505 }
1506
1507 return outPlane;
1508}
1509
1510static inline BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
1511 UINT32 width, UINT32 height,
1512 BYTE* WINPR_RESTRICT outPlanes[4])
1513{
1514 for (UINT32 i = 0; i < 4; i++)
1515 {
1516 outPlanes[i] =
1517 freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1518
1519 if (!outPlanes[i])
1520 return FALSE;
1521 }
1522
1523 return TRUE;
1524}
1525
1526BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1527 const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
1528 UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
1529 UINT32* WINPR_RESTRICT pDstSize)
1530{
1531 UINT32 size = 0;
1532 BYTE* dstp = nullptr;
1533 UINT32 dstSizes[4] = WINPR_C_ARRAY_INIT;
1534 BYTE FormatHeader = 0;
1535
1536 if (!context || !context->rlePlanesBuffer)
1537 return nullptr;
1538
1539 if (context->AllowSkipAlpha)
1540 FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1541
1542 const UINT32 planeSize = width * height;
1543
1544 if (!context->AllowSkipAlpha)
1545 format = planar_invert_format(context, TRUE, format);
1546
1547 if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
1548 context->planes))
1549 return nullptr;
1550
1551 if (context->AllowRunLengthEncoding)
1552 {
1553 if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1554 context->deltaPlanes))
1555 return nullptr;
1556
1557 if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1558 context->rlePlanesBuffer, dstSizes,
1559 context->AllowSkipAlpha))
1560 return nullptr;
1561
1562 {
1563 uint32_t offset = 0;
1564 FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1565 context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1566 offset += dstSizes[0];
1567 context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1568 offset += dstSizes[1];
1569 context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1570 offset += dstSizes[2];
1571 context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1572
1573#if defined(WITH_DEBUG_CODECS)
1574 WLog_DBG(TAG,
1575 "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
1576 " / %" PRIu32 "] ",
1577 dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
1578#endif
1579 }
1580 }
1581
1582 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1583 {
1584 if (!context->AllowRunLengthEncoding)
1585 return nullptr;
1586
1587 if (context->rlePlanes[0] == nullptr)
1588 return nullptr;
1589
1590 if (context->rlePlanes[1] == nullptr)
1591 return nullptr;
1592
1593 if (context->rlePlanes[2] == nullptr)
1594 return nullptr;
1595
1596 if (context->rlePlanes[3] == nullptr)
1597 return nullptr;
1598 }
1599
1600 if (!dstData)
1601 {
1602 size = 1;
1603
1604 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1605 {
1606 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1607 size += dstSizes[0];
1608 else
1609 size += planeSize;
1610 }
1611
1612 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1613 size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1614 else
1615 size += (planeSize * 3);
1616
1617 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1618 size++;
1619
1620 dstData = malloc(size);
1621
1622 if (!dstData)
1623 return nullptr;
1624
1625 *pDstSize = size;
1626 }
1627
1628 dstp = dstData;
1629 *dstp = FormatHeader; /* FormatHeader */
1630 dstp++;
1631
1632 /* AlphaPlane */
1633
1634 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1635 {
1636 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1637 {
1638 CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1639 dstp += dstSizes[0];
1640 }
1641 else
1642 {
1643 CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1644 dstp += planeSize;
1645 }
1646 }
1647
1648 /* LumaOrRedPlane */
1649
1650 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1651 {
1652 CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1653 dstp += dstSizes[1];
1654 }
1655 else
1656 {
1657 CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1658 dstp += planeSize;
1659 }
1660
1661 /* OrangeChromaOrGreenPlane */
1662
1663 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1664 {
1665 CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1666 dstp += dstSizes[2];
1667 }
1668 else
1669 {
1670 CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1671 dstp += planeSize;
1672 }
1673
1674 /* GreenChromeOrBluePlane */
1675
1676 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1677 {
1678 CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1679 dstp += dstSizes[3];
1680 }
1681 else
1682 {
1683 CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1684 dstp += planeSize;
1685 }
1686
1687 /* Pad1 (1 byte) */
1688
1689 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1690 {
1691 *dstp = 0;
1692 dstp++;
1693 }
1694
1695 const intptr_t diff = (dstp - dstData);
1696 if ((diff < 0) || (diff > UINT32_MAX))
1697 {
1698 free(dstData);
1699 return nullptr;
1700 }
1701 size = (UINT32)diff;
1702 *pDstSize = size;
1703 return dstData;
1704}
1705
1706BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1707 UINT32 width, UINT32 height)
1708{
1709 if (!context)
1710 return FALSE;
1711
1712 context->bgr = FALSE;
1713 context->maxWidth = PLANAR_ALIGN(width, 4);
1714 context->maxHeight = PLANAR_ALIGN(height, 4);
1715 {
1716 const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
1717 if (tmp > UINT32_MAX)
1718 return FALSE;
1719 context->maxPlaneSize = (UINT32)tmp;
1720 }
1721
1722 if (context->maxWidth > UINT32_MAX / 4)
1723 return FALSE;
1724 context->nTempStep = context->maxWidth * 4;
1725
1726 memset((void*)context->planes, 0, sizeof(context->planes));
1727 memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes));
1728 memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes));
1729
1730 if (context->maxPlaneSize > 0)
1731 {
1732 void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32);
1733 if (!tmp)
1734 return FALSE;
1735 context->planesBuffer = tmp;
1736
1737 tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32);
1738 if (!tmp)
1739 return FALSE;
1740 context->pTempData = tmp;
1741
1742 tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32);
1743 if (!tmp)
1744 return FALSE;
1745 context->deltaPlanesBuffer = tmp;
1746
1747 tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32);
1748 if (!tmp)
1749 return FALSE;
1750 context->rlePlanesBuffer = tmp;
1751
1752 context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
1753 context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
1754 context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
1755 context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
1756 context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
1757 context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
1758 context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
1759 context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
1760 }
1761 return TRUE;
1762}
1763
1764BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1765 UINT32 maxHeight)
1766{
1767 BITMAP_PLANAR_CONTEXT* context =
1768 (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);
1769
1770 if (!context)
1771 return nullptr;
1772
1773 if (flags & PLANAR_FORMAT_HEADER_NA)
1774 context->AllowSkipAlpha = TRUE;
1775
1776 if (flags & PLANAR_FORMAT_HEADER_RLE)
1777 context->AllowRunLengthEncoding = TRUE;
1778
1779 if (flags & PLANAR_FORMAT_HEADER_CS)
1780 context->AllowColorSubsampling = TRUE;
1781
1782 context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1783
1784 if (context->ColorLossLevel)
1785 context->AllowDynamicColorFidelity = TRUE;
1786
1787 if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1788 {
1789 WINPR_PRAGMA_DIAG_PUSH
1790 WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1791 freerdp_bitmap_planar_context_free(context);
1792 WINPR_PRAGMA_DIAG_POP
1793 return nullptr;
1794 }
1795
1796 return context;
1797}
1798
1799void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1800{
1801 if (!context)
1802 return;
1803
1804 winpr_aligned_free(context->pTempData);
1805 winpr_aligned_free(context->planesBuffer);
1806 winpr_aligned_free(context->deltaPlanesBuffer);
1807 winpr_aligned_free(context->rlePlanesBuffer);
1808 winpr_aligned_free(context);
1809}
1810
1811void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
1812{
1813 WINPR_ASSERT(planar);
1814 planar->bgr = bgr;
1815}
1816
1817void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
1818{
1819 WINPR_ASSERT(planar);
1820 planar->topdown = topdown;
1821}
primitives_t
Definition primitives.h:255