FreeRDP
Loading...
Searching...
No Matches
planar.c
1
22#include <freerdp/config.h>
23
24#include <winpr/crt.h>
25#include <winpr/wtypes.h>
26#include <winpr/assert.h>
27#include <winpr/cast.h>
28#include <winpr/print.h>
29
30#include <freerdp/primitives.h>
31#include <freerdp/log.h>
32#include <freerdp/codec/bitmap.h>
33#include <freerdp/codec/planar.h>
34
35#define TAG FREERDP_TAG("codec")
36
37#define PLANAR_ALIGN(val, align) \
38 ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
39
40typedef struct
41{
47 BYTE controlByte;
48 BYTE* rawValues;
49} RDP6_RLE_SEGMENT;
50
51typedef struct
52{
53 UINT32 cSegments;
54 RDP6_RLE_SEGMENT* segments;
55} RDP6_RLE_SEGMENTS;
56
57typedef struct
58{
67 BYTE formatHeader;
68} RDP6_BITMAP_STREAM;
69
70struct S_BITMAP_PLANAR_CONTEXT
71{
72 UINT32 maxWidth;
73 UINT32 maxHeight;
74 UINT32 maxPlaneSize;
75
76 BOOL AllowSkipAlpha;
77 BOOL AllowRunLengthEncoding;
78 BOOL AllowColorSubsampling;
79 BOOL AllowDynamicColorFidelity;
80
81 UINT32 ColorLossLevel;
82
83 BYTE* planes[4];
84 BYTE* planesBuffer;
85
86 BYTE* deltaPlanes[4];
87 BYTE* deltaPlanesBuffer;
88
89 BYTE* rlePlanes[4];
90 BYTE* rlePlanesBuffer;
91
92 BYTE* pTempData;
93 UINT32 nTempStep;
94
95 BOOL bgr;
96 BOOL topdown;
97};
98
99static inline BYTE PLANAR_CONTROL_BYTE(UINT32 nRunLength, UINT32 cRawBytes)
100{
101 return WINPR_ASSERTING_INT_CAST(UINT8, ((nRunLength & 0x0F) | ((cRawBytes & 0x0F) << 4)));
102}
103
104static inline BYTE PLANAR_CONTROL_BYTE_RUN_LENGTH(UINT32 controlByte)
105{
106 return (controlByte & 0x0F);
107}
108static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte)
109{
110 return ((controlByte >> 4) & 0x0F);
111}
112
113static inline UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
114 UINT32 DstFormat)
115{
116 WINPR_ASSERT(planar);
117 if (planar->bgr && alpha)
118 {
119 switch (DstFormat)
120 {
121 case PIXEL_FORMAT_ARGB32:
122 DstFormat = PIXEL_FORMAT_ABGR32;
123 break;
124 case PIXEL_FORMAT_XRGB32:
125 DstFormat = PIXEL_FORMAT_XBGR32;
126 break;
127 case PIXEL_FORMAT_ABGR32:
128 DstFormat = PIXEL_FORMAT_ARGB32;
129 break;
130 case PIXEL_FORMAT_XBGR32:
131 DstFormat = PIXEL_FORMAT_XRGB32;
132 break;
133 case PIXEL_FORMAT_BGRA32:
134 DstFormat = PIXEL_FORMAT_RGBA32;
135 break;
136 case PIXEL_FORMAT_BGRX32:
137 DstFormat = PIXEL_FORMAT_RGBX32;
138 break;
139 case PIXEL_FORMAT_RGBA32:
140 DstFormat = PIXEL_FORMAT_BGRA32;
141 break;
142 case PIXEL_FORMAT_RGBX32:
143 DstFormat = PIXEL_FORMAT_BGRX32;
144 break;
145 case PIXEL_FORMAT_RGB24:
146 DstFormat = PIXEL_FORMAT_BGR24;
147 break;
148 case PIXEL_FORMAT_BGR24:
149 DstFormat = PIXEL_FORMAT_RGB24;
150 break;
151 case PIXEL_FORMAT_RGB16:
152 DstFormat = PIXEL_FORMAT_BGR16;
153 break;
154 case PIXEL_FORMAT_BGR16:
155 DstFormat = PIXEL_FORMAT_RGB16;
156 break;
157 case PIXEL_FORMAT_ARGB15:
158 DstFormat = PIXEL_FORMAT_ABGR15;
159 break;
160 case PIXEL_FORMAT_RGB15:
161 DstFormat = PIXEL_FORMAT_BGR15;
162 break;
163 case PIXEL_FORMAT_ABGR15:
164 DstFormat = PIXEL_FORMAT_ARGB15;
165 break;
166 case PIXEL_FORMAT_BGR15:
167 DstFormat = PIXEL_FORMAT_RGB15;
168 break;
169 default:
170 break;
171 }
172 }
173 return DstFormat;
174}
175
176static inline BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
177 UINT32 width, UINT32 height,
178 BYTE* WINPR_RESTRICT outPlane,
179 UINT32* WINPR_RESTRICT dstSize);
180static inline BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
181 UINT32 width, UINT32 height,
182 BYTE* WINPR_RESTRICT outPlane);
183
184static inline INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
185 UINT32 nWidth, UINT32 nHeight)
186{
187 UINT32 used = 0;
188
189 WINPR_ASSERT(pSrcData);
190
191 for (UINT32 y = 0; y < nHeight; y++)
192 {
193 for (UINT32 x = 0; x < nWidth;)
194 {
195 if (used >= SrcSize)
196 {
197 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
198 SrcSize);
199 return -1;
200 }
201
202 const uint8_t controlByte = pSrcData[used++];
203 uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
204 uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
205
206 if (nRunLength == 1)
207 {
208 nRunLength = cRawBytes + 16;
209 cRawBytes = 0;
210 }
211 else if (nRunLength == 2)
212 {
213 nRunLength = cRawBytes + 32;
214 cRawBytes = 0;
215 }
216
217 used += cRawBytes;
218 x += cRawBytes;
219 x += nRunLength;
220
221 if (x > nWidth)
222 {
223 WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
224 return -1;
225 }
226
227 if (used > SrcSize)
228 {
229 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRId32, used,
230 INT32_MAX);
231 return -1;
232 }
233 }
234 }
235
236 if (used > INT32_MAX)
237 {
238 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
239 return -1;
240 }
241 return (INT32)used;
242}
243
244static inline UINT8 clamp(INT16 val)
245{
246 return (UINT8)val;
247}
248
249static inline INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
250 UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
251 UINT32 nWidth, UINT32 nHeight)
252{
253 BYTE* previousScanline = nullptr;
254 const BYTE* srcp = pSrcData;
255
256 WINPR_ASSERT(nHeight <= INT32_MAX);
257 WINPR_ASSERT(nWidth <= INT32_MAX);
258
259 for (UINT32 y = 0; y < nHeight; y++)
260 {
261 BYTE* dstp = &pDstData[1ULL * (y)*nWidth];
262 INT16 pixel = 0;
263 BYTE* currentScanline = dstp;
264
265 for (UINT32 x = 0; x < nWidth;)
266 {
267 const size_t cur = WINPR_ASSERTING_INT_CAST(size_t, (srcp - pSrcData));
268 if (cur + 1 > SrcSize * 1ll)
269 {
270 WLog_ERR(TAG, "error reading input buffer");
271 return -1;
272 }
273
274 BYTE controlByte = *srcp;
275 srcp++;
276
277 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
278 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
279
280 if (nRunLength == 1)
281 {
282 nRunLength = cRawBytes + 16;
283 cRawBytes = 0;
284 }
285 else if (nRunLength == 2)
286 {
287 nRunLength = cRawBytes + 32;
288 cRawBytes = 0;
289 }
290
291 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
292 {
293 WLog_ERR(TAG, "too many pixels in scanline");
294 return -1;
295 }
296
297 if (!previousScanline)
298 {
299 /* first scanline, absolute values */
300 while (cRawBytes > 0)
301 {
302 pixel = *srcp;
303 srcp++;
304 *dstp = clamp(pixel);
305 dstp++;
306 x++;
307 cRawBytes--;
308 }
309
310 while (nRunLength > 0)
311 {
312 *dstp = clamp(pixel);
313 dstp++;
314 x++;
315 nRunLength--;
316 }
317 }
318 else
319 {
320 /* delta values relative to previous scanline */
321 while (cRawBytes > 0)
322 {
323 UINT8 deltaValue = *srcp;
324 srcp++;
325
326 if (deltaValue & 1)
327 {
328 deltaValue = deltaValue >> 1;
329 deltaValue = deltaValue + 1;
330 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue);
331 }
332 else
333 {
334 deltaValue = deltaValue >> 1;
335 pixel = WINPR_ASSERTING_INT_CAST(INT16, deltaValue);
336 }
337
338 const INT16 delta =
339 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
340 *dstp = clamp(delta);
341 dstp++;
342 x++;
343 cRawBytes--;
344 }
345
346 while (nRunLength > 0)
347 {
348 const INT16 deltaValue =
349 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
350 *dstp = clamp(deltaValue);
351 dstp++;
352 x++;
353 nRunLength--;
354 }
355 }
356 }
357
358 previousScanline = currentScanline;
359 }
360
361 return (INT32)(srcp - pSrcData);
362}
363
364static inline INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
365 BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
366 UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
367 UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
368{
369 INT32 beg = 0;
370 INT32 end = 0;
371 INT32 inc = 0;
372 BYTE* previousScanline = nullptr;
373 const BYTE* srcp = pSrcData;
374
375 WINPR_ASSERT(nHeight <= INT32_MAX);
376 WINPR_ASSERT(nWidth <= INT32_MAX);
377 WINPR_ASSERT(nDstStep <= INT32_MAX);
378
379 if (vFlip)
380 {
381 beg = (INT32)nHeight - 1;
382 end = -1;
383 inc = -1;
384 }
385 else
386 {
387 beg = 0;
388 end = (INT32)nHeight;
389 inc = 1;
390 }
391
392 for (INT32 y = beg; y != end; y += inc)
393 {
394 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
395 BYTE* dstp = &pDstData[off];
396 BYTE* currentScanline = dstp;
397 INT16 pixel = 0;
398
399 for (INT32 x = 0; x < (INT32)nWidth;)
400 {
401 const size_t cur = WINPR_ASSERTING_INT_CAST(size_t, (srcp - pSrcData));
402 if (cur + 1 > SrcSize * 1ll)
403 {
404 WLog_ERR(TAG, "error reading input buffer");
405 return -1;
406 }
407
408 const BYTE controlByte = *srcp;
409 srcp++;
410
411 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
412 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
413
414 if (nRunLength == 1)
415 {
416 nRunLength = cRawBytes + 16;
417 cRawBytes = 0;
418 }
419 else if (nRunLength == 2)
420 {
421 nRunLength = cRawBytes + 32;
422 cRawBytes = 0;
423 }
424
425 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
426 {
427 WLog_ERR(TAG, "too many pixels in scanline");
428 return -1;
429 }
430
431 if (!previousScanline)
432 {
433 /* first scanline, absolute values */
434 while (cRawBytes > 0)
435 {
436 pixel = *srcp;
437 srcp++;
438 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
439 dstp += 4;
440 x++;
441 cRawBytes--;
442 }
443
444 while (nRunLength > 0)
445 {
446 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
447 dstp += 4;
448 x++;
449 nRunLength--;
450 }
451 }
452 else
453 {
454 /* delta values relative to previous scanline */
455 while (cRawBytes > 0)
456 {
457 BYTE deltaValue = *srcp;
458 srcp++;
459
460 if (deltaValue & 1)
461 {
462 deltaValue = deltaValue >> 1;
463 deltaValue = deltaValue + 1;
464 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue);
465 }
466 else
467 {
468 deltaValue = deltaValue >> 1;
469 pixel = deltaValue;
470 }
471
472 const INT16 delta =
473 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel);
474 *dstp = clamp(delta);
475 dstp += 4;
476 x++;
477 cRawBytes--;
478 }
479
480 while (nRunLength > 0)
481 {
482 const INT16 deltaValue =
483 WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]);
484 *dstp = clamp(deltaValue);
485 dstp += 4;
486 x++;
487 nRunLength--;
488 }
489 }
490 }
491
492 previousScanline = currentScanline;
493 }
494
495 return (INT32)(srcp - pSrcData);
496}
497
498static inline INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
499 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
500 BOOL vFlip)
501{
502 INT32 beg = 0;
503 INT32 end = (INT32)nHeight;
504 INT32 inc = 1;
505
506 WINPR_ASSERT(nHeight <= INT32_MAX);
507 WINPR_ASSERT(nWidth <= INT32_MAX);
508 WINPR_ASSERT(nDstStep <= INT32_MAX);
509
510 if (vFlip)
511 {
512 beg = (INT32)nHeight - 1;
513 end = -1;
514 inc = -1;
515 }
516
517 for (INT32 y = beg; y != end; y += inc)
518 {
519 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
520 BYTE* dstp = &pDstData[off];
521
522 for (INT32 x = 0; x < (INT32)nWidth; ++x)
523 {
524 *dstp = bValue;
525 dstp += 4;
526 }
527 }
528
529 return 0;
530}
531
532static inline BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
533 const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
534 const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
535{
536 WINPR_ASSERT(ppRgba);
537 WINPR_ASSERT(ppR);
538 WINPR_ASSERT(ppG);
539 WINPR_ASSERT(ppB);
540
541 switch (DstFormat)
542 {
543 case PIXEL_FORMAT_BGRA32:
544 for (UINT32 x = 0; x < width; x++)
545 {
546 *(*ppRgba)++ = *(*ppB)++;
547 *(*ppRgba)++ = *(*ppG)++;
548 *(*ppRgba)++ = *(*ppR)++;
549 *(*ppRgba)++ = *(*ppA)++;
550 }
551
552 return TRUE;
553
554 case PIXEL_FORMAT_BGRX32:
555 for (UINT32 x = 0; x < width; x++)
556 {
557 *(*ppRgba)++ = *(*ppB)++;
558 *(*ppRgba)++ = *(*ppG)++;
559 *(*ppRgba)++ = *(*ppR)++;
560 *(*ppRgba)++ = 0xFF;
561 }
562
563 return TRUE;
564
565 default:
566 if (ppA)
567 {
568 for (UINT32 x = 0; x < width; x++)
569 {
570 BYTE alpha = *(*ppA)++;
571 UINT32 color =
572 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
573 FreeRDPWriteColor(*ppRgba, DstFormat, color);
574 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
575 }
576 }
577 else
578 {
579 const BYTE alpha = 0xFF;
580
581 for (UINT32 x = 0; x < width; x++)
582 {
583 UINT32 color =
584 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
585 FreeRDPWriteColor(*ppRgba, DstFormat, color);
586 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
587 }
588 }
589
590 return TRUE;
591 }
592}
593
594static inline BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
595 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
596 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
597 UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
598 UINT32 totalHeight)
599{
600 INT32 beg = 0;
601 INT32 end = 0;
602 INT32 inc = 0;
603 const BYTE* pR = pSrcData[0];
604 const BYTE* pG = pSrcData[1];
605 const BYTE* pB = pSrcData[2];
606 const BYTE* pA = pSrcData[3];
607 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
608
609 if (vFlip)
610 {
611 beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1);
612 end = -1;
613 inc = -1;
614 }
615 else
616 {
617 beg = 0;
618 end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight);
619 inc = 1;
620 }
621
622 if (nYDst + nHeight > totalHeight)
623 {
624 WLog_ERR(TAG,
625 "planar plane destination Y %" PRIu32 " + height %" PRIu32
626 " exceeds totalHeight %" PRIu32,
627 nYDst, nHeight, totalHeight);
628 return FALSE;
629 }
630
631 if ((nXDst + nWidth) * bpp > nDstStep)
632 {
633 WLog_ERR(TAG,
634 "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
635 " exceeds stride %" PRIu32,
636 nXDst, nWidth, bpp, nDstStep);
637 return FALSE;
638 }
639
640 for (INT32 y = beg; y != end; y += inc)
641 {
642 BYTE* pRGB = nullptr;
643
644 if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight))
645 {
646 WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
647 nHeight);
648 return FALSE;
649 }
650
651 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp);
652 pRGB = &pDstData[off];
653
654 if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
655 return FALSE;
656 }
657
658 return TRUE;
659}
660
661static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
662 UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
663 UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
664{
665 size_t pos = 0;
666 WINPR_UNUSED(planeLength);
667
668 WINPR_ASSERT(plane);
669 WINPR_ASSERT(deltaPlane);
670
671 if (nWidth > nPlaneWidth * 2)
672 {
673 WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
674 nPlaneWidth);
675 return FALSE;
676 }
677
678 if (nHeight > nPlaneHeight * 2)
679 {
680 WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
681 nPlaneHeight);
682 return FALSE;
683 }
684
685 for (size_t y = 0; y < nHeight; y++)
686 {
687 const BYTE* src = plane + y / 2 * nPlaneWidth;
688
689 for (UINT32 x = 0; x < nWidth; x++)
690 {
691 deltaPlane[pos++] = src[x / 2];
692 }
693 }
694
695 return TRUE;
696}
697
698#if !defined(WITHOUT_FREERDP_3x_DEPRECATED)
699BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
700 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
701 UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
702 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
703 UINT32 nDstHeight, BOOL vFlip)
704{
705 return freerdp_bitmap_decompress_planar(planar, pSrcData, SrcSize, nSrcWidth, nSrcHeight,
706 pDstData, DstFormat, nDstStep, nXDst, nYDst, nDstWidth,
707 nDstHeight, vFlip);
708}
709#endif
710
711BOOL freerdp_bitmap_decompress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
712 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
713 UINT32 nSrcWidth, UINT32 nSrcHeight,
714 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
715 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
716 UINT32 nDstHeight, BOOL vFlip)
717{
718 BOOL useAlpha = FALSE;
719 INT32 status = 0;
720 INT32 rleSizes[4] = { 0, 0, 0, 0 };
721 UINT32 rawSizes[4] = WINPR_C_ARRAY_INIT;
722 UINT32 rawWidths[4] = WINPR_C_ARRAY_INIT;
723 UINT32 rawHeights[4] = WINPR_C_ARRAY_INIT;
724 const BYTE* planes[4] = WINPR_C_ARRAY_INIT;
725 const UINT32 w = MIN(nSrcWidth, nDstWidth);
726 const UINT32 h = MIN(nSrcHeight, nDstHeight);
727 const primitives_t* prims = primitives_get();
728
729 WINPR_ASSERT(planar);
730 WINPR_ASSERT(prims);
731
732 if (planar->maxWidth < nSrcWidth)
733 return FALSE;
734 if (planar->maxHeight < nSrcHeight)
735 return FALSE;
736
737 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
738 if (nDstStep <= 0)
739 nDstStep = nDstWidth * bpp;
740
741 const BYTE* srcp = pSrcData;
742
743 if (!pSrcData || (SrcSize < 1))
744 {
745 WLog_ERR(TAG, "Invalid argument pSrcData=%p [size=%" PRIu32 "]",
746 WINPR_CXX_COMPAT_CAST(const void*, pSrcData), SrcSize);
747 return FALSE;
748 }
749
750 if (!pDstData)
751 {
752 WLog_ERR(TAG, "Invalid argument pDstData=nullptr");
753 return FALSE;
754 }
755
756 const BYTE FormatHeader = *srcp++;
757 const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
758 const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) != 0;
759 const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) != 0;
760 const BYTE alpha = !(FormatHeader & PLANAR_FORMAT_HEADER_NA);
761
762 DstFormat = planar_invert_format(planar, alpha, DstFormat);
763
764 if (alpha)
765 useAlpha = FreeRDPColorHasAlpha(DstFormat);
766
767 // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
768 // alpha);
769
770 if (!cll && cs)
771 {
772 WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
773 return FALSE; /* Chroma subsampling requires YCoCg */
774 }
775
776 const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
777 const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
778 const UINT32 planeSize = nSrcWidth * nSrcHeight;
779 const UINT32 subSize = subWidth * subHeight;
780
781 if (!cs)
782 {
783 rawSizes[0] = planeSize; /* LumaOrRedPlane */
784 rawWidths[0] = nSrcWidth;
785 rawHeights[0] = nSrcHeight;
786 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
787 rawWidths[1] = nSrcWidth;
788 rawHeights[1] = nSrcHeight;
789 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
790 rawWidths[2] = nSrcWidth;
791 rawHeights[2] = nSrcHeight;
792 rawSizes[3] = planeSize; /* AlphaPlane */
793 rawWidths[3] = nSrcWidth;
794 rawHeights[3] = nSrcHeight;
795 }
796 else /* Chroma Subsampling */
797 {
798 rawSizes[0] = planeSize; /* LumaOrRedPlane */
799 rawWidths[0] = nSrcWidth;
800 rawHeights[0] = nSrcHeight;
801 rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
802 rawWidths[1] = subWidth;
803 rawHeights[1] = subHeight;
804 rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
805 rawWidths[2] = subWidth;
806 rawHeights[2] = subHeight;
807 rawSizes[3] = planeSize; /* AlphaPlane */
808 rawWidths[3] = nSrcWidth;
809 rawHeights[3] = nSrcHeight;
810 }
811
812 const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData));
813 if (SrcSize < diff)
814 {
815 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
816 return FALSE;
817 }
818
819 if (!rle) /* RAW */
820 {
821
822 UINT32 base = planeSize * 3;
823 if (cs)
824 base = planeSize + planeSize / 2;
825
826 if (alpha)
827 {
828 if ((SrcSize - diff) < (planeSize + base))
829 {
830 WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
831 (planeSize + base));
832 return FALSE;
833 }
834
835 planes[3] = srcp; /* AlphaPlane */
836 planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
837 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
838 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
839
840 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
841 {
842 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
843 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
844 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
845 return FALSE;
846 }
847 }
848 else
849 {
850 if ((SrcSize - diff) < base)
851 {
852 WLog_ERR(TAG, "plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff, base);
853 return FALSE;
854 }
855
856 planes[0] = srcp; /* LumaOrRedPlane */
857 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
858 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
859
860 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
861 {
862 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
863 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
864 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
865 return FALSE;
866 }
867 }
868 }
869 else /* RLE */
870 {
871 if (alpha)
872 {
873 planes[3] = srcp;
874 rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
875 rawHeights[3]); /* AlphaPlane */
876
877 if (rleSizes[3] < 0)
878 return FALSE;
879
880 planes[0] = planes[3] + rleSizes[3];
881 }
882 else
883 planes[0] = srcp;
884
885 const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData));
886 if (SrcSize < diff0)
887 {
888 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
889 return FALSE;
890 }
891 rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
892 rawHeights[0]); /* RedPlane */
893
894 if (rleSizes[0] < 0)
895 return FALSE;
896
897 planes[1] = planes[0] + rleSizes[0];
898
899 const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData));
900 if (SrcSize < diff1)
901 {
902 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
903 return FALSE;
904 }
905 rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
906 rawHeights[1]); /* GreenPlane */
907
908 if (rleSizes[1] < 1)
909 return FALSE;
910
911 planes[2] = planes[1] + rleSizes[1];
912 const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData));
913 if (SrcSize < diff2)
914 {
915 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
916 return FALSE;
917 }
918 rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
919 rawHeights[2]); /* BluePlane */
920
921 if (rleSizes[2] < 1)
922 return FALSE;
923 }
924
925 if (!cll) /* RGB */
926 {
927 UINT32 TempFormat = 0;
928 BYTE* pTempData = pDstData;
929 UINT32 nTempStep = nDstStep;
930 UINT32 nTotalHeight = nYDst + nDstHeight;
931
932 if (useAlpha)
933 TempFormat = PIXEL_FORMAT_BGRA32;
934 else
935 TempFormat = PIXEL_FORMAT_BGRX32;
936
937 TempFormat = planar_invert_format(planar, alpha, TempFormat);
938
939 if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
940 {
941 pTempData = planar->pTempData;
942 nTempStep = planar->nTempStep;
943 nTotalHeight = planar->maxHeight;
944 }
945
946 if (!rle) /* RAW */
947 {
948 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
949 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
950 return FALSE;
951
952 if (alpha)
953 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
954 else /* NoAlpha */
955 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
956
957 if ((SrcSize - (srcp - pSrcData)) == 1)
958 srcp++; /* pad */
959 }
960 else /* RLE */
961 {
962 if (nYDst + nSrcHeight > nTotalHeight)
963 {
964 WLog_ERR(TAG,
965 "planar plane destination Y %" PRIu32 " + height %" PRIu32
966 " exceeds totalHeight %" PRIu32,
967 nYDst, nSrcHeight, nTotalHeight);
968 return FALSE;
969 }
970
971 if ((nXDst + nSrcWidth) * bpp > nTempStep)
972 {
973 WLog_ERR(TAG,
974 "planar plane destination (X %" PRIu32 " + width %" PRIu32
975 ") * bpp %" PRIu32 " exceeds stride %" PRIu32,
976 nXDst, nSrcWidth, bpp, nTempStep);
977 return FALSE;
978 }
979
980 status = planar_decompress_plane_rle(
981 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep,
982 nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
983
984 if (status < 0)
985 return FALSE;
986
987 status = planar_decompress_plane_rle(
988 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep,
989 nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */
990
991 if (status < 0)
992 return FALSE;
993
994 status = planar_decompress_plane_rle(
995 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep,
996 nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
997
998 if (status < 0)
999 return FALSE;
1000
1001 srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
1002
1003 if (useAlpha)
1004 {
1005 status = planar_decompress_plane_rle(
1006 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData,
1007 nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */
1008 }
1009 else
1010 status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
1011 nSrcHeight, 3, vFlip);
1012
1013 if (status < 0)
1014 return FALSE;
1015
1016 if (alpha)
1017 srcp += rleSizes[3];
1018 }
1019
1020 if (pTempData != pDstData)
1021 {
1022 if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
1023 pTempData, TempFormat, nTempStep, nXDst, nYDst,
1024 nullptr, FREERDP_FLIP_NONE))
1025 {
1026 WLog_ERR(TAG, "planar image copy failed");
1027 return FALSE;
1028 }
1029 }
1030 }
1031 else /* YCoCg */
1032 {
1033 UINT32 TempFormat = 0;
1034 BYTE* pTempData = planar->pTempData;
1035 UINT32 nTempStep = planar->nTempStep;
1036 UINT32 nTotalHeight = planar->maxHeight;
1037 BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];
1038
1039 if (useAlpha)
1040 TempFormat = PIXEL_FORMAT_BGRA32;
1041 else
1042 TempFormat = PIXEL_FORMAT_BGRX32;
1043
1044 if (!pTempData)
1045 return FALSE;
1046
1047 if (rle) /* RLE encoded data. Decode and handle it like raw data. */
1048 {
1049 BYTE* rleBuffer[4] = WINPR_C_ARRAY_INIT;
1050
1051 if (!planar->rlePlanesBuffer)
1052 return FALSE;
1053
1054 rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
1055 rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
1056 rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
1057 rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
1058 if (useAlpha)
1059 {
1060 status = planar_decompress_plane_rle_only(
1061 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3],
1062 rawWidths[3], rawHeights[3]); /* AlphaPlane */
1063
1064 if (status < 0)
1065 return FALSE;
1066 }
1067
1068 if (alpha)
1069 srcp += rleSizes[3];
1070
1071 status = planar_decompress_plane_rle_only(
1072 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0],
1073 rawWidths[0], rawHeights[0]); /* LumaPlane */
1074
1075 if (status < 0)
1076 return FALSE;
1077
1078 status = planar_decompress_plane_rle_only(
1079 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1],
1080 rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */
1081
1082 if (status < 0)
1083 return FALSE;
1084
1085 status = planar_decompress_plane_rle_only(
1086 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2],
1087 rawWidths[2], rawHeights[2]); /* GreenChromaPlane */
1088
1089 if (status < 0)
1090 return FALSE;
1091
1092 planes[0] = rleBuffer[0];
1093 planes[1] = rleBuffer[1];
1094 planes[2] = rleBuffer[2];
1095 planes[3] = rleBuffer[3];
1096 }
1097
1098 /* RAW */
1099 {
1100 if (cs)
1101 { /* Chroma subsampling for Co and Cg:
1102 * Each pixel contains the value that should be expanded to
1103 * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
1104 if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
1105 rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
1106 return FALSE;
1107
1108 planes[1] = planar->deltaPlanes[0];
1109 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
1110 rawWidths[1] = nSrcWidth;
1111 rawHeights[1] = nSrcHeight;
1112
1113 if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
1114 rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
1115 return FALSE;
1116
1117 planes[2] = planar->deltaPlanes[1];
1118 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
1119 rawWidths[2] = nSrcWidth;
1120 rawHeights[2] = nSrcHeight;
1121 }
1122
1123 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
1124 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
1125 return FALSE;
1126
1127 if (alpha)
1128 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
1129 else /* NoAlpha */
1130 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
1131
1132 if ((SrcSize - (srcp - pSrcData)) == 1)
1133 srcp++; /* pad */
1134 }
1135
1136 WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
1137 int rc = prims->YCoCgToRGB_8u_AC4R(
1138 pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat,
1139 WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha);
1140 if (rc != PRIMITIVES_SUCCESS)
1141 {
1142 WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
1143 return FALSE;
1144 }
1145 }
1146
1147 WINPR_UNUSED(srcp);
1148 return TRUE;
1149}
1150
1151static inline BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
1152 const BYTE* WINPR_RESTRICT data, UINT32 format,
1153 UINT32 width, UINT32 height, UINT32 scanline,
1154 BYTE* WINPR_RESTRICT planes[4])
1155{
1156 WINPR_ASSERT(planar);
1157
1158 if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
1159 return FALSE;
1160
1161 if (scanline == 0)
1162 scanline = width * FreeRDPGetBytesPerPixel(format);
1163
1164 if (planar->topdown)
1165 {
1166 UINT32 k = 0;
1167 for (UINT32 i = 0; i < height; i++)
1168 {
1169 const BYTE* pixel = &data[1ULL * scanline * i];
1170
1171 for (UINT32 j = 0; j < width; j++)
1172 {
1173 const UINT32 color = FreeRDPReadColor(pixel, format);
1174 pixel += FreeRDPGetBytesPerPixel(format);
1175 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1176 &planes[0][k], nullptr);
1177 k++;
1178 }
1179 }
1180 }
1181 else
1182 {
1183 UINT32 k = 0;
1184
1185 for (INT64 i = (INT64)height - 1; i >= 0; i--)
1186 {
1187 const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];
1188
1189 for (UINT32 j = 0; j < width; j++)
1190 {
1191 const UINT32 color = FreeRDPReadColor(pixel, format);
1192 pixel += FreeRDPGetBytesPerPixel(format);
1193 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1194 &planes[0][k], nullptr);
1195 k++;
1196 }
1197 }
1198 }
1199 return TRUE;
1200}
1201
1202static inline UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1203 UINT32 cRawBytes, UINT32 nRunLength,
1204 BYTE* WINPR_RESTRICT pOutBuffer,
1205 UINT32 outBufferSize)
1206{
1207 const BYTE* pInput = pInBuffer;
1208 BYTE* pOutput = pOutBuffer;
1209 BYTE controlByte = 0;
1210 UINT32 nBytesToWrite = 0;
1211
1212 if (!cRawBytes && !nRunLength)
1213 return 0;
1214
1215 if (nRunLength < 3)
1216 {
1217 cRawBytes += nRunLength;
1218 nRunLength = 0;
1219 }
1220
1221 while (cRawBytes)
1222 {
1223 if (cRawBytes < 16)
1224 {
1225 if (nRunLength > 15)
1226 {
1227 if (nRunLength < 18)
1228 {
1229 controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
1230 nRunLength -= 13;
1231 cRawBytes = 0;
1232 }
1233 else
1234 {
1235 controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1236 nRunLength -= 15;
1237 cRawBytes = 0;
1238 }
1239 }
1240 else
1241 {
1242 controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1243 nRunLength = 0;
1244 cRawBytes = 0;
1245 }
1246 }
1247 else
1248 {
1249 controlByte = PLANAR_CONTROL_BYTE(0, 15);
1250 cRawBytes -= 15;
1251 }
1252
1253 if (outBufferSize < 1)
1254 return 0;
1255
1256 outBufferSize--;
1257 *pOutput = controlByte;
1258 pOutput++;
1259 nBytesToWrite = (controlByte >> 4);
1260
1261 if (nBytesToWrite)
1262 {
1263 if (outBufferSize < nBytesToWrite)
1264 return 0;
1265
1266 outBufferSize -= nBytesToWrite;
1267 CopyMemory(pOutput, pInput, nBytesToWrite);
1268 pOutput += nBytesToWrite;
1269 pInput += nBytesToWrite;
1270 }
1271 }
1272
1273 while (nRunLength)
1274 {
1275 if (nRunLength > 47)
1276 {
1277 if (nRunLength < 50)
1278 {
1279 controlByte = PLANAR_CONTROL_BYTE(2, 13);
1280 nRunLength -= 45;
1281 }
1282 else
1283 {
1284 controlByte = PLANAR_CONTROL_BYTE(2, 15);
1285 nRunLength -= 47;
1286 }
1287 }
1288 else if (nRunLength > 31)
1289 {
1290 controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1291 nRunLength = 0;
1292 }
1293 else if (nRunLength > 15)
1294 {
1295 controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1296 nRunLength = 0;
1297 }
1298 else
1299 {
1300 controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1301 nRunLength = 0;
1302 }
1303
1304 if (outBufferSize < 1)
1305 return 0;
1306
1307 --outBufferSize;
1308 *pOutput = controlByte;
1309 pOutput++;
1310 }
1311
1312 const intptr_t diff = (pOutput - pOutBuffer);
1313 if ((diff < 0) || (diff > UINT32_MAX))
1314 return 0;
1315 return (UINT32)diff;
1316}
1317
1318static inline UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1319 UINT32 inBufferSize,
1320 BYTE* WINPR_RESTRICT pOutBuffer,
1321 UINT32 outBufferSize)
1322{
1323 BYTE symbol = 0;
1324 const BYTE* pInput = pInBuffer;
1325 BYTE* pOutput = pOutBuffer;
1326 const BYTE* pBytes = nullptr;
1327 UINT32 cRawBytes = 0;
1328 UINT32 nRunLength = 0;
1329 UINT32 nBytesWritten = 0;
1330 UINT32 nTotalBytesWritten = 0;
1331
1332 if (!outBufferSize)
1333 return 0;
1334
1335 do
1336 {
1337 if (!inBufferSize)
1338 break;
1339
1340 const UINT32 bSymbolMatch = (symbol == *pInput) != 0;
1341 symbol = *pInput;
1342 pInput++;
1343 inBufferSize--;
1344
1345 if (nRunLength && !bSymbolMatch)
1346 {
1347 if (nRunLength < 3)
1348 {
1349 cRawBytes += nRunLength;
1350 nRunLength = 0;
1351 }
1352 else
1353 {
1354 pBytes = pInput - (cRawBytes + nRunLength + 1);
1355 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1356 pOutput, outBufferSize);
1357 nRunLength = 0;
1358
1359 if (!nBytesWritten || (nBytesWritten > outBufferSize))
1360 return nRunLength;
1361
1362 nTotalBytesWritten += nBytesWritten;
1363 outBufferSize -= nBytesWritten;
1364 pOutput += nBytesWritten;
1365 cRawBytes = 0;
1366 }
1367 }
1368
1369 nRunLength += bSymbolMatch;
1370 cRawBytes += (!bSymbolMatch) != 0;
1371 } while (outBufferSize);
1372
1373 if (cRawBytes || nRunLength)
1374 {
1375 pBytes = pInput - (cRawBytes + nRunLength);
1376 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1377 pOutput, outBufferSize);
1378
1379 if (!nBytesWritten)
1380 return 0;
1381
1382 nTotalBytesWritten += nBytesWritten;
1383 }
1384
1385 if (inBufferSize)
1386 return 0;
1387
1388 return nTotalBytesWritten;
1389}
1390
1391BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1392 UINT32 height, BYTE* WINPR_RESTRICT outPlane,
1393 UINT32* WINPR_RESTRICT dstSize)
1394{
1395 if (!outPlane)
1396 return FALSE;
1397
1398 UINT32 index = 0;
1399 const BYTE* pInput = inPlane;
1400 BYTE* pOutput = outPlane;
1401 UINT32 outBufferSize = *dstSize;
1402 UINT32 nTotalBytesWritten = 0;
1403
1404 while (outBufferSize > 0)
1405 {
1406 const UINT32 nBytesWritten =
1407 freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1408
1409 if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1410 return FALSE;
1411
1412 outBufferSize -= nBytesWritten;
1413 nTotalBytesWritten += nBytesWritten;
1414 pOutput += nBytesWritten;
1415 pInput += width;
1416 index++;
1417
1418 if (index >= height)
1419 break;
1420 }
1421
1422 *dstSize = nTotalBytesWritten;
1423 return TRUE;
1424}
1425
1426static inline BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
1427 UINT32 width, UINT32 height,
1428 BYTE* WINPR_RESTRICT outPlanes,
1429 UINT32* WINPR_RESTRICT dstSizes,
1430 BOOL skipAlpha)
1431{
1432 UINT32 outPlanesSize = width * height * 4;
1433
1434 /* AlphaPlane */
1435 if (skipAlpha)
1436 {
1437 dstSizes[0] = 0;
1438 }
1439 else
1440 {
1441 dstSizes[0] = outPlanesSize;
1442
1443 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1444 &dstSizes[0]))
1445 return FALSE;
1446
1447 outPlanes += dstSizes[0];
1448 outPlanesSize -= dstSizes[0];
1449 }
1450
1451 /* LumaOrRedPlane */
1452 dstSizes[1] = outPlanesSize;
1453
1454 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1455 &dstSizes[1]))
1456 return FALSE;
1457
1458 outPlanes += dstSizes[1];
1459 outPlanesSize -= dstSizes[1];
1460 /* OrangeChromaOrGreenPlane */
1461 dstSizes[2] = outPlanesSize;
1462
1463 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1464 &dstSizes[2]))
1465 return FALSE;
1466
1467 outPlanes += dstSizes[2];
1468 outPlanesSize -= dstSizes[2];
1469 /* GreenChromeOrBluePlane */
1470 dstSizes[3] = outPlanesSize;
1471
1472 return (freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1473 &dstSizes[3]));
1474}
1475
1476BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1477 UINT32 height, BYTE* WINPR_RESTRICT outPlane)
1478{
1479 if (!outPlane)
1480 {
1481 if (width * height == 0)
1482 return nullptr;
1483
1484 outPlane = (BYTE*)calloc(height, width);
1485 if (!outPlane)
1486 return nullptr;
1487 }
1488
1489 // first line is copied as is
1490 CopyMemory(outPlane, inPlane, width);
1491
1492 for (UINT32 y = 1; y < height; y++)
1493 {
1494 const size_t off = 1ull * width * y;
1495 BYTE* outPtr = &outPlane[off];
1496 const BYTE* srcPtr = &inPlane[off];
1497 const BYTE* prevLinePtr = &inPlane[off - width];
1498 for (UINT32 x = 0; x < width; x++)
1499 {
1500 const int delta = (int)srcPtr[x] - (int)prevLinePtr[x];
1501 const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1;
1502 const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i);
1503 const uint32_t s2c =
1504 (s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1);
1505 outPtr[x] = (BYTE)s2c;
1506 }
1507 }
1508
1509 return outPlane;
1510}
1511
1512static inline BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
1513 UINT32 width, UINT32 height,
1514 BYTE* WINPR_RESTRICT outPlanes[4])
1515{
1516 for (UINT32 i = 0; i < 4; i++)
1517 {
1518 outPlanes[i] =
1519 freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1520
1521 if (!outPlanes[i])
1522 return FALSE;
1523 }
1524
1525 return TRUE;
1526}
1527
1528BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1529 const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
1530 UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
1531 UINT32* WINPR_RESTRICT pDstSize)
1532{
1533 UINT32 size = 0;
1534 BYTE* dstp = nullptr;
1535 UINT32 dstSizes[4] = WINPR_C_ARRAY_INIT;
1536 BYTE FormatHeader = 0;
1537
1538 if (!context || !context->rlePlanesBuffer)
1539 return nullptr;
1540
1541 if (context->AllowSkipAlpha)
1542 FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1543
1544 const UINT32 planeSize = width * height;
1545
1546 if (!context->AllowSkipAlpha)
1547 format = planar_invert_format(context, TRUE, format);
1548
1549 if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
1550 context->planes))
1551 return nullptr;
1552
1553 if (context->AllowRunLengthEncoding)
1554 {
1555 if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1556 context->deltaPlanes))
1557 return nullptr;
1558
1559 if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1560 context->rlePlanesBuffer, dstSizes,
1561 context->AllowSkipAlpha))
1562 return nullptr;
1563
1564 {
1565 uint32_t offset = 0;
1566 FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1567 context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1568 offset += dstSizes[0];
1569 context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1570 offset += dstSizes[1];
1571 context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1572 offset += dstSizes[2];
1573 context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1574
1575#if defined(WITH_DEBUG_CODECS)
1576 WLog_DBG(TAG,
1577 "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
1578 " / %" PRIu32 "] ",
1579 dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
1580#endif
1581 }
1582 }
1583
1584 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1585 {
1586 if (!context->AllowRunLengthEncoding)
1587 return nullptr;
1588
1589 if (context->rlePlanes[0] == nullptr)
1590 return nullptr;
1591
1592 if (context->rlePlanes[1] == nullptr)
1593 return nullptr;
1594
1595 if (context->rlePlanes[2] == nullptr)
1596 return nullptr;
1597
1598 if (context->rlePlanes[3] == nullptr)
1599 return nullptr;
1600 }
1601
1602 if (!dstData)
1603 {
1604 size = 1;
1605
1606 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1607 {
1608 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1609 size += dstSizes[0];
1610 else
1611 size += planeSize;
1612 }
1613
1614 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1615 size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1616 else
1617 size += (planeSize * 3);
1618
1619 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1620 size++;
1621
1622 dstData = malloc(size);
1623
1624 if (!dstData)
1625 return nullptr;
1626
1627 *pDstSize = size;
1628 }
1629
1630 dstp = dstData;
1631 *dstp = FormatHeader; /* FormatHeader */
1632 dstp++;
1633
1634 /* AlphaPlane */
1635
1636 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1637 {
1638 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1639 {
1640 CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1641 dstp += dstSizes[0];
1642 }
1643 else
1644 {
1645 CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1646 dstp += planeSize;
1647 }
1648 }
1649
1650 /* LumaOrRedPlane */
1651
1652 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1653 {
1654 CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1655 dstp += dstSizes[1];
1656 }
1657 else
1658 {
1659 CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1660 dstp += planeSize;
1661 }
1662
1663 /* OrangeChromaOrGreenPlane */
1664
1665 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1666 {
1667 CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1668 dstp += dstSizes[2];
1669 }
1670 else
1671 {
1672 CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1673 dstp += planeSize;
1674 }
1675
1676 /* GreenChromeOrBluePlane */
1677
1678 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1679 {
1680 CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1681 dstp += dstSizes[3];
1682 }
1683 else
1684 {
1685 CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1686 dstp += planeSize;
1687 }
1688
1689 /* Pad1 (1 byte) */
1690
1691 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1692 {
1693 *dstp = 0;
1694 dstp++;
1695 }
1696
1697 const intptr_t diff = (dstp - dstData);
1698 if ((diff < 0) || (diff > UINT32_MAX))
1699 {
1700 free(dstData);
1701 return nullptr;
1702 }
1703 size = (UINT32)diff;
1704 *pDstSize = size;
1705 return dstData;
1706}
1707
1708BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1709 UINT32 width, UINT32 height)
1710{
1711 if (!context)
1712 return FALSE;
1713
1714 context->bgr = FALSE;
1715 context->maxWidth = PLANAR_ALIGN(width, 4);
1716 context->maxHeight = PLANAR_ALIGN(height, 4);
1717 {
1718 const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
1719 if (tmp > UINT32_MAX)
1720 return FALSE;
1721 context->maxPlaneSize = (UINT32)tmp;
1722 }
1723
1724 if (context->maxWidth > UINT32_MAX / 4)
1725 return FALSE;
1726 context->nTempStep = context->maxWidth * 4;
1727
1728 memset((void*)context->planes, 0, sizeof(context->planes));
1729 memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes));
1730 memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes));
1731
1732 if (context->maxPlaneSize > 0)
1733 {
1734 void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32);
1735 if (!tmp)
1736 return FALSE;
1737 context->planesBuffer = tmp;
1738
1739 tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32);
1740 if (!tmp)
1741 return FALSE;
1742 context->pTempData = tmp;
1743
1744 tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32);
1745 if (!tmp)
1746 return FALSE;
1747 context->deltaPlanesBuffer = tmp;
1748
1749 tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32);
1750 if (!tmp)
1751 return FALSE;
1752 context->rlePlanesBuffer = tmp;
1753
1754 context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
1755 context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
1756 context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
1757 context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
1758 context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
1759 context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
1760 context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
1761 context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
1762 }
1763 return TRUE;
1764}
1765
1766BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1767 UINT32 maxHeight)
1768{
1769 BITMAP_PLANAR_CONTEXT* context =
1770 (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);
1771
1772 if (!context)
1773 return nullptr;
1774
1775 if (flags & PLANAR_FORMAT_HEADER_NA)
1776 context->AllowSkipAlpha = TRUE;
1777
1778 if (flags & PLANAR_FORMAT_HEADER_RLE)
1779 context->AllowRunLengthEncoding = TRUE;
1780
1781 if (flags & PLANAR_FORMAT_HEADER_CS)
1782 context->AllowColorSubsampling = TRUE;
1783
1784 context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1785
1786 if (context->ColorLossLevel)
1787 context->AllowDynamicColorFidelity = TRUE;
1788
1789 if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1790 {
1791 WINPR_PRAGMA_DIAG_PUSH
1792 WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1793 freerdp_bitmap_planar_context_free(context);
1794 WINPR_PRAGMA_DIAG_POP
1795 return nullptr;
1796 }
1797
1798 return context;
1799}
1800
1801void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1802{
1803 if (!context)
1804 return;
1805
1806 winpr_aligned_free(context->pTempData);
1807 winpr_aligned_free(context->planesBuffer);
1808 winpr_aligned_free(context->deltaPlanesBuffer);
1809 winpr_aligned_free(context->rlePlanesBuffer);
1810 winpr_aligned_free(context);
1811}
1812
1813void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
1814{
1815 WINPR_ASSERT(planar);
1816 planar->bgr = bgr;
1817}
1818
1819void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
1820{
1821 WINPR_ASSERT(planar);
1822 planar->topdown = topdown;
1823}
primitives_t
Definition primitives.h:255