FreeRDP
Loading...
Searching...
No Matches
h264_videotoolbox.c
1
18#include <freerdp/config.h>
19
20#include <stdlib.h>
21#include <string.h>
22
23#include <CoreMedia/CoreMedia.h>
24#include <CoreVideo/CoreVideo.h>
25#include <VideoToolbox/VideoToolbox.h>
26
27#include <winpr/assert.h>
28#include <freerdp/log.h>
29
30#include "h264.h"
31
32typedef struct
33{
34 BYTE* data;
35 size_t size;
36 UINT32 spsId;
37} H264_VT_PARAMETER_SET;
38
39typedef struct
40{
41 VTDecompressionSessionRef session;
42 CMVideoFormatDescriptionRef format;
43 CVPixelBufferRef image;
44 BOOL imageLocked;
45 BOOL parametersChanged;
46 BOOL needsIdr;
47 UINT32 ppsId;
48 OSType pixelFormat;
49 H264_VT_PARAMETER_SET sps[32];
50 H264_VT_PARAMETER_SET pps[256];
51} H264_CONTEXT_VIDEOTOOLBOX;
52
53typedef struct
54{
55 const BYTE* data;
56 size_t size;
57 size_t offset;
58 UINT32 bits;
59 UINT32 byte;
60 UINT32 zeros;
61} H264_VT_BITS;
62
63typedef struct
64{
65 CVPixelBufferRef image;
66 OSStatus status;
67 UINT32 count;
68} H264_VT_OUTPUT;
69
70/* Read only the RBSP identifiers needed to select the matching SPS/PPS. VideoToolbox
71 * parses the complete parameter sets and slices. Keep emulation-prevention handling
72 * here as the identifiers are Exp-Golomb coded, not necessarily byte aligned. */
73WINPR_ATTR_NODISCARD
74static BOOL videotoolbox_read_bit(H264_VT_BITS* bits, UINT32* value)
75{
76 WINPR_ASSERT(bits);
77 WINPR_ASSERT(value);
78 WINPR_ASSERT(bits->data);
79
80 if (bits->bits == 0)
81 {
82 if (bits->offset >= bits->size)
83 return FALSE;
84 UINT32 byte = bits->data[bits->offset++];
85 if ((bits->zeros == 2) && (byte == 3))
86 {
87 if ((bits->offset >= bits->size) || (bits->data[bits->offset] > 3))
88 return FALSE;
89 byte = bits->data[bits->offset++];
90 bits->zeros = 0;
91 }
92 bits->zeros = (byte == 0) ? MIN(bits->zeros + 1, 2) : 0;
93 bits->byte = byte;
94 bits->bits = 8;
95 }
96 *value = (bits->byte >> --bits->bits) & 1;
97 return TRUE;
98}
99
100WINPR_ATTR_NODISCARD
101static BOOL videotoolbox_read_ue(H264_VT_BITS* bits, UINT32* value)
102{
103 WINPR_ASSERT(bits);
104 WINPR_ASSERT(value);
105
106 UINT32 bit = 0;
107 UINT32 zeros = 0;
108 while (TRUE)
109 {
110 if (!videotoolbox_read_bit(bits, &bit))
111 return FALSE;
112 if (bit != 0)
113 break;
114 if (++zeros > 31)
115 return FALSE;
116 }
117 UINT32 result = 1;
118 for (UINT32 x = 0; x < zeros; x++)
119 {
120 if (!videotoolbox_read_bit(bits, &bit))
121 return FALSE;
122 result = (result << 1) | bit;
123 }
124 *value = result - 1;
125 return TRUE;
126}
127
128WINPR_ATTR_NODISCARD
129static size_t videotoolbox_start_code(const BYTE* data, size_t size, size_t* offset)
130{
131 WINPR_ASSERT(data);
132 WINPR_ASSERT(offset);
133
134 for (; (*offset < size) && ((size - *offset) >= 3); (*offset)++)
135 {
136 const BYTE* p = &data[*offset];
137 if ((p[0] != 0) || (p[1] != 0))
138 continue;
139 if (p[2] == 1)
140 return 3;
141 if (((size - *offset) >= 4) && (p[2] == 0) && (p[3] == 1))
142 return 4;
143 }
144 *offset = size;
145 return 0;
146}
147
148WINPR_ATTR_NODISCARD
149static BOOL videotoolbox_next_nal(const BYTE* data, size_t size, size_t* offset, const BYTE** nal,
150 size_t* length)
151{
152 WINPR_ASSERT(data);
153 WINPR_ASSERT(offset);
154 WINPR_ASSERT(nal);
155 WINPR_ASSERT(length);
156
157 const size_t previous = *offset;
158 const size_t prefix = videotoolbox_start_code(data, size, offset);
159 if (prefix == 0)
160 return FALSE;
161 /* Annex B permits leading_zero_8bits, but not arbitrary leading bytes. */
162 for (size_t x = previous; x < *offset; x++)
163 {
164 if (data[x] != 0)
165 return FALSE;
166 }
167 *offset += prefix;
168 const size_t start = *offset;
169 (void)videotoolbox_start_code(data, size, offset);
170 size_t end = *offset;
171 while ((end > start) && (data[end - 1] == 0))
172 end--;
173 if ((end == start) || ((data[start] & 0x80) != 0))
174 return FALSE;
175 *nal = &data[start];
176 *length = end - start;
177 return TRUE;
178}
179
180WINPR_ATTR_NODISCARD
181static BOOL videotoolbox_save_parameter(H264_CONTEXT_VIDEOTOOLBOX* sys, const BYTE* nal,
182 size_t length)
183{
184 WINPR_ASSERT(sys);
185 WINPR_ASSERT(nal);
186
187 H264_VT_BITS bits = { .data = nal + 1, .size = length - 1 };
188 UINT32 id = 0;
189 UINT32 spsId = 0;
190 H264_VT_PARAMETER_SET* parameter = nullptr;
191 if ((nal[0] & 0x1F) == 7)
192 {
193 /* profile_idc, constraint flags and level_idc precede seq_parameter_set_id. */
194 for (UINT32 x = 0; x < 24; x++)
195 {
196 if (!videotoolbox_read_bit(&bits, &id))
197 return FALSE;
198 }
199 if (!videotoolbox_read_ue(&bits, &id) || (id >= ARRAYSIZE(sys->sps)))
200 return FALSE;
201 parameter = &sys->sps[id];
202 }
203 else
204 {
205 if (!videotoolbox_read_ue(&bits, &id) || (id >= ARRAYSIZE(sys->pps)) ||
206 !videotoolbox_read_ue(&bits, &spsId) || (spsId >= ARRAYSIZE(sys->sps)))
207 return FALSE;
208 parameter = &sys->pps[id];
209 }
210 if ((parameter->size == length) && (memcmp(parameter->data, nal, length) == 0))
211 return TRUE;
212 BYTE* copy = malloc(length);
213 if (!copy)
214 return FALSE;
215 memcpy(copy, nal, length);
216 free(parameter->data);
217 parameter->data = copy;
218 parameter->size = length;
219 parameter->spsId = spsId;
220 sys->parametersChanged = TRUE;
221 return TRUE;
222}
223
224static void videotoolbox_release_image(H264_CONTEXT* h264)
225{
226 WINPR_ASSERT(h264);
227
228 H264_CONTEXT_VIDEOTOOLBOX* sys = h264->pSystemData;
229 WINPR_ASSERT(sys);
230
231 if (sys->image)
232 {
233 if (sys->imageLocked)
234 CVPixelBufferUnlockBaseAddress(sys->image, kCVPixelBufferLock_ReadOnly);
235 CVPixelBufferRelease(sys->image);
236 sys->image = nullptr;
237 }
238 sys->imageLocked = FALSE;
239 for (size_t x = 0; x < 3; x++)
240 {
241 h264->pYUVData[x] = nullptr;
242 h264->iStride[x] = 0;
243 }
244 h264->YUVWidth = 0;
245 h264->YUVHeight = 0;
246}
247
248static void videotoolbox_destroy_session(H264_CONTEXT* h264)
249{
250 WINPR_ASSERT(h264);
251
252 H264_CONTEXT_VIDEOTOOLBOX* sys = h264->pSystemData;
253 WINPR_ASSERT(sys);
254
255 if (sys->session)
256 {
257 VTDecompressionSessionInvalidate(sys->session);
258 CFRelease(sys->session);
259 sys->session = nullptr;
260 }
261 if (sys->format)
262 {
263 CFRelease(sys->format);
264 sys->format = nullptr;
265 }
266 sys->needsIdr = TRUE;
267 h264->hwAccel = FALSE;
268}
269
270static void videotoolbox_output(WINPR_ATTR_UNUSED void* context, void* sourceFrame, OSStatus status,
271 WINPR_ATTR_UNUSED VTDecodeInfoFlags flags, CVImageBufferRef image,
272 WINPR_ATTR_UNUSED CMTime pts, WINPR_ATTR_UNUSED CMTime duration)
273{
274 WINPR_ASSERT(sourceFrame);
275 /* context is unused; image may be NULL on failure or when a frame is dropped. */
276 H264_VT_OUTPUT* output = sourceFrame;
277 output->count++;
278 output->status = status;
279 if (image)
280 {
281 if (output->image)
282 CVPixelBufferRelease(output->image);
283 output->image = CVPixelBufferRetain(image);
284 }
285}
286
287WINPR_ATTR_NODISCARD
288static BOOL videotoolbox_ensure_session(H264_CONTEXT* h264, UINT32 ppsId)
289{
290 WINPR_ASSERT(h264);
291
292 H264_CONTEXT_VIDEOTOOLBOX* sys = h264->pSystemData;
293 WINPR_ASSERT(sys);
294
295 if (sys->session && !sys->parametersChanged && (sys->ppsId == ppsId))
296 return TRUE;
297 const H264_VT_PARAMETER_SET* pps = &sys->pps[ppsId];
298 const H264_VT_PARAMETER_SET* sps = &sys->sps[pps->spsId];
299 if (!pps->data || !sps->data)
300 return FALSE;
301 const BYTE* parameters[] = { sps->data, pps->data };
302 const size_t lengths[] = { sps->size, pps->size };
303 CMVideoFormatDescriptionRef format = nullptr;
304 OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
305 kCFAllocatorDefault, ARRAYSIZE(parameters), parameters, lengths, 4, &format);
306 if (status != noErr)
307 {
308 WLog_Print(h264->log, WLOG_WARN, "VideoToolbox H.264 format creation failed: %d", status);
309 return FALSE;
310 }
311 /* Preserve sample values, including auxiliary AVC444 chroma encoded in luma.
312 * Do not ask VideoToolbox to scale, convert to RGB, or change the video range. */
313 const BOOL fullRange =
314 CMFormatDescriptionGetExtension(format, kCMFormatDescriptionExtension_FullRangeVideo) ==
315 kCFBooleanTrue;
316 const OSType pixelFormat = fullRange ? kCVPixelFormatType_420YpCbCr8PlanarFullRange
317 : kCVPixelFormatType_420YpCbCr8Planar;
318 if (sys->session && (sys->pixelFormat == pixelFormat) &&
319 VTDecompressionSessionCanAcceptFormatDescription(sys->session, format))
320 {
321 CFRelease(sys->format);
322 sys->format = format;
323 sys->parametersChanged = FALSE;
324 sys->ppsId = ppsId;
325 return TRUE;
326 }
327
328 videotoolbox_destroy_session(h264);
329 CFNumberRef pixel = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat);
330 if (!pixel)
331 {
332 CFRelease(format);
333 return FALSE;
334 }
335 const void* keys[] = { kCVPixelBufferPixelFormatTypeKey };
336 const void* values[] = { pixel };
337 CFDictionaryRef attributes =
338 CFDictionaryCreate(kCFAllocatorDefault, keys, values, ARRAYSIZE(keys),
339 &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
340 CFRelease(pixel);
341 if (!attributes)
342 {
343 CFRelease(format);
344 return FALSE;
345 }
346 const VTDecompressionOutputCallbackRecord callback = { videotoolbox_output, nullptr };
347 /* A NULL decoder specification leaves hardware/software selection to Apple.
348 * SoftwareGdi and H264_CONTEXT_OPTION_HW_ACCEL deliberately do not control it. */
349 status = VTDecompressionSessionCreate(kCFAllocatorDefault, format, nullptr, attributes,
350 &callback, &sys->session);
351 CFRelease(attributes);
352 if (status != noErr)
353 {
354 WLog_Print(h264->log, WLOG_WARN, "VideoToolbox H.264 session creation failed: %d", status);
355 CFRelease(format);
356 return FALSE;
357 }
358 sys->format = format;
359 sys->pixelFormat = pixelFormat;
360 sys->ppsId = ppsId;
361 sys->parametersChanged = FALSE;
362 /* RealTime is a scheduling hint, not permission to drop RDP subframes. */
363 (void)VTSessionSetProperty(sys->session, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
364 if (__builtin_available(macOS 10.9, iOS 17.0, tvOS 17.0, *))
365 {
366 CFTypeRef hardware = nullptr;
367 status = VTSessionCopyProperty(
368 sys->session, kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder,
369 kCFAllocatorDefault, &hardware);
370 if (status == noErr)
371 {
372 h264->hwAccel = (hardware == kCFBooleanTrue);
373 WLog_Print(h264->log, WLOG_DEBUG, "VideoToolbox H.264 decoder: %s",
374 h264->hwAccel ? "hardware" : "software");
375 }
376 if (hardware)
377 CFRelease(hardware);
378 }
379 return TRUE;
380}
381
382WINPR_ATTR_NODISCARD
383static BOOL videotoolbox_map_image(H264_CONTEXT* h264)
384{
385 WINPR_ASSERT(h264);
386
387 H264_CONTEXT_VIDEOTOOLBOX* sys = h264->pSystemData;
388 WINPR_ASSERT(sys);
389
390 CVPixelBufferRef image = sys->image;
391 WINPR_ASSERT(image);
392
393 const size_t width = CVPixelBufferGetWidth(image);
394 const size_t height = CVPixelBufferGetHeight(image);
395 if ((width == 0) || (height == 0) || (width > UINT32_MAX) || (height > UINT32_MAX) ||
396 (CVPixelBufferGetPixelFormatType(image) != sys->pixelFormat) ||
397 (CVPixelBufferGetPlaneCount(image) != 3))
398 return FALSE;
399 if (CVPixelBufferLockBaseAddress(image, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess)
400 return FALSE;
401 sys->imageLocked = TRUE;
402 for (size_t x = 0; x < 3; x++)
403 {
404 const size_t stride = CVPixelBufferGetBytesPerRowOfPlane(image, x);
405 const size_t planeWidth = (x == 0) ? width : (width + 1) / 2;
406 const size_t planeHeight = (x == 0) ? height : (height + 1) / 2;
407 BYTE* data = CVPixelBufferGetBaseAddressOfPlane(image, x);
408 if (!data || (stride < planeWidth) || (stride > UINT32_MAX) ||
409 (CVPixelBufferGetWidthOfPlane(image, x) < planeWidth) ||
410 (CVPixelBufferGetHeightOfPlane(image, x) < planeHeight))
411 return FALSE;
412 h264->pYUVData[x] = data;
413 h264->iStride[x] = (UINT32)stride;
414 }
415 h264->YUVWidth = (UINT32)width;
416 h264->YUVHeight = (UINT32)height;
417 return TRUE;
418}
419
420WINPR_ATTR_NODISCARD
421static int videotoolbox_decompress(H264_CONTEXT* WINPR_RESTRICT h264,
422 const BYTE* WINPR_RESTRICT data, UINT32 size)
423{
424 WINPR_ASSERT(h264);
425 WINPR_ASSERT(data || (size == 0));
426
427 H264_CONTEXT_VIDEOTOOLBOX* sys = h264->pSystemData;
428 WINPR_ASSERT(sys);
429
430 CMBlockBufferRef block = nullptr;
431 CMSampleBufferRef sample = nullptr;
432 H264_VT_OUTPUT output = { 0 };
433 int result = -1;
434 size_t sampleSize = 0;
435 size_t offset = 0;
436 UINT32 ppsId = UINT32_MAX;
437 BOOL idr = FALSE;
438 videotoolbox_release_image(h264);
439 if (size == 0)
440 return 0;
441 while (offset < size)
442 {
443 const BYTE* nal = nullptr;
444 size_t length = 0;
445 if (!videotoolbox_next_nal(data, size, &offset, &nal, &length))
446 goto fail;
447 const BYTE type = nal[0] & 0x1F;
448 if ((type == 7) || (type == 8))
449 {
450 if (!videotoolbox_save_parameter(sys, nal, length))
451 goto fail;
452 continue;
453 }
454 if ((type == 1) || (type == 5))
455 {
456 H264_VT_BITS bits = { .data = nal + 1, .size = length - 1 };
457 UINT32 value = 0;
458 UINT32 slicePps = 0;
459 if (!videotoolbox_read_ue(&bits, &value) || !videotoolbox_read_ue(&bits, &value) ||
460 !videotoolbox_read_ue(&bits, &slicePps) || (slicePps >= ARRAYSIZE(sys->pps)))
461 goto fail;
462 if ((ppsId != UINT32_MAX) && ((ppsId != slicePps) || (idr != (type == 5))))
463 goto fail;
464 ppsId = slicePps;
465 idr = (type == 5);
466 }
467 else if ((type == 9) || (type == 10) || (type == 11) || (type == 12))
468 continue;
469 else if (type != 6) /* Only ordinary AVC slices and SEI are submitted. */
470 goto fail;
471 if ((length > UINT32_MAX - 4) || (sampleSize > UINT32_MAX - length - 4))
472 goto fail;
473 sampleSize += length + 4;
474 }
475 /* Parameter sets may arrive separately from the first picture (MS-RDPEVOR). */
476 if (ppsId == UINT32_MAX)
477 return 0;
478 if (!videotoolbox_ensure_session(h264, ppsId))
479 goto fail;
480 if (sys->needsIdr && !idr)
481 {
482 WLog_Print(h264->log, WLOG_WARN, "VideoToolbox H.264 decoder requires an IDR picture");
483 goto fail;
484 }
485 OSStatus status = CMBlockBufferCreateWithMemoryBlock(
486 kCFAllocatorDefault, nullptr, sampleSize, kCFAllocatorDefault, nullptr, 0, sampleSize,
487 kCMBlockBufferAssureMemoryNowFlag, &block);
488 if (status != noErr)
489 goto fail;
490 char* bytes = nullptr;
491 status = CMBlockBufferGetDataPointer(block, 0, nullptr, nullptr, &bytes);
492 if ((status != noErr) || !bytes)
493 goto fail;
494 offset = 0;
495 size_t written = 0;
496 while (offset < size)
497 {
498 const BYTE* nal = nullptr;
499 size_t length = 0;
500 if (!videotoolbox_next_nal(data, size, &offset, &nal, &length))
501 goto fail;
502 const BYTE type = nal[0] & 0x1F;
503 if ((type != 1) && (type != 5) && (type != 6))
504 continue;
505 bytes[written++] = (char)((length >> 24) & 0xFF);
506 bytes[written++] = (char)((length >> 16) & 0xFF);
507 bytes[written++] = (char)((length >> 8) & 0xFF);
508 bytes[written++] = (char)(length & 0xFF);
509 memcpy(&bytes[written], nal, length);
510 written += length;
511 }
512 WINPR_ASSERT(written == sampleSize);
513 status = CMSampleBufferCreateReady(kCFAllocatorDefault, block, sys->format, 1, 0, nullptr, 1,
514 &sampleSize, &sample);
515 if (status != noErr)
516 goto fail;
517 /* Neither asynchronous nor temporal processing is enabled: the callback completes
518 * before this returns. AVC444 feeds main/auxiliary pictures through the SAME session,
519 * and FreeRDP must consume each output before submitting the following subframe. */
520 status = VTDecompressionSessionDecodeFrame(sys->session, sample, 0, &output, nullptr);
521 if ((status != noErr) || (output.status != noErr) || (output.count != 1) || !output.image)
522 {
523 WLog_Print(h264->log, WLOG_WARN,
524 "VideoToolbox H.264 decode failed: %d / %d, outputs=%" PRIu32, status,
525 output.status, output.count);
526 goto fail;
527 }
528 sys->image = output.image;
529 output.image = nullptr;
530 if (!videotoolbox_map_image(h264))
531 goto fail;
532 sys->needsIdr = FALSE;
533 result = 1;
534fail:
535 if (output.image)
536 CVPixelBufferRelease(output.image);
537 if (sample)
538 CFRelease(sample);
539 if (block)
540 CFRelease(block);
541 if (result < 0)
542 {
543 /* A skipped/dropped reference picture makes subsequent P pictures unsafe. Keep
544 * cached parameter sets, but require an IDR rather than displaying stale pixels. */
545 videotoolbox_release_image(h264);
546 videotoolbox_destroy_session(h264);
547 }
548 return result;
549}
550
551static void videotoolbox_uninit(H264_CONTEXT* h264)
552{
553 WINPR_ASSERT(h264);
554
555 H264_CONTEXT_VIDEOTOOLBOX* sys = h264->pSystemData;
556 if (!sys)
557 return;
558 videotoolbox_release_image(h264);
559 videotoolbox_destroy_session(h264);
560 for (size_t x = 0; x < ARRAYSIZE(sys->sps); x++)
561 free(sys->sps[x].data);
562 for (size_t x = 0; x < ARRAYSIZE(sys->pps); x++)
563 free(sys->pps[x].data);
564 free(sys);
565 h264->pSystemData = nullptr;
566}
567
568WINPR_ATTR_NODISCARD
569static BOOL videotoolbox_init(H264_CONTEXT* h264)
570{
571 WINPR_ASSERT(h264);
572
573 /* Let registration continue to an encoder-capable backend for compressor contexts. */
574 if (h264->Compressor)
575 return FALSE;
576 H264_CONTEXT_VIDEOTOOLBOX* sys = calloc(1, sizeof(*sys));
577 if (!sys)
578 return FALSE;
579 sys->needsIdr = TRUE;
580 h264->pSystemData = sys;
581 h264->hwAccel = FALSE;
582 /* The actual session is created lazily when the stream supplies SPS/PPS. */
583 return TRUE;
584}
585
586const H264_CONTEXT_SUBSYSTEM g_Subsystem_VideoToolbox = { "VideoToolbox", videotoolbox_init,
587 videotoolbox_uninit,
588 videotoolbox_decompress, nullptr };