FreeRDP
Loading...
Searching...
No Matches
h264_videotoolbox.m
1
18#include <freerdp/config.h>
19
20#include <stdlib.h>
21#include <string.h>
22
23#include <CoreMedia/CoreMedia.h>
24#include <CoreVideo/CoreVideo.h>
25#include <VideoToolbox/VideoToolbox.h>
26
27#include <winpr/assert.h>
28#include <freerdp/log.h>
29
30#include "h264.h"
31
32typedef struct
33{
34 BYTE *data;
35 size_t size;
36 UINT32 spsId;
37} H264_VT_PARAMETER_SET;
38
39typedef struct
40{
41 VTDecompressionSessionRef session;
42 CMVideoFormatDescriptionRef format;
43 CVPixelBufferRef image;
44 BOOL imageLocked;
45 BOOL parametersChanged;
46 BOOL needsIdr;
47 UINT32 ppsId;
48 OSType pixelFormat;
49 H264_VT_PARAMETER_SET sps[32];
50 H264_VT_PARAMETER_SET pps[256];
51} H264_CONTEXT_VIDEOTOOLBOX;
52
53typedef struct
54{
55 const BYTE *data;
56 size_t size;
57 size_t offset;
58 UINT32 bits;
59 UINT32 byte;
60 UINT32 zeros;
61} H264_VT_BITS;
62
63typedef struct
64{
65 CVPixelBufferRef image;
66 OSStatus status;
67 UINT32 count;
68} H264_VT_OUTPUT;
69
70WINPR_ATTR_MALLOC(free, 1)
71static char *osstatus2str(OSStatus status)
72{
73 char *str = nullptr;
74 size_t len = 0;
75 (void)winpr_asprintf(&str, &len, "OSStatus %ld", status);
76 return str;
77}
78
79/* Read only the RBSP identifiers needed to select the matching SPS/PPS. VideoToolbox
80 * parses the complete parameter sets and slices. Keep emulation-prevention handling
81 * here as the identifiers are Exp-Golomb coded, not necessarily byte aligned. */
82WINPR_ATTR_NODISCARD
83static BOOL videotoolbox_read_bit(H264_VT_BITS *bits, UINT32 *value)
84{
85 WINPR_ASSERT(bits);
86 WINPR_ASSERT(value);
87 WINPR_ASSERT(bits->data);
88
89 if (bits->bits == 0)
90 {
91 if (bits->offset >= bits->size)
92 return FALSE;
93 UINT32 byte = bits->data[bits->offset++];
94 if ((bits->zeros == 2) && (byte == 3))
95 {
96 if ((bits->offset >= bits->size) || (bits->data[bits->offset] > 3))
97 return FALSE;
98 byte = bits->data[bits->offset++];
99 bits->zeros = 0;
100 }
101 bits->zeros = (byte == 0) ? MIN(bits->zeros + 1, 2) : 0;
102 bits->byte = byte;
103 bits->bits = 8;
104 }
105 *value = (bits->byte >> --bits->bits) & 1;
106 return TRUE;
107}
108
109WINPR_ATTR_NODISCARD
110static BOOL videotoolbox_read_ue(H264_VT_BITS *bits, UINT32 *value)
111{
112 WINPR_ASSERT(bits);
113 WINPR_ASSERT(value);
114
115 UINT32 bit = 0;
116 UINT32 zeros = 0;
117 while (TRUE)
118 {
119 if (!videotoolbox_read_bit(bits, &bit))
120 return FALSE;
121 if (bit != 0)
122 break;
123 if (++zeros > 31)
124 return FALSE;
125 }
126 UINT32 result = 1;
127 for (UINT32 x = 0; x < zeros; x++)
128 {
129 if (!videotoolbox_read_bit(bits, &bit))
130 return FALSE;
131 result = (result << 1) | bit;
132 }
133 *value = result - 1;
134 return TRUE;
135}
136
137WINPR_ATTR_NODISCARD
138static size_t videotoolbox_start_code(const BYTE *data, size_t size, size_t *offset)
139{
140 WINPR_ASSERT(data);
141 WINPR_ASSERT(offset);
142
143 for (; (*offset < size) && ((size - *offset) >= 3); (*offset)++)
144 {
145 const BYTE *p = &data[*offset];
146 if ((p[0] != 0) || (p[1] != 0))
147 continue;
148 if (p[2] == 1)
149 return 3;
150 if (((size - *offset) >= 4) && (p[2] == 0) && (p[3] == 1))
151 return 4;
152 }
153 *offset = size;
154 return 0;
155}
156
157WINPR_ATTR_NODISCARD
158static BOOL videotoolbox_next_nal(const BYTE *data, size_t size, size_t *offset, const BYTE **nal,
159 size_t *length)
160{
161 WINPR_ASSERT(data);
162 WINPR_ASSERT(offset);
163 WINPR_ASSERT(nal);
164 WINPR_ASSERT(length);
165
166 const size_t previous = *offset;
167 const size_t prefix = videotoolbox_start_code(data, size, offset);
168 if (prefix == 0)
169 return FALSE;
170 /* Annex B permits leading_zero_8bits, but not arbitrary leading bytes. */
171 for (size_t x = previous; x < *offset; x++)
172 {
173 if (data[x] != 0)
174 return FALSE;
175 }
176 *offset += prefix;
177 const size_t start = *offset;
178 (void)videotoolbox_start_code(data, size, offset);
179 size_t end = *offset;
180 while ((end > start) && (data[end - 1] == 0))
181 end--;
182 if ((end == start) || ((data[start] & 0x80) != 0))
183 return FALSE;
184 *nal = &data[start];
185 *length = end - start;
186 return TRUE;
187}
188
189WINPR_ATTR_NODISCARD
190static BOOL videotoolbox_save_parameter(H264_CONTEXT_VIDEOTOOLBOX *sys, const BYTE *nal,
191 size_t length)
192{
193 WINPR_ASSERT(sys);
194 WINPR_ASSERT(nal);
195
196 H264_VT_BITS bits = { .data = nal + 1, .size = length - 1 };
197 UINT32 id = 0;
198 UINT32 spsId = 0;
199 H264_VT_PARAMETER_SET *parameter = nullptr;
200 if ((nal[0] & 0x1F) == 7)
201 {
202 /* profile_idc, constraint flags and level_idc precede seq_parameter_set_id. */
203 for (UINT32 x = 0; x < 24; x++)
204 {
205 if (!videotoolbox_read_bit(&bits, &id))
206 return FALSE;
207 }
208 if (!videotoolbox_read_ue(&bits, &id) || (id >= ARRAYSIZE(sys->sps)))
209 return FALSE;
210 parameter = &sys->sps[id];
211 }
212 else
213 {
214 if (!videotoolbox_read_ue(&bits, &id) || (id >= ARRAYSIZE(sys->pps)) ||
215 !videotoolbox_read_ue(&bits, &spsId) || (spsId >= ARRAYSIZE(sys->sps)))
216 return FALSE;
217 parameter = &sys->pps[id];
218 }
219 if ((parameter->size == length) && (memcmp(parameter->data, nal, length) == 0))
220 return TRUE;
221 BYTE *copy = malloc(length);
222 if (!copy)
223 return FALSE;
224 memcpy(copy, nal, length);
225 free(parameter->data);
226 parameter->data = copy;
227 parameter->size = length;
228 parameter->spsId = spsId;
229 sys->parametersChanged = TRUE;
230 return TRUE;
231}
232
233static void videotoolbox_release_image(H264_CONTEXT *h264)
234{
235 WINPR_ASSERT(h264);
236
237 H264_CONTEXT_VIDEOTOOLBOX *sys = h264->pSystemData;
238 WINPR_ASSERT(sys);
239
240 if (sys->image)
241 {
242 if (sys->imageLocked)
243 CVPixelBufferUnlockBaseAddress(sys->image, kCVPixelBufferLock_ReadOnly);
244 CVPixelBufferRelease(sys->image);
245 sys->image = nullptr;
246 }
247 sys->imageLocked = FALSE;
248 for (size_t x = 0; x < 3; x++)
249 {
250 h264->pYUVData[x] = nullptr;
251 h264->iStride[x] = 0;
252 }
253 h264->YUVWidth = 0;
254 h264->YUVHeight = 0;
255}
256
257static void videotoolbox_destroy_session(H264_CONTEXT *h264)
258{
259 WINPR_ASSERT(h264);
260
261 H264_CONTEXT_VIDEOTOOLBOX *sys = h264->pSystemData;
262 WINPR_ASSERT(sys);
263
264 if (sys->session)
265 {
266 VTDecompressionSessionInvalidate(sys->session);
267 CFRelease(sys->session);
268 sys->session = nullptr;
269 }
270 if (sys->format)
271 {
272 CFRelease(sys->format);
273 sys->format = nullptr;
274 }
275 sys->needsIdr = TRUE;
276 h264->hwAccel = FALSE;
277}
278
279static void videotoolbox_output(WINPR_ATTR_UNUSED void *context, void *sourceFrame, OSStatus status,
280 WINPR_ATTR_UNUSED VTDecodeInfoFlags flags, CVImageBufferRef image,
281 WINPR_ATTR_UNUSED CMTime pts, WINPR_ATTR_UNUSED CMTime duration)
282{
283 WINPR_ASSERT(sourceFrame);
284 /* context is unused; image may be NULL on failure or when a frame is dropped. */
285 H264_VT_OUTPUT *output = sourceFrame;
286 output->count++;
287 output->status = status;
288 if (image)
289 {
290 if (output->image)
291 CVPixelBufferRelease(output->image);
292 output->image = CVPixelBufferRetain(image);
293 }
294}
295
296WINPR_ATTR_NODISCARD
297static BOOL videotoolbox_ensure_session(H264_CONTEXT *h264, UINT32 ppsId)
298{
299 WINPR_ASSERT(h264);
300
301 H264_CONTEXT_VIDEOTOOLBOX *sys = h264->pSystemData;
302 WINPR_ASSERT(sys);
303
304 if (sys->session && !sys->parametersChanged && (sys->ppsId == ppsId))
305 return TRUE;
306 const H264_VT_PARAMETER_SET *pps = &sys->pps[ppsId];
307 const H264_VT_PARAMETER_SET *sps = &sys->sps[pps->spsId];
308 if (!pps->data || !sps->data)
309 return FALSE;
310 const BYTE *parameters[] = { sps->data, pps->data };
311 const size_t lengths[] = { sps->size, pps->size };
312 CMVideoFormatDescriptionRef format = nullptr;
313 OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
314 kCFAllocatorDefault, ARRAYSIZE(parameters), parameters, lengths, 4, &format);
315 if (status != noErr)
316 {
317 char *str = osstatus2str(status);
318 WLog_Print(h264->log, WLOG_WARN, "VideoToolbox H.264 format creation failed: %s", str);
319 free(str);
320 return FALSE;
321 }
322 /* Preserve sample values, including auxiliary AVC444 chroma encoded in luma.
323 * Do not ask VideoToolbox to scale, convert to RGB, or change the video range. */
324 const BOOL fullRange =
325 CMFormatDescriptionGetExtension(format, kCMFormatDescriptionExtension_FullRangeVideo) ==
326 kCFBooleanTrue;
327 const OSType pixelFormat = fullRange ? kCVPixelFormatType_420YpCbCr8PlanarFullRange
328 : kCVPixelFormatType_420YpCbCr8Planar;
329 if (sys->session && (sys->pixelFormat == pixelFormat) &&
330 VTDecompressionSessionCanAcceptFormatDescription(sys->session, format))
331 {
332 CFRelease(sys->format);
333 sys->format = format;
334 sys->parametersChanged = FALSE;
335 sys->ppsId = ppsId;
336 return TRUE;
337 }
338
339 videotoolbox_destroy_session(h264);
340 CFNumberRef pixel = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat);
341 if (!pixel)
342 {
343 CFRelease(format);
344 return FALSE;
345 }
346 const void *keys[] = { kCVPixelBufferPixelFormatTypeKey };
347 const void *values[] = { pixel };
348 CFDictionaryRef attributes =
349 CFDictionaryCreate(kCFAllocatorDefault, keys, values, ARRAYSIZE(keys),
350 &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
351 CFRelease(pixel);
352 if (!attributes)
353 {
354 CFRelease(format);
355 return FALSE;
356 }
357 const VTDecompressionOutputCallbackRecord callback = { videotoolbox_output, nullptr };
358 /* A NULL decoder specification leaves hardware/software selection to Apple.
359 * SoftwareGdi and H264_CONTEXT_OPTION_HW_ACCEL deliberately do not control it. */
360 status = VTDecompressionSessionCreate(kCFAllocatorDefault, format, nullptr, attributes,
361 &callback, &sys->session);
362 CFRelease(attributes);
363 if (status != noErr)
364 {
365 char *str = osstatus2str(status);
366 WLog_Print(h264->log, WLOG_WARN, "VideoToolbox H.264 session creation failed: %s", str);
367 free(str);
368 CFRelease(format);
369 return FALSE;
370 }
371 sys->format = format;
372 sys->pixelFormat = pixelFormat;
373 sys->ppsId = ppsId;
374 sys->parametersChanged = FALSE;
375 /* RealTime is a scheduling hint, not permission to drop RDP subframes. */
376 (void)VTSessionSetProperty(sys->session, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
377 if (__builtin_available(macOS 10.9, iOS 17.0, tvOS 17.0, *))
378 {
379 CFTypeRef hardware = nullptr;
380 status = VTSessionCopyProperty(
381 sys->session, kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder,
382 kCFAllocatorDefault, &hardware);
383 if (status == noErr)
384 {
385 h264->hwAccel = (hardware == kCFBooleanTrue);
386 WLog_Print(h264->log, WLOG_DEBUG, "VideoToolbox H.264 decoder: %s",
387 h264->hwAccel ? "hardware" : "software");
388 }
389 if (hardware)
390 CFRelease(hardware);
391 }
392 return TRUE;
393}
394
395WINPR_ATTR_NODISCARD
396static BOOL videotoolbox_map_image(H264_CONTEXT *h264)
397{
398 WINPR_ASSERT(h264);
399
400 H264_CONTEXT_VIDEOTOOLBOX *sys = h264->pSystemData;
401 WINPR_ASSERT(sys);
402
403 CVPixelBufferRef image = sys->image;
404 WINPR_ASSERT(image);
405
406 const size_t width = CVPixelBufferGetWidth(image);
407 const size_t height = CVPixelBufferGetHeight(image);
408 if ((width == 0) || (height == 0) || (width > UINT32_MAX) || (height > UINT32_MAX) ||
409 (CVPixelBufferGetPixelFormatType(image) != sys->pixelFormat) ||
410 (CVPixelBufferGetPlaneCount(image) != 3))
411 return FALSE;
412 if (CVPixelBufferLockBaseAddress(image, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess)
413 return FALSE;
414 sys->imageLocked = TRUE;
415 for (size_t x = 0; x < 3; x++)
416 {
417 const size_t stride = CVPixelBufferGetBytesPerRowOfPlane(image, x);
418 const size_t planeWidth = (x == 0) ? width : (width + 1) / 2;
419 const size_t planeHeight = (x == 0) ? height : (height + 1) / 2;
420 BYTE *data = CVPixelBufferGetBaseAddressOfPlane(image, x);
421 if (!data || (stride < planeWidth) || (stride > UINT32_MAX) ||
422 (CVPixelBufferGetWidthOfPlane(image, x) < planeWidth) ||
423 (CVPixelBufferGetHeightOfPlane(image, x) < planeHeight))
424 return FALSE;
425 h264->pYUVData[x] = data;
426 h264->iStride[x] = (UINT32)stride;
427 }
428 h264->YUVWidth = (UINT32)width;
429 h264->YUVHeight = (UINT32)height;
430 return TRUE;
431}
432
433WINPR_ATTR_NODISCARD
434static int videotoolbox_decompress(H264_CONTEXT *WINPR_RESTRICT h264,
435 const BYTE *WINPR_RESTRICT data, UINT32 size)
436{
437 WINPR_ASSERT(h264);
438 WINPR_ASSERT(data || (size == 0));
439
440 H264_CONTEXT_VIDEOTOOLBOX *sys = h264->pSystemData;
441 WINPR_ASSERT(sys);
442
443 CMBlockBufferRef block = nullptr;
444 CMSampleBufferRef sample = nullptr;
445 H264_VT_OUTPUT output = { 0 };
446 int result = -1;
447 size_t sampleSize = 0;
448 size_t offset = 0;
449 UINT32 ppsId = UINT32_MAX;
450 BOOL idr = FALSE;
451 videotoolbox_release_image(h264);
452 if (size == 0)
453 return 0;
454 while (offset < size)
455 {
456 const BYTE *nal = nullptr;
457 size_t length = 0;
458 if (!videotoolbox_next_nal(data, size, &offset, &nal, &length))
459 goto fail;
460 const BYTE type = nal[0] & 0x1F;
461 if ((type == 7) || (type == 8))
462 {
463 if (!videotoolbox_save_parameter(sys, nal, length))
464 goto fail;
465 continue;
466 }
467 if ((type == 1) || (type == 5))
468 {
469 H264_VT_BITS bits = { .data = nal + 1, .size = length - 1 };
470 UINT32 value = 0;
471 UINT32 slicePps = 0;
472 if (!videotoolbox_read_ue(&bits, &value) || !videotoolbox_read_ue(&bits, &value) ||
473 !videotoolbox_read_ue(&bits, &slicePps) || (slicePps >= ARRAYSIZE(sys->pps)))
474 goto fail;
475 if ((ppsId != UINT32_MAX) && ((ppsId != slicePps) || (idr != (type == 5))))
476 goto fail;
477 ppsId = slicePps;
478 idr = (type == 5);
479 }
480 else if ((type == 9) || (type == 10) || (type == 11) || (type == 12))
481 continue;
482 else if (type != 6) /* Only ordinary AVC slices and SEI are submitted. */
483 goto fail;
484 if ((length > UINT32_MAX - 4) || (sampleSize > UINT32_MAX - length - 4))
485 goto fail;
486 sampleSize += length + 4;
487 }
488 /* Parameter sets may arrive separately from the first picture (MS-RDPEVOR). */
489 if (ppsId == UINT32_MAX)
490 return 0;
491 if (!videotoolbox_ensure_session(h264, ppsId))
492 goto fail;
493 if (sys->needsIdr && !idr)
494 {
495 WLog_Print(h264->log, WLOG_WARN, "VideoToolbox H.264 decoder requires an IDR picture");
496 goto fail;
497 }
498 OSStatus status = CMBlockBufferCreateWithMemoryBlock(
499 kCFAllocatorDefault, nullptr, sampleSize, kCFAllocatorDefault, nullptr, 0, sampleSize,
500 kCMBlockBufferAssureMemoryNowFlag, &block);
501 if (status != noErr)
502 goto fail;
503 char *bytes = nullptr;
504 status = CMBlockBufferGetDataPointer(block, 0, nullptr, nullptr, &bytes);
505 if ((status != noErr) || !bytes)
506 goto fail;
507 offset = 0;
508 size_t written = 0;
509 while (offset < size)
510 {
511 const BYTE *nal = nullptr;
512 size_t length = 0;
513 if (!videotoolbox_next_nal(data, size, &offset, &nal, &length))
514 goto fail;
515 const BYTE type = nal[0] & 0x1F;
516 if ((type != 1) && (type != 5) && (type != 6))
517 continue;
518 bytes[written++] = (char)((length >> 24) & 0xFF);
519 bytes[written++] = (char)((length >> 16) & 0xFF);
520 bytes[written++] = (char)((length >> 8) & 0xFF);
521 bytes[written++] = (char)(length & 0xFF);
522 memcpy(&bytes[written], nal, length);
523 written += length;
524 }
525 WINPR_ASSERT(written == sampleSize);
526 status = CMSampleBufferCreateReady(kCFAllocatorDefault, block, sys->format, 1, 0, nullptr, 1,
527 &sampleSize, &sample);
528 if (status != noErr)
529 goto fail;
530 /* Neither asynchronous nor temporal processing is enabled: the callback completes
531 * before this returns. AVC444 feeds main/auxiliary pictures through the SAME session,
532 * and FreeRDP must consume each output before submitting the following subframe. */
533 status = VTDecompressionSessionDecodeFrame(sys->session, sample, 0, &output, nullptr);
534 if ((status != noErr) || (output.status != noErr) || (output.count != 1) || !output.image)
535 {
536 char *str1 = osstatus2str(status);
537 char *str2 = osstatus2str(output.status);
538 WLog_Print(h264->log, WLOG_WARN,
539 "VideoToolbox H.264 decode failed: %s / %s, outputs=%" PRIu32, str1, str2,
540 output.count);
541 free(str1);
542 free(str2);
543 goto fail;
544 }
545 sys->image = output.image;
546 output.image = nullptr;
547 if (!videotoolbox_map_image(h264))
548 goto fail;
549 sys->needsIdr = FALSE;
550 result = 1;
551fail:
552 if (output.image)
553 CVPixelBufferRelease(output.image);
554 if (sample)
555 CFRelease(sample);
556 if (block)
557 CFRelease(block);
558 if (result < 0)
559 {
560 /* A skipped/dropped reference picture makes subsequent P pictures unsafe. Keep
561 * cached parameter sets, but require an IDR rather than displaying stale pixels. */
562 videotoolbox_release_image(h264);
563 videotoolbox_destroy_session(h264);
564 }
565 return result;
566}
567
568/* The encoder: VideoToolbox compresses NV12 into AVCC (length-prefixed NAL units, the
569 * parameter sets kept in the format description); RDP's AVC420 carries an Annex B stream, so
570 * the lengths become start codes and SPS/PPS go in front of every keyframe. */
571typedef struct
572{
573 VTCompressionSessionRef session;
574 UINT32 width;
575 UINT32 height;
576 UINT32 frameRate;
577 UINT32 bitRate;
578 UINT32 qp;
579 H264_RATECONTROL_MODE rateControl;
580 BOOL fullRange;
581 INT64 frameIndex;
582 BYTE *data;
583 size_t size;
584 size_t length;
585 OSStatus status;
586 BOOL failed;
587 wLog *log;
588} H264_CONTEXT_VT_ENCODER;
589
590/* full range: FreeRDP's RGB to YUV conversion uses the whole 0..255 range
591 * video range: FreeRDP's camera path supplies video-range YUV */
592WINPR_ATTR_NODISCARD
593static BOOL videotoolbox_encoder_full_range(const H264_CONTEXT *h264)
594{
595 WINPR_ASSERT(h264);
596 switch (h264->UsageType)
597 {
598 case H264_CAMERA_VIDEO_REAL_TIME:
599 case H264_CAMERA_VIDEO_NON_REAL_TIME:
600 return FALSE;
601 default:
602 return TRUE;
603 }
604}
605
606WINPR_ATTR_NODISCARD
607static BOOL videotoolbox_encoder_append(H264_CONTEXT_VT_ENCODER *enc, const void *data, size_t size)
608{
609 WINPR_ASSERT(enc);
610
611 if (size > SIZE_MAX - enc->length)
612 return FALSE;
613 const size_t need = enc->length + size;
614 if (need > enc->size)
615 {
616 size_t newSize = MAX(need, 4096);
617 if (enc->size <= SIZE_MAX / 2)
618 newSize = MAX(newSize, enc->size * 2);
619 BYTE *tmp = realloc(enc->data, newSize);
620 if (!tmp)
621 return FALSE;
622 enc->data = tmp;
623 enc->size = newSize;
624 }
625 if (size > 0)
626 memcpy(&enc->data[enc->length], data, size);
627 enc->length = need;
628 return TRUE;
629}
630
631WINPR_ATTR_NODISCARD
632static BOOL videotoolbox_encoder_append_nal(H264_CONTEXT_VT_ENCODER *enc, const void *nal,
633 size_t size)
634{
635 static const BYTE startCode[] = { 0x00, 0x00, 0x00, 0x01 };
636 return videotoolbox_encoder_append(enc, startCode, sizeof(startCode)) &&
637 videotoolbox_encoder_append(enc, nal, size);
638}
639
640WINPR_ATTR_NODISCARD
641static BOOL videotoolbox_encoder_is_keyframe(CMSampleBufferRef sample)
642{
643 /* a sample without attachments is a sync sample */
644 CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample, false);
645 if (!attachments || (CFArrayGetCount(attachments) < 1))
646 return TRUE;
647 CFDictionaryRef dict = CFArrayGetValueAtIndex(attachments, 0);
648 const void *notSync = nullptr;
649 if (dict && CFDictionaryGetValueIfPresent(dict, kCMSampleAttachmentKey_NotSync, &notSync) &&
650 notSync && CFBooleanGetValue((CFBooleanRef)notSync))
651 return FALSE;
652 return TRUE;
653}
654
655static void videotoolbox_encoder_fail(H264_CONTEXT_VT_ENCODER *enc, OSStatus status,
656 const char *what)
657{
658 WINPR_ASSERT(enc);
659 enc->failed = TRUE;
660 enc->status = status;
661 char *str = osstatus2str(status);
662 WLog_Print(enc->log, WLOG_ERROR, "VideoToolbox H.264 encoder: %s failed: %s", what, str);
663 free(str);
664}
665
666static void videotoolbox_encoder_output(void *refcon, WINPR_ATTR_UNUSED void *frameRefcon,
667 OSStatus status, VTEncodeInfoFlags flags,
668 CMSampleBufferRef sample)
669{
670 H264_CONTEXT_VT_ENCODER *enc = refcon;
671 WINPR_ASSERT(enc);
672
673 if (status != noErr)
674 {
675 videotoolbox_encoder_fail(enc, status, "encoding a frame");
676 return;
677 }
678 /* a dropped frame has no sample: nothing to send */
679 if (!sample || ((flags & kVTEncodeInfo_FrameDropped) != 0))
680 return;
681
682 int nalHeaderLength = 4;
683 CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sample);
684 if (format)
685 {
686 size_t count = 0;
687 OSStatus rc = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
688 format, 0, nullptr, nullptr, &count, &nalHeaderLength);
689 if (rc != noErr)
690 {
691 videotoolbox_encoder_fail(enc, rc, "reading the parameter sets");
692 return;
693 }
694 if (videotoolbox_encoder_is_keyframe(sample))
695 {
696 for (size_t x = 0; x < count; x++)
697 {
698 const uint8_t *set = nullptr;
699 size_t setSize = 0;
700 rc = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, x, &set, &setSize,
701 nullptr, nullptr);
702 if (rc != noErr)
703 {
704 videotoolbox_encoder_fail(enc, rc, "reading a parameter set");
705 return;
706 }
707 if (!videotoolbox_encoder_append_nal(enc, set, setSize))
708 {
709 videotoolbox_encoder_fail(enc, kVTAllocationFailedErr,
710 "copying a parameter set");
711 return;
712 }
713 }
714 }
715 }
716 if ((nalHeaderLength < 1) || (nalHeaderLength > 4))
717 {
718 videotoolbox_encoder_fail(enc, kVTParameterErr, "the NAL length size");
719 return;
720 }
721
722 CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample);
723 if (!block)
724 {
725 videotoolbox_encoder_fail(enc, kVTParameterErr, "getting the frame data");
726 return;
727 }
728
729 size_t total = 0;
730 size_t contiguous = 0;
731 char *ptr = nullptr;
732 BYTE *copy = nullptr;
733 OSStatus rc = CMBlockBufferGetDataPointer(block, 0, &contiguous, &total, &ptr);
734 if ((rc == noErr) && (contiguous < total))
735 {
736 /* not one piece: copy it */
737 copy = malloc(total);
738 if (!copy)
739 rc = kVTAllocationFailedErr;
740 else
741 rc = CMBlockBufferCopyDataBytes(block, 0, total, copy);
742 ptr = (char *)copy;
743 }
744 if (rc != noErr)
745 {
746 videotoolbox_encoder_fail(enc, rc, "reading the frame data");
747 free(copy);
748 return;
749 }
750
751 const BYTE *data = (const BYTE *)ptr;
752 const size_t header = (size_t)nalHeaderLength;
753 size_t offset = 0;
754 while (total - offset > header)
755 {
756 size_t nalSize = 0;
757 for (size_t x = 0; x < header; x++)
758 nalSize = (nalSize << 8) | data[offset + x];
759 offset += header;
760 if (nalSize > total - offset)
761 {
762 videotoolbox_encoder_fail(enc, kVTParameterErr, "a NAL unit longer than the frame");
763 break;
764 }
765 if (!videotoolbox_encoder_append_nal(enc, &data[offset], nalSize))
766 {
767 videotoolbox_encoder_fail(enc, kVTAllocationFailedErr, "copying a NAL unit");
768 break;
769 }
770 offset += nalSize;
771 }
772 free(copy);
773}
774
775static void videotoolbox_encoder_destroy(H264_CONTEXT_VT_ENCODER *enc)
776{
777 WINPR_ASSERT(enc);
778
779 if (enc->session)
780 {
781 (void)VTCompressionSessionCompleteFrames(enc->session, kCMTimeInvalid);
782 VTCompressionSessionInvalidate(enc->session);
783 CFRelease(enc->session);
784 enc->session = nullptr;
785 }
786}
787
788static BOOL videotoolbox_encoder_set(H264_CONTEXT_VT_ENCODER *enc, CFStringRef key, CFTypeRef value,
789 const char *name)
790{
791 WINPR_ASSERT(enc);
792 WINPR_ASSERT(name);
793
794 if (!value)
795 return FALSE;
796 const OSStatus rc = VTSessionSetProperty(enc->session, key, value);
797 if (rc != noErr)
798 {
799 char *str = osstatus2str(rc);
800 WLog_Print(enc->log, WLOG_DEBUG, "VideoToolbox H.264 encoder: %s not set: %s", name, str);
801 free(str);
802 return FALSE;
803 }
804 return TRUE;
805}
806
807static BOOL videotoolbox_encoder_set_int(H264_CONTEXT_VT_ENCODER *enc, CFStringRef key,
808 int32_t value, const char *name)
809{
810 CFNumberRef number = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
811 const BOOL rc = videotoolbox_encoder_set(enc, key, number, name);
812 if (number)
813 CFRelease(number);
814 return rc;
815}
816
817static BOOL videotoolbox_encoder_set_float(H264_CONTEXT_VT_ENCODER *enc, CFStringRef key,
818 float value, const char *name)
819{
820 CFNumberRef number = CFNumberCreate(kCFAllocatorDefault, kCFNumberFloat32Type, &value);
821 const BOOL rc = videotoolbox_encoder_set(enc, key, number, name);
822 if (number)
823 CFRelease(number);
824 return rc;
825}
826
827WINPR_ATTR_NODISCARD
828static BOOL videotoolbox_encoder_set_qp(H264_CONTEXT_VT_ENCODER *enc, UINT32 qp)
829{
830 const int32_t value = (int32_t)MIN(qp, 51);
831 if (__builtin_available(macOS 13.0, iOS 16.0, *))
832 {
833 /* a constant QP: the same lower and upper bound */
834 if (videotoolbox_encoder_set_int(enc, kVTCompressionPropertyKey_MinAllowedFrameQP, value,
835 "MinAllowedFrameQP") &&
836 videotoolbox_encoder_set_int(enc, kVTCompressionPropertyKey_MaxAllowedFrameQP, value,
837 "MaxAllowedFrameQP"))
838 return TRUE;
839 }
840 /* otherwise the 0..1 quality scale (1 best), from the QP scale (0..51, 0 best) */
841 return videotoolbox_encoder_set_float(enc, kVTCompressionPropertyKey_Quality,
842 1.0f - (1.0f * (float)value / 51.0f), "Quality");
843}
844
845WINPR_ATTR_NODISCARD
846static BOOL videotoolbox_encoder_create(H264_CONTEXT *h264, H264_CONTEXT_VT_ENCODER *enc,
847 UINT32 width, UINT32 height, UINT32 frameRate,
848 UINT32 bitRate)
849{
850 WINPR_ASSERT(h264);
851 WINPR_ASSERT(enc);
852
853 BOOL rc = FALSE;
854 CFMutableDictionaryRef source = nullptr;
855 CFMutableDictionaryRef surface = nullptr;
856 CFNumberRef number = nullptr;
857
858 videotoolbox_encoder_destroy(enc);
859 if ((width > INT32_MAX) || (height > INT32_MAX) || (frameRate > INT32_MAX))
860 return FALSE;
861
862 source = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks,
863 &kCFTypeDictionaryValueCallBacks);
864 surface = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks,
865 &kCFTypeDictionaryValueCallBacks);
866 const BOOL fullRange = videotoolbox_encoder_full_range(h264);
867 const int32_t pixelFormat = fullRange ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
868 : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
869 number = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat);
870 if (!source || !surface || !number)
871 goto fail;
872 CFDictionarySetValue(source, kCVPixelBufferPixelFormatTypeKey, number);
873 /* IOSurface backed buffers go to the hardware encoder without a copy */
874 CFDictionarySetValue(source, kCVPixelBufferIOSurfacePropertiesKey, surface);
875
876 OSStatus status = VTCompressionSessionCreate(
877 kCFAllocatorDefault, (int32_t)width, (int32_t)height, kCMVideoCodecType_H264, nullptr,
878 source, kCFAllocatorDefault, videotoolbox_encoder_output, enc, &enc->session);
879 if (status != noErr)
880 {
881 char *str = osstatus2str(status);
882 WLog_Print(h264->log, WLOG_WARN,
883 "VideoToolbox H.264 encoder: VTCompressionSessionCreate %" PRIu32 "x%" PRIu32
884 " failed: %s",
885 width, height, str);
886 free(str);
887 enc->session = nullptr;
888 goto fail;
889 }
890
891 /* every frame goes out at once: no frame reordering (B frames would hold frames back),
892 * real time scheduling */
893 (void)videotoolbox_encoder_set(enc, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue,
894 "RealTime");
895 if (!videotoolbox_encoder_set(enc, kVTCompressionPropertyKey_AllowFrameReordering,
896 kCFBooleanFalse, "AllowFrameReordering"))
897 goto fail;
898 /* Baseline, as OpenH264 sends: every RDP client decodes it */
899 (void)videotoolbox_encoder_set(enc, kVTCompressionPropertyKey_ProfileLevel,
900 kVTProfileLevel_H264_Baseline_AutoLevel, "ProfileLevel");
901 (void)videotoolbox_encoder_set_int(enc, kVTCompressionPropertyKey_MaxKeyFrameInterval, 30,
902 "MaxKeyFrameInterval");
903 (void)videotoolbox_encoder_set_int(enc, kVTCompressionPropertyKey_ExpectedFrameRate,
904 (int32_t)frameRate, "ExpectedFrameRate");
905
906 switch (h264->RateControlMode)
907 {
908 case H264_RATECONTROL_CQP:
909 if (!videotoolbox_encoder_set_qp(enc, h264->QP))
910 goto fail;
911 break;
912 case H264_RATECONTROL_VBR:
913 default:
914 if (!videotoolbox_encoder_set_int(enc, kVTCompressionPropertyKey_AverageBitRate,
915 (int32_t)MIN(bitRate, INT32_MAX), "AverageBitRate"))
916 goto fail;
917 break;
918 }
919
920 status = VTCompressionSessionPrepareToEncodeFrames(enc->session);
921 if (status != noErr)
922 {
923 char *str = osstatus2str(status);
924 WLog_Print(h264->log, WLOG_WARN,
925 "VideoToolbox H.264 encoder: VTCompressionSessionPrepareToEncodeFrames failed: "
926 "%s",
927 str);
928 free(str);
929 goto fail;
930 }
931
932 /* only logged: to yuv_ensure_buffer h264->hwAccel means NV12 YUV buffers, and this encoder
933 * takes the I420 buffers FreeRDP fills */
934 BOOL hardware = FALSE;
935 if (__builtin_available(macOS 10.9, iOS 17.4, tvOS 17.4, *))
936 {
937 CFTypeRef usingHardware = nullptr;
938 if ((VTSessionCopyProperty(enc->session,
939 kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder,
940 kCFAllocatorDefault, (void *)&usingHardware) == noErr) &&
941 usingHardware)
942 {
943 hardware = (usingHardware == kCFBooleanTrue) ? TRUE : FALSE;
944 CFRelease(usingHardware);
945 }
946 }
947
948 enc->width = width;
949 enc->height = height;
950 enc->frameRate = frameRate;
951 enc->bitRate = bitRate;
952 enc->qp = h264->QP;
953 enc->rateControl = h264->RateControlMode;
954 enc->fullRange = fullRange;
955 enc->frameIndex = 0;
956 WLog_Print(
957 h264->log, WLOG_DEBUG,
958 "VideoToolbox H.264 encoder: %" PRIu32 "x%" PRIu32 " at %" PRIu32 " fps, %s, hardware %s",
959 width, height, frameRate,
960 (h264->RateControlMode == H264_RATECONTROL_CQP) ? "constant QP" : "variable bit rate",
961 hardware ? "yes" : "no");
962 rc = TRUE;
963
964fail:
965 if (number)
966 CFRelease(number);
967 if (surface)
968 CFRelease(surface);
969 if (source)
970 CFRelease(source);
971 if (!rc)
972 videotoolbox_encoder_destroy(enc);
973 return rc;
974}
975
976WINPR_ATTR_NODISCARD
977static BOOL videotoolbox_encoder_update(H264_CONTEXT *h264, H264_CONTEXT_VT_ENCODER *enc,
978 UINT32 width, UINT32 height, UINT32 frameRate,
979 UINT32 bitRate)
980{
981 WINPR_ASSERT(h264);
982 WINPR_ASSERT(enc);
983
984 if (!enc->session || (enc->width != width) || (enc->height != height) ||
985 (enc->frameRate != frameRate) || (enc->rateControl != h264->RateControlMode) ||
986 (enc->fullRange != videotoolbox_encoder_full_range(h264)))
987 return videotoolbox_encoder_create(h264, enc, width, height, frameRate, bitRate);
988
989 /* bit rate and QP can change while the session runs; start over where they can't */
990 if ((h264->RateControlMode != H264_RATECONTROL_CQP) && (enc->bitRate != bitRate))
991 {
992 if (!videotoolbox_encoder_set_int(enc, kVTCompressionPropertyKey_AverageBitRate,
993 (int32_t)MIN(bitRate, INT32_MAX), "AverageBitRate"))
994 return videotoolbox_encoder_create(h264, enc, width, height, frameRate, bitRate);
995 enc->bitRate = bitRate;
996 }
997 if ((h264->RateControlMode == H264_RATECONTROL_CQP) && (enc->qp != h264->QP))
998 {
999 if (!videotoolbox_encoder_set_qp(enc, h264->QP))
1000 return videotoolbox_encoder_create(h264, enc, width, height, frameRate, bitRate);
1001 enc->qp = h264->QP;
1002 }
1003 return TRUE;
1004}
1005
1006/* I420 planes (FreeRDP's YUV buffers) into the NV12 picture the encoder takes */
1007WINPR_ATTR_NODISCARD
1008static BOOL videotoolbox_encoder_copy(CVPixelBufferRef image, const BYTE **WINPR_RESTRICT pSrcYuv,
1009 const UINT32 *WINPR_RESTRICT pStride, UINT32 width,
1010 UINT32 height)
1011{
1012 if (CVPixelBufferLockBaseAddress(image, 0) != kCVReturnSuccess)
1013 return FALSE;
1014
1015 BOOL rc = FALSE;
1016 BYTE *luma = CVPixelBufferGetBaseAddressOfPlane(image, 0);
1017 BYTE *chroma = CVPixelBufferGetBaseAddressOfPlane(image, 1);
1018 const size_t lumaStride = CVPixelBufferGetBytesPerRowOfPlane(image, 0);
1019 const size_t chromaStride = CVPixelBufferGetBytesPerRowOfPlane(image, 1);
1020 if (!luma || !chroma || (lumaStride < width) || (chromaStride < width) ||
1021 (CVPixelBufferGetHeightOfPlane(image, 0) < height) ||
1022 (CVPixelBufferGetHeightOfPlane(image, 1) < height / 2))
1023 goto out;
1024
1025 for (UINT32 y = 0; y < height; y++)
1026 memcpy(&luma[y * lumaStride], &pSrcYuv[0][1ull * y * pStride[0]], width);
1027 for (UINT32 y = 0; y < height / 2; y++)
1028 {
1029 const BYTE *u = &pSrcYuv[1][1ull * y * pStride[1]];
1030 const BYTE *v = &pSrcYuv[2][1ull * y * pStride[2]];
1031 BYTE *uv = &chroma[y * chromaStride];
1032 for (UINT32 x = 0; x < width / 2; x++)
1033 {
1034 uv[2 * x] = u[x];
1035 uv[2 * x + 1] = v[x];
1036 }
1037 }
1038 rc = TRUE;
1039out:
1040 (void)CVPixelBufferUnlockBaseAddress(image, 0);
1041 return rc;
1042}
1043
1044WINPR_ATTR_NODISCARD
1045static int videotoolbox_compress(H264_CONTEXT *WINPR_RESTRICT h264,
1046 const BYTE **WINPR_RESTRICT pSrcYuv,
1047 const UINT32 *WINPR_RESTRICT pStride,
1048 BYTE **WINPR_RESTRICT ppDstData, UINT32 *WINPR_RESTRICT pDstSize)
1049{
1050 WINPR_ASSERT(h264);
1051 WINPR_ASSERT(pSrcYuv);
1052 WINPR_ASSERT(pStride);
1053 WINPR_ASSERT(ppDstData);
1054 WINPR_ASSERT(pDstSize);
1055
1056 H264_CONTEXT_VT_ENCODER *enc = h264->pSystemData;
1057 if (!enc || !pSrcYuv[0] || !pSrcYuv[1] || !pSrcYuv[2])
1058 return -1;
1059 if ((h264->width == 0) || (h264->height == 0) || (h264->width > UINT16_MAX) ||
1060 (h264->height > UINT16_MAX))
1061 return -1;
1062
1063 /* 4:2:0 needs even sizes; the YUV buffers are padded (yuv_ensure_buffer), so one more
1064 * column or row is readable */
1065 const UINT32 width = (h264->width + 1) & ~1u;
1066 const UINT32 height = (h264->height + 1) & ~1u;
1067 const UINT32 frameRate = (h264->FrameRate > 0) ? MIN(h264->FrameRate, 1000) : 30;
1068 const UINT32 bitRate = (h264->BitRate > 0) ? h264->BitRate : 1000000;
1069
1070 if (!videotoolbox_encoder_update(h264, enc, width, height, frameRate, bitRate))
1071 return -1;
1072
1073 CVPixelBufferRef image = nullptr;
1074 CVPixelBufferPoolRef pool = VTCompressionSessionGetPixelBufferPool(enc->session);
1075 const OSType pixelFormat = enc->fullRange ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
1076 : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1077 const CVReturn cv = pool ? CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &image)
1078 : CVPixelBufferCreate(kCFAllocatorDefault, width, height, pixelFormat,
1079 nullptr, &image);
1080 if ((cv != kCVReturnSuccess) || !image)
1081 {
1082 WLog_Print(h264->log, WLOG_ERROR, "VideoToolbox H.264 encoder: no picture buffer: %d",
1083 (int)cv);
1084 return -1;
1085 }
1086 if (!videotoolbox_encoder_copy(image, pSrcYuv, pStride, width, height))
1087 {
1088 CVPixelBufferRelease(image);
1089 return -1;
1090 }
1091
1092 enc->length = 0;
1093 enc->failed = FALSE;
1094 enc->status = noErr;
1095 const CMTime pts = CMTimeMake(enc->frameIndex++, (int32_t)frameRate);
1096 const CMTime duration = CMTimeMake(1, (int32_t)frameRate);
1097 OSStatus status = VTCompressionSessionEncodeFrame(enc->session, image, pts, duration, nullptr,
1098 nullptr, nullptr);
1099 CVPixelBufferRelease(image);
1100 if (status == noErr)
1101 /* the output callback has run for this frame when this returns */
1102 status = VTCompressionSessionCompleteFrames(enc->session, pts);
1103 if ((status != noErr) || enc->failed)
1104 {
1105 char *str1 = osstatus2str(status);
1106 char *str2 = osstatus2str(enc->status);
1107 WLog_Print(h264->log, WLOG_ERROR, "VideoToolbox H.264 encoder: frame failed: %s / %s", str1,
1108 str2);
1109 free(str1);
1110 free(str2);
1111 /* the next frame starts a new session, with a keyframe */
1112 videotoolbox_encoder_destroy(enc);
1113 return -1;
1114 }
1115
1116 if ((enc->length == 0) || (enc->length > UINT32_MAX))
1117 return 0;
1118
1119 *ppDstData = enc->data;
1120 *pDstSize = (UINT32)enc->length;
1121 return 1;
1122}
1123
1124static void videotoolbox_encoder_uninit(H264_CONTEXT *h264)
1125{
1126 WINPR_ASSERT(h264);
1127
1128 H264_CONTEXT_VT_ENCODER *enc = h264->pSystemData;
1129 if (!enc)
1130 return;
1131 videotoolbox_encoder_destroy(enc);
1132 free(enc->data);
1133 free(enc);
1134 h264->pSystemData = nullptr;
1135}
1136
1137WINPR_ATTR_NODISCARD
1138static BOOL videotoolbox_encoder_init(H264_CONTEXT *h264)
1139{
1140 WINPR_ASSERT(h264);
1141
1142 H264_CONTEXT_VT_ENCODER *enc = calloc(1, sizeof(*enc));
1143 if (!enc)
1144 return FALSE;
1145 enc->log = h264->log;
1146 h264->pSystemData = enc;
1147
1148 /* a session for the real size is made with the first frame; one small session now tells
1149 * whether this system has an H.264 encoder at all, so that registration can go on to
1150 * another backend if not */
1151 if (!videotoolbox_encoder_create(h264, enc, 64, 64, 30, 1000000))
1152 {
1153 videotoolbox_encoder_uninit(h264);
1154 return FALSE;
1155 }
1156 videotoolbox_encoder_destroy(enc);
1157 return TRUE;
1158}
1159
1160static void videotoolbox_uninit(H264_CONTEXT *h264)
1161{
1162 WINPR_ASSERT(h264);
1163
1164 if (h264->Compressor)
1165 {
1166 videotoolbox_encoder_uninit(h264);
1167 return;
1168 }
1169
1170 H264_CONTEXT_VIDEOTOOLBOX *sys = h264->pSystemData;
1171 if (!sys)
1172 return;
1173 videotoolbox_release_image(h264);
1174 videotoolbox_destroy_session(h264);
1175 for (size_t x = 0; x < ARRAYSIZE(sys->sps); x++)
1176 free(sys->sps[x].data);
1177 for (size_t x = 0; x < ARRAYSIZE(sys->pps); x++)
1178 free(sys->pps[x].data);
1179 free(sys);
1180 h264->pSystemData = nullptr;
1181}
1182
1183WINPR_ATTR_NODISCARD
1184static BOOL videotoolbox_init(H264_CONTEXT *h264)
1185{
1186 WINPR_ASSERT(h264);
1187
1188 if (h264->Compressor)
1189 return videotoolbox_encoder_init(h264);
1190
1191 H264_CONTEXT_VIDEOTOOLBOX *sys = calloc(1, sizeof(*sys));
1192 if (!sys)
1193 return FALSE;
1194 sys->needsIdr = TRUE;
1195 h264->pSystemData = sys;
1196 h264->hwAccel = FALSE;
1197 /* The actual session is created lazily when the stream supplies SPS/PPS. */
1198 return TRUE;
1199}
1200
1201const H264_CONTEXT_SUBSYSTEM g_Subsystem_VideoToolbox = { "VideoToolbox", videotoolbox_init,
1202 videotoolbox_uninit,
1203 videotoolbox_decompress,
1204 videotoolbox_compress };