FreeRDP
Loading...
Searching...
No Matches
video_main.c
1
20#include <freerdp/config.h>
21
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include <winpr/crt.h>
27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/synch.h>
30#include <winpr/print.h>
31#include <winpr/stream.h>
32#include <winpr/cmdline.h>
33#include <winpr/collections.h>
34#include <winpr/interlocked.h>
35#include <winpr/sysinfo.h>
36
37#include <freerdp/freerdp.h>
38#include <freerdp/addin.h>
39#include <freerdp/primitives.h>
40#include <freerdp/client/channels.h>
41#include <freerdp/client/geometry.h>
42#include <freerdp/client/video.h>
43#include <freerdp/channels/log.h>
44#include <freerdp/codec/h264.h>
45#include <freerdp/codec/yuv.h>
46#include <freerdp/timer.h>
47
48#define TAG CHANNELS_TAG("video.client")
49
50#include "video_main.h"
51
52typedef struct
53{
54 IWTSPlugin wtsPlugin;
55
56 IWTSListener* controlListener;
57 IWTSListener* dataListener;
58 GENERIC_LISTENER_CALLBACK* control_callback;
59 GENERIC_LISTENER_CALLBACK* data_callback;
60
61 VideoClientContext* context;
62 BOOL initialized;
63 rdpContext* rdpcontext;
64} VIDEO_PLUGIN;
65
66#define XF_VIDEO_UNLIMITED_RATE 31
67
68static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
69 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
70
71typedef struct
72{
73 BYTE PresentationId;
74 UINT32 ScaledWidth;
75 UINT32 ScaledHeight;
76
77 UINT64 startTimeStamp;
78 UINT64 publishOffset;
79 wStream* currentSample;
80 UINT64 lastPublishTime;
81 UINT64 nextPublishTime;
82 volatile LONG refCounter;
83 H264_CONTEXT* h264;
84 VideoSurface* surface;
85 MAPPED_GEOMETRY* geometry;
86 VideoClientContext* video;
87} PresentationContext;
88
89typedef struct
90{
91 BYTE PresentationId;
92 UINT64 publishTime;
93 UINT64 hnsDuration;
94 MAPPED_GEOMETRY* geometry;
95 UINT32 w, h;
96 UINT32 scanline;
97 BYTE* surfaceData;
98} VideoFrame;
99
101struct s_VideoClientContextPriv
102{
103 VideoClientContext* video;
104 GeometryClientContext* geometry;
105 wQueue* frames;
106 CRITICAL_SECTION framesLock;
107 wBufferPool* surfacePool;
108 UINT32 publishedFrames;
109 UINT32 droppedFrames;
110 UINT32 lastSentRate;
111 UINT64 nextFeedbackTime;
112 PresentationContext* currentPresentation;
113 FreeRDP_TimerID timerID;
114};
115
116static void PresentationContext_unref(PresentationContext** presentation);
117static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
118
119WINPR_ATTR_NODISCARD
120static const char* video_command_name(BYTE cmd)
121{
122 switch (cmd)
123 {
124 case TSMM_START_PRESENTATION:
125 return "start";
126 case TSMM_STOP_PRESENTATION:
127 return "stop";
128 default:
129 return "<unknown>";
130 }
131}
132
133static void video_client_context_set_geometry(VideoClientContext* video,
134 GeometryClientContext* geometry)
135{
136 WINPR_ASSERT(video);
137 WINPR_ASSERT(video->priv);
138 video->priv->geometry = geometry;
139}
140
141WINPR_ATTR_MALLOC(VideoClientContextPriv_free, 1)
142static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
143{
144 VideoClientContextPriv* ret = nullptr;
145
146 WINPR_ASSERT(video);
147 ret = calloc(1, sizeof(*ret));
148 if (!ret)
149 return nullptr;
150
151 ret->frames = Queue_New(TRUE, 10, 2);
152 if (!ret->frames)
153 {
154 WLog_ERR(TAG, "unable to allocate frames queue");
155 goto fail;
156 }
157
158 ret->surfacePool = BufferPool_New(FALSE, 0, 16);
159 if (!ret->surfacePool)
160 {
161 WLog_ERR(TAG, "unable to create surface pool");
162 goto fail;
163 }
164
165 if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
166 {
167 WLog_ERR(TAG, "unable to initialize frames lock");
168 goto fail;
169 }
170
171 ret->video = video;
172
173 /* don't set to unlimited so that we have the chance to send a feedback in
174 * the first second (for servers that want feedback directly)
175 */
176 ret->lastSentRate = 30;
177 return ret;
178
179fail:
180 VideoClientContextPriv_free(ret);
181 return nullptr;
182}
183
184WINPR_ATTR_NODISCARD
185static BOOL PresentationContext_ref(PresentationContext* presentation)
186{
187 WINPR_ASSERT(presentation);
188
189 const LONG val = InterlockedIncrement(&presentation->refCounter);
190 return val > 0;
191}
192
193static void PresentationContext_free(PresentationContext* presentation)
194{
195 if (!presentation)
196 return;
197
198 MAPPED_GEOMETRY* geometry = presentation->geometry;
199 if (geometry)
200 {
201 geometry->MappedGeometryUpdate = nullptr;
202 geometry->MappedGeometryClear = nullptr;
203 geometry->custom = nullptr;
204 mappedGeometryUnref(geometry);
205 }
206
207 h264_context_free(presentation->h264);
208 Stream_Free(presentation->currentSample, TRUE);
209 presentation->video->deleteSurface(presentation->video, presentation->surface);
210 free(presentation);
211}
212
213WINPR_ATTR_MALLOC(PresentationContext_free, 1)
214static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
215 UINT32 x, UINT32 y, UINT32 width, UINT32 height)
216{
217 size_t s = 4ULL * width * height;
218 PresentationContext* ret = nullptr;
219
220 WINPR_ASSERT(video);
221
222 if (s > INT32_MAX)
223 return nullptr;
224
225 ret = calloc(1, sizeof(*ret));
226 if (!ret)
227 return nullptr;
228
229 ret->video = video;
230 ret->PresentationId = PresentationId;
231
232 ret->h264 = h264_context_new(FALSE);
233 if (!ret->h264)
234 {
235 WLog_ERR(TAG, "unable to create a h264 context");
236 goto fail;
237 }
238
239 VIDEO_PLUGIN* plugin = (VIDEO_PLUGIN*)video->handle;
240 WINPR_ASSERT(plugin);
241 WINPR_ASSERT(plugin->rdpcontext);
242 if (!h264_context_set_option(
243 ret->h264, H264_CONTEXT_OPTION_HW_ACCEL,
244 (UINT32)freerdp_settings_get_bool(plugin->rdpcontext->settings, FreeRDP_SoftwareGdi)))
245 goto fail;
246 if (!h264_context_reset(ret->h264, width, height))
247 goto fail;
248
249 ret->currentSample = Stream_New(nullptr, 4096);
250 if (!ret->currentSample)
251 {
252 WLog_ERR(TAG, "unable to create current packet stream");
253 goto fail;
254 }
255
256 ret->surface = video->createSurface(video, x, y, width, height);
257 if (!ret->surface)
258 {
259 WLog_ERR(TAG, "unable to create surface");
260 goto fail;
261 }
262
263 if (!PresentationContext_ref(ret))
264 goto fail;
265
266 return ret;
267
268fail:
269 PresentationContext_free(ret);
270 return nullptr;
271}
272
273static void PresentationContext_unref(PresentationContext** ppresentation)
274{
275 WINPR_ASSERT(ppresentation);
276
277 PresentationContext* presentation = *ppresentation;
278 if (!presentation)
279 return;
280
281 if (InterlockedDecrement(&presentation->refCounter) > 0)
282 return;
283 *ppresentation = nullptr;
284
285 PresentationContext_free(presentation);
286}
287
288static void VideoFrame_free(VideoClientContextPriv* priv, VideoFrame* frame)
289{
290 WINPR_ASSERT(priv);
291 if (!frame)
292 return;
293
294 mappedGeometryUnref(frame->geometry);
295
296 BufferPool_Return(priv->surfacePool, frame->surfaceData);
297 free(frame);
298}
299
300WINPR_ATTR_MALLOC(VideoFrame_free, 1)
301static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
302 MAPPED_GEOMETRY* geom)
303{
304 WINPR_ASSERT(priv);
305 WINPR_ASSERT(presentation);
306 WINPR_ASSERT(geom);
307
308 const VideoSurface* surface = presentation->surface;
309 WINPR_ASSERT(surface);
310
311 VideoFrame* frame = calloc(1, sizeof(VideoFrame));
312 if (!frame)
313 goto fail;
314 frame->PresentationId = presentation->PresentationId;
315
316 mappedGeometryRef(geom);
317
318 frame->publishTime = presentation->lastPublishTime;
319 frame->geometry = geom;
320 frame->w = surface->alignedWidth;
321 frame->h = surface->alignedHeight;
322 frame->scanline = surface->scanline;
323
324 frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
325 if (!frame->surfaceData)
326 goto fail;
327
328 return frame;
329
330fail:
331 VideoFrame_free(priv, frame);
332 return nullptr;
333}
334
335void VideoClientContextPriv_free(VideoClientContextPriv* priv)
336{
337 if (!priv)
338 return;
339
340 EnterCriticalSection(&priv->framesLock);
341
342 if (priv->frames)
343 {
344 while (Queue_Count(priv->frames))
345 {
346 VideoFrame* frame = Queue_Dequeue(priv->frames);
347 if (frame)
348 VideoFrame_free(priv, frame);
349 }
350 }
351
352 Queue_Free(priv->frames);
353 LeaveCriticalSection(&priv->framesLock);
354
355 DeleteCriticalSection(&priv->framesLock);
356
357 if (priv->currentPresentation)
358 PresentationContext_unref(&priv->currentPresentation);
359
360 BufferPool_Free(priv->surfacePool);
361 free(priv);
362}
363
364WINPR_ATTR_NODISCARD
365static UINT video_channel_write(VIDEO_PLUGIN* video, const BYTE* data, UINT32 length)
366{
367 WINPR_ASSERT(video);
368
369 if (!video->control_callback || !video->control_callback->channel_callback)
370 return ERROR_BAD_CONFIGURATION;
371 IWTSVirtualChannel* channel = video->control_callback->channel_callback->channel;
372 if (!channel || !channel->Write)
373 return ERROR_BAD_CONFIGURATION;
374 return channel->Write(channel, length, data, nullptr);
375}
376
377WINPR_ATTR_NODISCARD
378static UINT video_control_send_presentation_response(VideoClientContext* context,
379 TSMM_PRESENTATION_RESPONSE* resp)
380{
381 BYTE buf[12] = WINPR_C_ARRAY_INIT;
382
383 WINPR_ASSERT(context);
384 WINPR_ASSERT(resp);
385
386 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
387 WINPR_ASSERT(video);
388
389 wStream* s = Stream_New(buf, 12);
390 if (!s)
391 return CHANNEL_RC_NO_MEMORY;
392
393 Stream_Write_UINT32(s, 12); /* cbSize */
394 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
395 Stream_Write_UINT8(s, resp->PresentationId);
396 Stream_Zero(s, 3);
397 Stream_SealLength(s);
398 Stream_Free(s, FALSE);
399
400 return video_channel_write(video, buf, sizeof(buf));
401}
402
403WINPR_ATTR_NODISCARD
404static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
405{
406 WINPR_ASSERT(geometry);
407
408 PresentationContext* presentation = (PresentationContext*)geometry->custom;
409 WINPR_ASSERT(presentation);
410
411 RDP_RECT* r = &geometry->geometry.boundingRect;
412 WLog_DBG(TAG,
413 "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
414 ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
415 "-%" PRId16 "x%" PRId16 ")",
416 geometry->topLevelLeft, geometry->topLevelTop,
417 geometry->topLevelRight - geometry->topLevelLeft,
418 geometry->topLevelBottom - geometry->topLevelTop,
419
420 geometry->left, geometry->top, geometry->right - geometry->left,
421 geometry->bottom - geometry->top,
422
423 r->x, r->y, r->width, r->height);
424
425 WINPR_ASSERT(presentation->surface);
426 presentation->surface->x =
427 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
428 presentation->surface->y =
429 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
430
431 return TRUE;
432}
433
434WINPR_ATTR_NODISCARD
435static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
436{
437 WINPR_ASSERT(geometry);
438
439 PresentationContext* presentation = (PresentationContext*)geometry->custom;
440 WINPR_ASSERT(presentation);
441
442 mappedGeometryUnref(presentation->geometry);
443 presentation->geometry = nullptr;
444 return TRUE;
445}
446
447WINPR_ATTR_NODISCARD
448static UINT video_PresentationRequest(VideoClientContext* video,
449 const TSMM_PRESENTATION_REQUEST* req)
450{
451 UINT ret = CHANNEL_RC_OK;
452
453 WINPR_ASSERT(video);
454 WINPR_ASSERT(req);
455
456 VideoClientContextPriv* priv = video->priv;
457 WINPR_ASSERT(priv);
458
459 EnterCriticalSection(&priv->framesLock);
460 if (req->Command == TSMM_START_PRESENTATION)
461 {
462 MAPPED_GEOMETRY* geom = nullptr;
463 TSMM_PRESENTATION_RESPONSE resp = WINPR_C_ARRAY_INIT;
464
465 if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
466 {
467 WLog_ERR(TAG, "not a H264 video, ignoring request");
468 goto fail;
469 }
470
471 if (priv->currentPresentation)
472 {
473 if (priv->currentPresentation->PresentationId == req->PresentationId)
474 {
475 WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
476 req->PresentationId);
477 goto fail;
478 }
479
480 WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
481 PresentationContext_unref(&priv->currentPresentation);
482 }
483
484 if (!priv->geometry)
485 {
486 WLog_ERR(TAG, "geometry channel not ready, ignoring request");
487 goto fail;
488 }
489
490 geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
491 if (!geom)
492 {
493 WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
494 goto fail;
495 }
496
497 WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
498 priv->currentPresentation = PresentationContext_new(
499 video, req->PresentationId,
500 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
501 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
502 req->SourceHeight);
503 if (!priv->currentPresentation)
504 {
505 WLog_ERR(TAG, "unable to create presentation video");
506 ret = CHANNEL_RC_NO_MEMORY;
507 goto fail;
508 }
509
510 mappedGeometryRef(geom);
511 priv->currentPresentation->geometry = geom;
512
513 priv->currentPresentation->video = video;
514 priv->currentPresentation->ScaledWidth = req->ScaledWidth;
515 priv->currentPresentation->ScaledHeight = req->ScaledHeight;
516
517 geom->custom = priv->currentPresentation;
518 geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
519 geom->MappedGeometryClear = video_onMappedGeometryClear;
520
521 /* send back response */
522 resp.PresentationId = req->PresentationId;
523 ret = video_control_send_presentation_response(video, &resp);
524 }
525 else if (req->Command == TSMM_STOP_PRESENTATION)
526 {
527 WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
528 if (!priv->currentPresentation)
529 {
530 WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
531 goto fail;
532 }
533
534 priv->droppedFrames = 0;
535 priv->publishedFrames = 0;
536 PresentationContext_unref(&priv->currentPresentation);
537 }
538
539fail:
540 LeaveCriticalSection(&priv->framesLock);
541 return ret;
542}
543
544WINPR_ATTR_NODISCARD
545static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
546{
547 TSMM_PRESENTATION_REQUEST req = WINPR_C_ARRAY_INIT;
548
549 WINPR_ASSERT(context);
550 WINPR_ASSERT(s);
551
552 if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
553 return ERROR_INVALID_DATA;
554
555 Stream_Read_UINT8(s, req.PresentationId);
556 Stream_Read_UINT8(s, req.Version);
557 Stream_Read_UINT8(s, req.Command);
558 Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
559
560 Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
561 Stream_Seek_UINT16(s); /* reserved */
562
563 Stream_Read_UINT32(s, req.SourceWidth);
564 Stream_Read_UINT32(s, req.SourceHeight);
565 Stream_Read_UINT32(s, req.ScaledWidth);
566 Stream_Read_UINT32(s, req.ScaledHeight);
567 Stream_Read_UINT64(s, req.hnsTimestampOffset);
568 Stream_Read_UINT64(s, req.GeometryMappingId);
569 Stream_Read(s, req.VideoSubtypeId, 16);
570
571 Stream_Read_UINT32(s, req.cbExtra);
572
573 if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
574 return ERROR_INVALID_DATA;
575
576 req.pExtraData = Stream_Pointer(s);
577
578 WLog_DBG(TAG,
579 "presentationReq: id:%" PRIu8 " version:%" PRIu8
580 " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
581 "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
582 req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
583 req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
584 req.GeometryMappingId);
585
586 return video_PresentationRequest(context, &req);
587}
588
594WINPR_ATTR_NODISCARD
595static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
596{
597 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
598 UINT ret = CHANNEL_RC_OK;
599 UINT32 cbSize = 0;
600 UINT32 packetType = 0;
601
602 WINPR_ASSERT(callback);
603 WINPR_ASSERT(s);
604
605 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
606 WINPR_ASSERT(video);
607
608 VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;
609 WINPR_ASSERT(context);
610
611 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
612 return ERROR_INVALID_DATA;
613
614 Stream_Read_UINT32(s, cbSize);
615 if (cbSize < 8)
616 {
617 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
618 return ERROR_INVALID_DATA;
619 }
620 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
621 return ERROR_INVALID_DATA;
622
623 Stream_Read_UINT32(s, packetType);
624 switch (packetType)
625 {
626 case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
627 ret = video_read_tsmm_presentation_req(context, s);
628 break;
629 default:
630 WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
631 ret = ERROR_UNSUPPORTED_TYPE;
632 break;
633 }
634
635 return ret;
636}
637
638static UINT video_control_send_client_notification(VideoClientContext* context,
639 const TSMM_CLIENT_NOTIFICATION* notif)
640{
641 BYTE buf[100] = WINPR_C_ARRAY_INIT;
642
643 WINPR_ASSERT(context);
644 WINPR_ASSERT(notif);
645
646 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)context->handle;
647 WINPR_ASSERT(video);
648
649 wStream* s = Stream_New(buf, 32);
650 if (!s)
651 return CHANNEL_RC_NO_MEMORY;
652
653 UINT32 cbSize = 16;
654 Stream_Seek_UINT32(s); /* cbSize */
655 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
656 Stream_Write_UINT8(s, notif->PresentationId);
657 Stream_Write_UINT8(s, notif->NotificationType);
658 Stream_Zero(s, 2);
659 if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
660 {
661 Stream_Write_UINT32(s, 16); /* cbData */
662
663 /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
664 Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
665 Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
666 Stream_Zero(s, 4ULL * 2ULL);
667
668 cbSize += 4UL * 4UL;
669 }
670 else
671 {
672 Stream_Write_UINT32(s, 0); /* cbData */
673 }
674
675 Stream_SealLength(s);
676 Stream_ResetPosition(s);
677 Stream_Write_UINT32(s, cbSize);
678 Stream_Free(s, FALSE);
679
680 return video_channel_write(video, buf, cbSize);
681}
682
683static void video_timer(VideoClientContext* video, UINT64 now)
684{
685 VideoFrame* frame = nullptr;
686
687 WINPR_ASSERT(video);
688
689 VideoClientContextPriv* priv = video->priv;
690 WINPR_ASSERT(priv);
691
692 EnterCriticalSection(&priv->framesLock);
693 PresentationContext* presentation = video->priv->currentPresentation;
694 do
695 {
696 const VideoFrame* peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
697 if (!peekFrame)
698 break;
699
700 if (peekFrame->publishTime > now)
701 break;
702
703 if (frame)
704 {
705 WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
706 priv->droppedFrames++;
707 VideoFrame_free(priv, frame);
708 }
709 frame = Queue_Dequeue(priv->frames);
710 } while (1);
711
712 if (frame)
713 {
714 if (presentation && (presentation->PresentationId == frame->PresentationId))
715 {
716 VideoSurface* surface = presentation->surface;
717 const size_t frameSize = 1ull * frame->scanline * frame->h;
718 const size_t surfaceSize = 1ull * surface->scanline * surface->alignedHeight;
719
720 /* the presentation id is reused by the server across presentations of different
721 * sizes, so a frame queued for a previous, larger presentation can outlive it in
722 * the queue. copying it would write past the smaller surface->data buffer. */
723 if (frameSize > surfaceSize)
724 WLog_WARN(TAG, "dropping stale frame of %" PRIuz " bytes, surface holds %" PRIuz,
725 frameSize, surfaceSize);
726 else
727 {
728 priv->publishedFrames++;
729 memcpy(surface->data, frame->surfaceData, frameSize);
730
731 WINPR_ASSERT(video->showSurface);
732 if (!video->showSurface(video, surface, presentation->ScaledWidth,
733 presentation->ScaledHeight))
734 WLog_WARN(TAG, "showSurface failed");
735 }
736 }
737 VideoFrame_free(priv, frame);
738 }
739
740 if (priv->nextFeedbackTime < now)
741 {
742 /* we can compute some feedback only if we have some published frames and
743 * a current presentation
744 */
745 if (priv->publishedFrames && priv->currentPresentation)
746 {
747 UINT32 computedRate = 0;
748
749 if (!PresentationContext_ref(priv->currentPresentation))
750 WLog_WARN(TAG, "PresentationContext_ref(priv->currentPresentation) failed");
751
752 if (priv->droppedFrames)
753 {
759 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
760 computedRate = 24;
761 else
762 {
763 computedRate = priv->lastSentRate - 2;
764 if (!computedRate)
765 computedRate = 2;
766 }
767 }
768 else
769 {
774 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
775 computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
776 else
777 {
778 computedRate = priv->lastSentRate + 2;
779 if (computedRate > XF_VIDEO_UNLIMITED_RATE)
780 computedRate = XF_VIDEO_UNLIMITED_RATE;
781 }
782 }
783
784 if (computedRate != priv->lastSentRate)
785 {
786 TSMM_CLIENT_NOTIFICATION notif = WINPR_C_ARRAY_INIT;
787
788 WINPR_ASSERT(priv->currentPresentation);
789 notif.PresentationId = priv->currentPresentation->PresentationId;
790 notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
791 if (computedRate == XF_VIDEO_UNLIMITED_RATE)
792 {
793 notif.FramerateOverride.Flags = 0x01;
794 notif.FramerateOverride.DesiredFrameRate = 0x00;
795 }
796 else
797 {
798 notif.FramerateOverride.Flags = 0x02;
799 notif.FramerateOverride.DesiredFrameRate = computedRate;
800 }
801
802 video_control_send_client_notification(video, &notif);
803 priv->lastSentRate = computedRate;
804
805 WLog_VRB(TAG,
806 "server notified with rate %" PRIu32 " published=%" PRIu32
807 " dropped=%" PRIu32,
808 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
809 }
810
811 PresentationContext_unref(&priv->currentPresentation);
812 }
813
814 priv->droppedFrames = 0;
815 priv->publishedFrames = 0;
816 priv->nextFeedbackTime = now + 1000;
817 }
818 LeaveCriticalSection(&priv->framesLock);
819}
820
821WINPR_ATTR_NODISCARD
822static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
823{
824 int status = 0;
825 UINT res = CHANNEL_RC_OK;
826
827 WINPR_ASSERT(context);
828 WINPR_ASSERT(data);
829
830 VideoClientContextPriv* priv = context->priv;
831 WINPR_ASSERT(priv);
832
833 PresentationContext* presentation = priv->currentPresentation;
834 if (!presentation)
835 {
836 WLog_ERR(TAG, "no current presentation");
837 return CHANNEL_RC_OK;
838 }
839
840 if (!PresentationContext_ref(presentation))
841 return ERROR_INTERNAL_ERROR;
842
843 EnterCriticalSection(&priv->framesLock);
844 if (presentation->PresentationId != data->PresentationId)
845 {
846 WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
847 presentation->PresentationId, data->PresentationId);
848 goto out;
849 }
850
851 if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
852 {
853 WLog_ERR(TAG, "unable to expand the current packet");
854 res = CHANNEL_RC_NO_MEMORY;
855 goto out;
856 }
857
858 Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
859
860 if (data->CurrentPacketIndex == data->PacketsInSample)
861 {
862 VideoSurface* surface = presentation->surface;
863 H264_CONTEXT* h264 = presentation->h264;
864 const UINT64 startTime = winpr_GetTickCount64NS();
865 MAPPED_GEOMETRY* geom = presentation->geometry;
866
867 const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
868 WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
869 Stream_SealLength(presentation->currentSample);
870 Stream_ResetPosition(presentation->currentSample);
871
872 if (data->SampleNumber == 1)
873 {
874 presentation->lastPublishTime = startTime;
875 }
876
877 presentation->lastPublishTime += 100ull * data->hnsDuration;
878 const size_t len = Stream_Length(presentation->currentSample);
879 if (len > UINT32_MAX)
880 goto out;
881
882 BOOL enqueueResult = 0;
883 VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
884 if (!frame)
885 {
886 WLog_ERR(TAG, "unable to create frame");
887 res = CHANNEL_RC_NO_MEMORY;
888 goto out;
889 }
890
891 status = avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
892 frame->surfaceData, surface->format, surface->scanline,
893 surface->alignedWidth, surface->alignedHeight, &rect, 1);
894 if (status < 0)
895 {
896 VideoFrame_free(priv, frame);
897 goto out;
898 }
899
900 enqueueResult = Queue_Enqueue(priv->frames, frame);
901
902 if (!enqueueResult)
903 {
904 WLog_ERR(TAG, "unable to enqueue frame");
905 VideoFrame_free(priv, frame);
906 res = CHANNEL_RC_NO_MEMORY;
907 goto out;
908 }
909
910 // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
911 WLog_DBG(TAG, "scheduling frame in %" PRIu64 " ns", (frame->publishTime - startTime));
912 }
913
914out:
915 PresentationContext_unref(&priv->currentPresentation);
916 LeaveCriticalSection(&priv->framesLock);
917
918 return res;
919}
920
921WINPR_ATTR_NODISCARD
922static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
923{
924 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
925 UINT32 cbSize = 0;
926 UINT32 packetType = 0;
927 TSMM_VIDEO_DATA data;
928
929 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)callback->plugin;
930 WINPR_ASSERT(video);
931
932 VideoClientContext* context = (VideoClientContext*)video->wtsPlugin.pInterface;
933
934 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
935 return ERROR_INVALID_DATA;
936
937 Stream_Read_UINT32(s, cbSize);
938 if (cbSize < 8)
939 {
940 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
941 return ERROR_INVALID_DATA;
942 }
943
944 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
945 return ERROR_INVALID_DATA;
946
947 Stream_Read_UINT32(s, packetType);
948 if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
949 {
950 WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
951 return ERROR_INVALID_DATA;
952 }
953
954 if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
955 return ERROR_INVALID_DATA;
956
957 Stream_Read_UINT8(s, data.PresentationId);
958 Stream_Read_UINT8(s, data.Version);
959 Stream_Read_UINT8(s, data.Flags);
960 Stream_Seek_UINT8(s); /* reserved */
961 Stream_Read_UINT64(s, data.hnsTimestamp);
962 Stream_Read_UINT64(s, data.hnsDuration);
963 Stream_Read_UINT16(s, data.CurrentPacketIndex);
964 Stream_Read_UINT16(s, data.PacketsInSample);
965 Stream_Read_UINT32(s, data.SampleNumber);
966 Stream_Read_UINT32(s, data.cbSample);
967 if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
968 return ERROR_INVALID_DATA;
969 data.pSample = Stream_Pointer(s);
970
971 /*
972 WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
973 duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
974 cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
975 data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
976 data.cbSample);
977 */
978
979 return video_VideoData(context, &data);
980}
981
987WINPR_ATTR_NODISCARD
988static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
989{
990 if (pChannelCallback)
991 {
992 GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
993 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
994 if (video && video->control_callback)
995 {
996 video->control_callback->channel_callback = nullptr;
997 }
998 }
999 free(pChannelCallback);
1000 return CHANNEL_RC_OK;
1001}
1002
1003WINPR_ATTR_NODISCARD
1004static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1005{
1006 if (pChannelCallback)
1007 {
1008 GENERIC_CHANNEL_CALLBACK* listener_callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
1009 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)listener_callback->plugin;
1010 if (video && video->data_callback)
1011 {
1012 video->data_callback->channel_callback = nullptr;
1013 }
1014 }
1015 free(pChannelCallback);
1016 return CHANNEL_RC_OK;
1017}
1018
1024// NOLINTBEGIN(readability-non-const-parameter)
1025WINPR_ATTR_NODISCARD
1026static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1027 IWTSVirtualChannel* channel,
1028 WINPR_ATTR_UNUSED BYTE* Data,
1029 WINPR_ATTR_UNUSED BOOL* pbAccept,
1030 IWTSVirtualChannelCallback** ppCallback)
1031// NOLINTEND(readability-non-const-parameter)
1032{
1033 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1034
1035 GENERIC_CHANNEL_CALLBACK* callback =
1036 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1037 if (!callback)
1038 {
1039 WLog_ERR(TAG, "calloc failed!");
1040 return CHANNEL_RC_NO_MEMORY;
1041 }
1042
1043 callback->iface.OnDataReceived = video_control_on_data_received;
1044 callback->iface.OnClose = video_control_on_close;
1045 callback->plugin = listener_callback->plugin;
1046 callback->channel_mgr = listener_callback->channel_mgr;
1047 callback->channel = channel;
1048 listener_callback->channel_callback = callback;
1049
1050 *ppCallback = &callback->iface;
1051
1052 return CHANNEL_RC_OK;
1053}
1054
1055// NOLINTBEGIN(readability-non-const-parameter)
1056WINPR_ATTR_NODISCARD
1057static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1058 IWTSVirtualChannel* pChannel,
1059 WINPR_ATTR_UNUSED BYTE* Data,
1060 WINPR_ATTR_UNUSED BOOL* pbAccept,
1061 IWTSVirtualChannelCallback** ppCallback)
1062// NOLINTEND(readability-non-const-parameter)
1063{
1064 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1065
1066 GENERIC_CHANNEL_CALLBACK* callback =
1067 (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1068 if (!callback)
1069 {
1070 WLog_ERR(TAG, "calloc failed!");
1071 return CHANNEL_RC_NO_MEMORY;
1072 }
1073
1074 callback->iface.OnDataReceived = video_data_on_data_received;
1075 callback->iface.OnClose = video_data_on_close;
1076 callback->plugin = listener_callback->plugin;
1077 callback->channel_mgr = listener_callback->channel_mgr;
1078 callback->channel = pChannel;
1079 listener_callback->channel_callback = callback;
1080
1081 *ppCallback = &callback->iface;
1082
1083 return CHANNEL_RC_OK;
1084}
1085
1086WINPR_ATTR_NODISCARD
1087static uint64_t timer_cb(WINPR_ATTR_UNUSED rdpContext* context, void* userdata,
1088 WINPR_ATTR_UNUSED FreeRDP_TimerID timerID, uint64_t timestamp,
1089 uint64_t interval)
1090{
1091 VideoClientContext* video = userdata;
1092 if (!video)
1093 return 0;
1094 if (!video->timer)
1095 return 0;
1096
1097 video->timer(video, timestamp);
1098
1099 return interval;
1100}
1101
1107WINPR_ATTR_NODISCARD
1108static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1109{
1110 UINT status = 0;
1111 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1112
1113 if (video->initialized)
1114 {
1115 WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1116 return ERROR_INVALID_DATA;
1117 }
1118
1119 {
1120 GENERIC_LISTENER_CALLBACK* callback =
1121 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1122 if (!callback)
1123 {
1124 WLog_ERR(TAG, "calloc for control callback failed!");
1125 return CHANNEL_RC_NO_MEMORY;
1126 }
1127
1128 callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1129 callback->plugin = plugin;
1130 callback->channel_mgr = channelMgr;
1131
1132 status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1133 &callback->iface, &(video->controlListener));
1134 video->control_callback = callback;
1135 if (status != CHANNEL_RC_OK)
1136 return status;
1137 }
1138 video->controlListener->pInterface = video->wtsPlugin.pInterface;
1139
1140 {
1141 GENERIC_LISTENER_CALLBACK* callback =
1142 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1143 if (!callback)
1144 {
1145 WLog_ERR(TAG, "calloc for data callback failed!");
1146 return CHANNEL_RC_NO_MEMORY;
1147 }
1148
1149 callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1150 callback->plugin = plugin;
1151 callback->channel_mgr = channelMgr;
1152
1153 status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1154 &callback->iface, &(video->dataListener));
1155 video->data_callback = callback;
1156 if (status == CHANNEL_RC_OK)
1157 video->dataListener->pInterface = video->wtsPlugin.pInterface;
1158 }
1159
1160 if (status == CHANNEL_RC_OK)
1161 video->context->priv->timerID =
1162 freerdp_timer_add(video->rdpcontext, 20000000, timer_cb, video->context, true);
1163 video->initialized = video->context->priv->timerID != 0;
1164 if (!video->initialized)
1165 status = ERROR_INTERNAL_ERROR;
1166 return status;
1167}
1168
1174WINPR_ATTR_NODISCARD
1175static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1176{
1177 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1178 if (!video)
1179 return CHANNEL_RC_INVALID_INSTANCE;
1180
1181 if (video->context && video->context->priv)
1182 freerdp_timer_remove(video->rdpcontext, video->context->priv->timerID);
1183
1184 if (video->control_callback)
1185 {
1186 IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1187 if (mgr)
1188 IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1189 }
1190 if (video->data_callback)
1191 {
1192 IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1193 if (mgr)
1194 IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1195 }
1196
1197 if (video->context)
1198 VideoClientContextPriv_free(video->context->priv);
1199
1200 free(video->control_callback);
1201 free(video->data_callback);
1202 free(video->wtsPlugin.pInterface);
1203 free(pPlugin);
1204 return CHANNEL_RC_OK;
1205}
1206
1215WINPR_ATTR_NODISCARD
1216FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1217{
1218 UINT error = ERROR_INTERNAL_ERROR;
1219
1220 VIDEO_PLUGIN* videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1221 if (!videoPlugin)
1222 {
1223 videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1224 if (!videoPlugin)
1225 {
1226 WLog_ERR(TAG, "calloc failed!");
1227 return CHANNEL_RC_NO_MEMORY;
1228 }
1229
1230 videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1231 videoPlugin->wtsPlugin.Connected = nullptr;
1232 videoPlugin->wtsPlugin.Disconnected = nullptr;
1233 videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1234
1235 VideoClientContext* videoContext =
1236 (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1237 if (!videoContext)
1238 {
1239 WLog_ERR(TAG, "calloc failed!");
1240 free(videoPlugin);
1241 return CHANNEL_RC_NO_MEMORY;
1242 }
1243
1244 VideoClientContextPriv* priv = VideoClientContextPriv_new(videoContext);
1245 if (!priv)
1246 {
1247 WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1248 free(videoContext);
1249 free(videoPlugin);
1250 return CHANNEL_RC_NO_MEMORY;
1251 }
1252
1253 videoContext->handle = (void*)videoPlugin;
1254 videoContext->priv = priv;
1255 videoContext->timer = video_timer;
1256 videoContext->setGeometry = video_client_context_set_geometry;
1257
1258 videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1259 videoPlugin->context = videoContext;
1260 videoPlugin->rdpcontext = pEntryPoints->GetRdpContext(pEntryPoints);
1261 if (videoPlugin->rdpcontext)
1262 error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1263 }
1264 else
1265 {
1266 WLog_ERR(TAG, "could not get video Plugin.");
1267 return CHANNEL_RC_BAD_CHANNEL;
1268 }
1269
1270 return error;
1271}
freerdp_settings_get_bool
WINPR_ATTR_NODISCARD FREERDP_API BOOL freerdp_settings_get_bool(const rdpSettings *settings, FreeRDP_Settings_Keys_Bool id)
Returns a boolean settings value.
GENERIC_CHANNEL_CALLBACK
Definition include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition include/freerdp/client/channels.h:43
RDP_RECT
Definition types.h:101
RECTANGLE_16
Definition types.h:109
RTL_CRITICAL_SECTION
Definition include/winpr/synch.h:160
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition client/video.h:36