FreeRDP
Loading...
Searching...
No Matches
video_main.c
1
20#include <freerdp/config.h>
21
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include <winpr/crt.h>
27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/synch.h>
30#include <winpr/print.h>
31#include <winpr/stream.h>
32#include <winpr/cmdline.h>
33#include <winpr/collections.h>
34#include <winpr/interlocked.h>
35#include <winpr/sysinfo.h>
36
37#include <freerdp/addin.h>
38#include <freerdp/primitives.h>
39#include <freerdp/client/channels.h>
40#include <freerdp/client/geometry.h>
41#include <freerdp/client/video.h>
42#include <freerdp/channels/log.h>
43#include <freerdp/codec/h264.h>
44#include <freerdp/codec/yuv.h>
45#include <freerdp/timer.h>
46
47#define TAG CHANNELS_TAG("video")
48
49#include "video_main.h"
50
51typedef struct
52{
53 IWTSPlugin wtsPlugin;
54
55 IWTSListener* controlListener;
56 IWTSListener* dataListener;
57 GENERIC_LISTENER_CALLBACK* control_callback;
58 GENERIC_LISTENER_CALLBACK* data_callback;
59
60 VideoClientContext* context;
61 BOOL initialized;
62 rdpContext* rdpcontext;
63} VIDEO_PLUGIN;
64
65#define XF_VIDEO_UNLIMITED_RATE 31
66
67static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
68 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
69
70typedef struct
71{
72 VideoClientContext* video;
73 BYTE PresentationId;
74 UINT32 ScaledWidth, ScaledHeight;
75 MAPPED_GEOMETRY* geometry;
76
77 UINT64 startTimeStamp;
78 UINT64 publishOffset;
79 H264_CONTEXT* h264;
80 wStream* currentSample;
81 UINT64 lastPublishTime, nextPublishTime;
82 volatile LONG refCounter;
83 VideoSurface* surface;
84} PresentationContext;
85
86typedef struct
87{
88 UINT64 publishTime;
89 UINT64 hnsDuration;
90 MAPPED_GEOMETRY* geometry;
91 UINT32 w, h;
92 UINT32 scanline;
93 BYTE* surfaceData;
94 PresentationContext* presentation;
95} VideoFrame;
96
98struct s_VideoClientContextPriv
99{
100 VideoClientContext* video;
101 GeometryClientContext* geometry;
102 wQueue* frames;
103 CRITICAL_SECTION framesLock;
104 wBufferPool* surfacePool;
105 UINT32 publishedFrames;
106 UINT32 droppedFrames;
107 UINT32 lastSentRate;
108 UINT64 nextFeedbackTime;
109 PresentationContext* currentPresentation;
110 FreeRDP_TimerID timerID;
111};
112
113static void PresentationContext_unref(PresentationContext** presentation);
114static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
115
116static const char* video_command_name(BYTE cmd)
117{
118 switch (cmd)
119 {
120 case TSMM_START_PRESENTATION:
121 return "start";
122 case TSMM_STOP_PRESENTATION:
123 return "stop";
124 default:
125 return "<unknown>";
126 }
127}
128
129static void video_client_context_set_geometry(VideoClientContext* video,
130 GeometryClientContext* geometry)
131{
132 WINPR_ASSERT(video);
133 WINPR_ASSERT(video->priv);
134 video->priv->geometry = geometry;
135}
136
137static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
138{
139 VideoClientContextPriv* ret = NULL;
140
141 WINPR_ASSERT(video);
142 ret = calloc(1, sizeof(*ret));
143 if (!ret)
144 return NULL;
145
146 ret->frames = Queue_New(TRUE, 10, 2);
147 if (!ret->frames)
148 {
149 WLog_ERR(TAG, "unable to allocate frames queue");
150 goto fail;
151 }
152
153 ret->surfacePool = BufferPool_New(FALSE, 0, 16);
154 if (!ret->surfacePool)
155 {
156 WLog_ERR(TAG, "unable to create surface pool");
157 goto fail;
158 }
159
160 if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
161 {
162 WLog_ERR(TAG, "unable to initialize frames lock");
163 goto fail;
164 }
165
166 ret->video = video;
167
168 /* don't set to unlimited so that we have the chance to send a feedback in
169 * the first second (for servers that want feedback directly)
170 */
171 ret->lastSentRate = 30;
172 return ret;
173
174fail:
175 VideoClientContextPriv_free(ret);
176 return NULL;
177}
178
179static BOOL PresentationContext_ref(PresentationContext* presentation)
180{
181 WINPR_ASSERT(presentation);
182
183 InterlockedIncrement(&presentation->refCounter);
184 return TRUE;
185}
186
187static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
188 UINT32 x, UINT32 y, UINT32 width, UINT32 height)
189{
190 size_t s = 4ULL * width * height;
191 PresentationContext* ret = NULL;
192
193 WINPR_ASSERT(video);
194
195 if (s > INT32_MAX)
196 return NULL;
197
198 ret = calloc(1, sizeof(*ret));
199 if (!ret)
200 return NULL;
201
202 ret->video = video;
203 ret->PresentationId = PresentationId;
204
205 ret->h264 = h264_context_new(FALSE);
206 if (!ret->h264)
207 {
208 WLog_ERR(TAG, "unable to create a h264 context");
209 goto fail;
210 }
211 if (!h264_context_reset(ret->h264, width, height))
212 goto fail;
213
214 ret->currentSample = Stream_New(NULL, 4096);
215 if (!ret->currentSample)
216 {
217 WLog_ERR(TAG, "unable to create current packet stream");
218 goto fail;
219 }
220
221 ret->surface = video->createSurface(video, x, y, width, height);
222 if (!ret->surface)
223 {
224 WLog_ERR(TAG, "unable to create surface");
225 goto fail;
226 }
227
228 if (!PresentationContext_ref(ret))
229 goto fail;
230
231 return ret;
232
233fail:
234 PresentationContext_unref(&ret);
235 return NULL;
236}
237
238static void PresentationContext_unref(PresentationContext** ppresentation)
239{
240 PresentationContext* presentation = NULL;
241 MAPPED_GEOMETRY* geometry = NULL;
242
243 WINPR_ASSERT(ppresentation);
244
245 presentation = *ppresentation;
246 if (!presentation)
247 return;
248
249 if (InterlockedDecrement(&presentation->refCounter) > 0)
250 return;
251
252 geometry = presentation->geometry;
253 if (geometry)
254 {
255 geometry->MappedGeometryUpdate = NULL;
256 geometry->MappedGeometryClear = NULL;
257 geometry->custom = NULL;
258 mappedGeometryUnref(geometry);
259 }
260
261 h264_context_free(presentation->h264);
262 Stream_Free(presentation->currentSample, TRUE);
263 presentation->video->deleteSurface(presentation->video, presentation->surface);
264 free(presentation);
265 *ppresentation = NULL;
266}
267
268static void VideoFrame_free(VideoFrame** pframe)
269{
270 VideoFrame* frame = NULL;
271
272 WINPR_ASSERT(pframe);
273 frame = *pframe;
274 if (!frame)
275 return;
276
277 mappedGeometryUnref(frame->geometry);
278
279 WINPR_ASSERT(frame->presentation);
280 WINPR_ASSERT(frame->presentation->video);
281 WINPR_ASSERT(frame->presentation->video->priv);
282 BufferPool_Return(frame->presentation->video->priv->surfacePool, frame->surfaceData);
283 PresentationContext_unref(&frame->presentation);
284 free(frame);
285 *pframe = NULL;
286}
287
288static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
289 MAPPED_GEOMETRY* geom)
290{
291 VideoFrame* frame = NULL;
292 const VideoSurface* surface = NULL;
293
294 WINPR_ASSERT(priv);
295 WINPR_ASSERT(presentation);
296 WINPR_ASSERT(geom);
297
298 surface = presentation->surface;
299 WINPR_ASSERT(surface);
300
301 frame = calloc(1, sizeof(VideoFrame));
302 if (!frame)
303 goto fail;
304
305 mappedGeometryRef(geom);
306
307 frame->publishTime = presentation->lastPublishTime;
308 frame->geometry = geom;
309 frame->w = surface->alignedWidth;
310 frame->h = surface->alignedHeight;
311 frame->scanline = surface->scanline;
312
313 frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
314 if (!frame->surfaceData)
315 goto fail;
316
317 frame->presentation = presentation;
318 if (!PresentationContext_ref(frame->presentation))
319 goto fail;
320
321 return frame;
322
323fail:
324 VideoFrame_free(&frame);
325 return NULL;
326}
327
328void VideoClientContextPriv_free(VideoClientContextPriv* priv)
329{
330 if (!priv)
331 return;
332
333 EnterCriticalSection(&priv->framesLock);
334
335 if (priv->frames)
336 {
337 while (Queue_Count(priv->frames))
338 {
339 VideoFrame* frame = Queue_Dequeue(priv->frames);
340 if (frame)
341 VideoFrame_free(&frame);
342 }
343 }
344
345 Queue_Free(priv->frames);
346 LeaveCriticalSection(&priv->framesLock);
347
348 DeleteCriticalSection(&priv->framesLock);
349
350 if (priv->currentPresentation)
351 PresentationContext_unref(&priv->currentPresentation);
352
353 BufferPool_Free(priv->surfacePool);
354 free(priv);
355}
356
357static UINT video_control_send_presentation_response(VideoClientContext* context,
358 TSMM_PRESENTATION_RESPONSE* resp)
359{
360 BYTE buf[12] = { 0 };
361 wStream* s = NULL;
362 VIDEO_PLUGIN* video = NULL;
363 IWTSVirtualChannel* channel = NULL;
364 UINT ret = 0;
365
366 WINPR_ASSERT(context);
367 WINPR_ASSERT(resp);
368
369 video = (VIDEO_PLUGIN*)context->handle;
370 WINPR_ASSERT(video);
371
372 s = Stream_New(buf, 12);
373 if (!s)
374 return CHANNEL_RC_NO_MEMORY;
375
376 Stream_Write_UINT32(s, 12); /* cbSize */
377 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
378 Stream_Write_UINT8(s, resp->PresentationId);
379 Stream_Zero(s, 3);
380 Stream_SealLength(s);
381
382 channel = video->control_callback->channel_callback->channel;
383 ret = channel->Write(channel, 12, buf, NULL);
384 Stream_Free(s, FALSE);
385
386 return ret;
387}
388
389static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
390{
391 PresentationContext* presentation = NULL;
392 RDP_RECT* r = NULL;
393
394 WINPR_ASSERT(geometry);
395
396 presentation = (PresentationContext*)geometry->custom;
397 WINPR_ASSERT(presentation);
398
399 r = &geometry->geometry.boundingRect;
400 WLog_DBG(TAG,
401 "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
402 ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
403 "-%" PRId16 "x%" PRId16 ")",
404 geometry->topLevelLeft, geometry->topLevelTop,
405 geometry->topLevelRight - geometry->topLevelLeft,
406 geometry->topLevelBottom - geometry->topLevelTop,
407
408 geometry->left, geometry->top, geometry->right - geometry->left,
409 geometry->bottom - geometry->top,
410
411 r->x, r->y, r->width, r->height);
412
413 presentation->surface->x =
414 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
415 presentation->surface->y =
416 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
417
418 return TRUE;
419}
420
421static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
422{
423 PresentationContext* presentation = NULL;
424
425 WINPR_ASSERT(geometry);
426
427 presentation = (PresentationContext*)geometry->custom;
428 WINPR_ASSERT(presentation);
429
430 mappedGeometryUnref(presentation->geometry);
431 presentation->geometry = NULL;
432 return TRUE;
433}
434
435static UINT video_PresentationRequest(VideoClientContext* video,
436 const TSMM_PRESENTATION_REQUEST* req)
437{
438 UINT ret = CHANNEL_RC_OK;
439
440 WINPR_ASSERT(video);
441 WINPR_ASSERT(req);
442
443 VideoClientContextPriv* priv = video->priv;
444 WINPR_ASSERT(priv);
445
446 if (req->Command == TSMM_START_PRESENTATION)
447 {
448 MAPPED_GEOMETRY* geom = NULL;
449 TSMM_PRESENTATION_RESPONSE resp;
450
451 if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
452 {
453 WLog_ERR(TAG, "not a H264 video, ignoring request");
454 return CHANNEL_RC_OK;
455 }
456
457 if (priv->currentPresentation)
458 {
459 if (priv->currentPresentation->PresentationId == req->PresentationId)
460 {
461 WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
462 req->PresentationId);
463 return CHANNEL_RC_OK;
464 }
465
466 WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
467 PresentationContext_unref(&priv->currentPresentation);
468 }
469
470 if (!priv->geometry)
471 {
472 WLog_ERR(TAG, "geometry channel not ready, ignoring request");
473 return CHANNEL_RC_OK;
474 }
475
476 geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
477 if (!geom)
478 {
479 WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
480 return CHANNEL_RC_OK;
481 }
482
483 WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
484 priv->currentPresentation = PresentationContext_new(
485 video, req->PresentationId,
486 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
487 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
488 req->SourceHeight);
489 if (!priv->currentPresentation)
490 {
491 WLog_ERR(TAG, "unable to create presentation video");
492 return CHANNEL_RC_NO_MEMORY;
493 }
494
495 mappedGeometryRef(geom);
496 priv->currentPresentation->geometry = geom;
497
498 priv->currentPresentation->video = video;
499 priv->currentPresentation->ScaledWidth = req->ScaledWidth;
500 priv->currentPresentation->ScaledHeight = req->ScaledHeight;
501
502 geom->custom = priv->currentPresentation;
503 geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
504 geom->MappedGeometryClear = video_onMappedGeometryClear;
505
506 /* send back response */
507 resp.PresentationId = req->PresentationId;
508 ret = video_control_send_presentation_response(video, &resp);
509 }
510 else if (req->Command == TSMM_STOP_PRESENTATION)
511 {
512 WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
513 if (!priv->currentPresentation)
514 {
515 WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
516 return CHANNEL_RC_OK;
517 }
518
519 priv->droppedFrames = 0;
520 priv->publishedFrames = 0;
521 PresentationContext_unref(&priv->currentPresentation);
522 }
523
524 return ret;
525}
526
527static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
528{
529 TSMM_PRESENTATION_REQUEST req = { 0 };
530
531 WINPR_ASSERT(context);
532 WINPR_ASSERT(s);
533
534 if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
535 return ERROR_INVALID_DATA;
536
537 Stream_Read_UINT8(s, req.PresentationId);
538 Stream_Read_UINT8(s, req.Version);
539 Stream_Read_UINT8(s, req.Command);
540 Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
541
542 Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
543 Stream_Seek_UINT16(s); /* reserved */
544
545 Stream_Read_UINT32(s, req.SourceWidth);
546 Stream_Read_UINT32(s, req.SourceHeight);
547 Stream_Read_UINT32(s, req.ScaledWidth);
548 Stream_Read_UINT32(s, req.ScaledHeight);
549 Stream_Read_UINT64(s, req.hnsTimestampOffset);
550 Stream_Read_UINT64(s, req.GeometryMappingId);
551 Stream_Read(s, req.VideoSubtypeId, 16);
552
553 Stream_Read_UINT32(s, req.cbExtra);
554
555 if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
556 return ERROR_INVALID_DATA;
557
558 req.pExtraData = Stream_Pointer(s);
559
560 WLog_DBG(TAG,
561 "presentationReq: id:%" PRIu8 " version:%" PRIu8
562 " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
563 "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
564 req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
565 req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
566 req.GeometryMappingId);
567
568 return video_PresentationRequest(context, &req);
569}
570
576static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
577{
578 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
579 VIDEO_PLUGIN* video = NULL;
580 VideoClientContext* context = NULL;
581 UINT ret = CHANNEL_RC_OK;
582 UINT32 cbSize = 0;
583 UINT32 packetType = 0;
584
585 WINPR_ASSERT(callback);
586 WINPR_ASSERT(s);
587
588 video = (VIDEO_PLUGIN*)callback->plugin;
589 WINPR_ASSERT(video);
590
591 context = (VideoClientContext*)video->wtsPlugin.pInterface;
592 WINPR_ASSERT(context);
593
594 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
595 return ERROR_INVALID_DATA;
596
597 Stream_Read_UINT32(s, cbSize);
598 if (cbSize < 8)
599 {
600 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
601 return ERROR_INVALID_DATA;
602 }
603 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
604 return ERROR_INVALID_DATA;
605
606 Stream_Read_UINT32(s, packetType);
607 switch (packetType)
608 {
609 case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
610 ret = video_read_tsmm_presentation_req(context, s);
611 break;
612 default:
613 WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
614 ret = ERROR_UNSUPPORTED_TYPE;
615 break;
616 }
617
618 return ret;
619}
620
621static UINT video_control_send_client_notification(VideoClientContext* context,
622 const TSMM_CLIENT_NOTIFICATION* notif)
623{
624 BYTE buf[100];
625 wStream* s = NULL;
626 VIDEO_PLUGIN* video = NULL;
627 IWTSVirtualChannel* channel = NULL;
628 UINT ret = 0;
629 UINT32 cbSize = 0;
630
631 WINPR_ASSERT(context);
632 WINPR_ASSERT(notif);
633
634 video = (VIDEO_PLUGIN*)context->handle;
635 WINPR_ASSERT(video);
636
637 s = Stream_New(buf, 32);
638 if (!s)
639 return CHANNEL_RC_NO_MEMORY;
640
641 cbSize = 16;
642 Stream_Seek_UINT32(s); /* cbSize */
643 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
644 Stream_Write_UINT8(s, notif->PresentationId);
645 Stream_Write_UINT8(s, notif->NotificationType);
646 Stream_Zero(s, 2);
647 if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
648 {
649 Stream_Write_UINT32(s, 16); /* cbData */
650
651 /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
652 Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
653 Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
654 Stream_Zero(s, 4ULL * 2ULL);
655
656 cbSize += 4UL * 4UL;
657 }
658 else
659 {
660 Stream_Write_UINT32(s, 0); /* cbData */
661 }
662
663 Stream_SealLength(s);
664 Stream_SetPosition(s, 0);
665 Stream_Write_UINT32(s, cbSize);
666 Stream_Free(s, FALSE);
667
668 WINPR_ASSERT(video->control_callback);
669 WINPR_ASSERT(video->control_callback->channel_callback);
670
671 channel = video->control_callback->channel_callback->channel;
672 WINPR_ASSERT(channel);
673 WINPR_ASSERT(channel->Write);
674
675 ret = channel->Write(channel, cbSize, buf, NULL);
676
677 return ret;
678}
679
680static void video_timer(VideoClientContext* video, UINT64 now)
681{
682 PresentationContext* presentation = NULL;
683 VideoClientContextPriv* priv = NULL;
684 VideoFrame* peekFrame = NULL;
685 VideoFrame* frame = NULL;
686
687 WINPR_ASSERT(video);
688
689 priv = video->priv;
690 WINPR_ASSERT(priv);
691
692 EnterCriticalSection(&priv->framesLock);
693 do
694 {
695 peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
696 if (!peekFrame)
697 break;
698
699 if (peekFrame->publishTime > now)
700 break;
701
702 if (frame)
703 {
704 WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
705 priv->droppedFrames++;
706 VideoFrame_free(&frame);
707 }
708 frame = peekFrame;
709 Queue_Dequeue(priv->frames);
710 } while (1);
711 LeaveCriticalSection(&priv->framesLock);
712
713 if (!frame)
714 goto treat_feedback;
715
716 presentation = frame->presentation;
717
718 priv->publishedFrames++;
719 memcpy(presentation->surface->data, frame->surfaceData, 1ull * frame->scanline * frame->h);
720
721 WINPR_ASSERT(video->showSurface);
722 video->showSurface(video, presentation->surface, presentation->ScaledWidth,
723 presentation->ScaledHeight);
724
725 VideoFrame_free(&frame);
726
727treat_feedback:
728 if (priv->nextFeedbackTime < now)
729 {
730 /* we can compute some feedback only if we have some published frames and
731 * a current presentation
732 */
733 if (priv->publishedFrames && priv->currentPresentation)
734 {
735 UINT32 computedRate = 0;
736
737 PresentationContext_ref(priv->currentPresentation);
738
739 if (priv->droppedFrames)
740 {
746 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
747 computedRate = 24;
748 else
749 {
750 computedRate = priv->lastSentRate - 2;
751 if (!computedRate)
752 computedRate = 2;
753 }
754 }
755 else
756 {
761 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
762 computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
763 else
764 {
765 computedRate = priv->lastSentRate + 2;
766 if (computedRate > XF_VIDEO_UNLIMITED_RATE)
767 computedRate = XF_VIDEO_UNLIMITED_RATE;
768 }
769 }
770
771 if (computedRate != priv->lastSentRate)
772 {
773 TSMM_CLIENT_NOTIFICATION notif;
774
775 WINPR_ASSERT(priv->currentPresentation);
776 notif.PresentationId = priv->currentPresentation->PresentationId;
777 notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
778 if (computedRate == XF_VIDEO_UNLIMITED_RATE)
779 {
780 notif.FramerateOverride.Flags = 0x01;
781 notif.FramerateOverride.DesiredFrameRate = 0x00;
782 }
783 else
784 {
785 notif.FramerateOverride.Flags = 0x02;
786 notif.FramerateOverride.DesiredFrameRate = computedRate;
787 }
788
789 video_control_send_client_notification(video, &notif);
790 priv->lastSentRate = computedRate;
791
792 WLog_DBG(TAG,
793 "server notified with rate %" PRIu32 " published=%" PRIu32
794 " dropped=%" PRIu32,
795 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
796 }
797
798 PresentationContext_unref(&priv->currentPresentation);
799 }
800
801 WLog_DBG(TAG, "currentRate=%" PRIu32 " published=%" PRIu32 " dropped=%" PRIu32,
802 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
803
804 priv->droppedFrames = 0;
805 priv->publishedFrames = 0;
806 priv->nextFeedbackTime = now + 1000;
807 }
808}
809
810static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
811{
812 VideoClientContextPriv* priv = NULL;
813 PresentationContext* presentation = NULL;
814 int status = 0;
815
816 WINPR_ASSERT(context);
817 WINPR_ASSERT(data);
818
819 priv = context->priv;
820 WINPR_ASSERT(priv);
821
822 presentation = priv->currentPresentation;
823 if (!presentation)
824 {
825 WLog_ERR(TAG, "no current presentation");
826 return CHANNEL_RC_OK;
827 }
828
829 if (presentation->PresentationId != data->PresentationId)
830 {
831 WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
832 presentation->PresentationId, data->PresentationId);
833 return CHANNEL_RC_OK;
834 }
835
836 if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
837 {
838 WLog_ERR(TAG, "unable to expand the current packet");
839 return CHANNEL_RC_NO_MEMORY;
840 }
841
842 Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
843
844 if (data->CurrentPacketIndex == data->PacketsInSample)
845 {
846 VideoSurface* surface = presentation->surface;
847 H264_CONTEXT* h264 = presentation->h264;
848 UINT64 startTime = GetTickCount64();
849 UINT64 timeAfterH264 = 0;
850 MAPPED_GEOMETRY* geom = presentation->geometry;
851
852 const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
853 WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
854 Stream_SealLength(presentation->currentSample);
855 Stream_SetPosition(presentation->currentSample, 0);
856
857 timeAfterH264 = GetTickCount64();
858 if (data->SampleNumber == 1)
859 {
860 presentation->lastPublishTime = startTime;
861 }
862
863 presentation->lastPublishTime += (data->hnsDuration / 10000);
864 if (presentation->lastPublishTime <= timeAfterH264 + 10)
865 {
866 int dropped = 0;
867
868 const size_t len = Stream_Length(presentation->currentSample);
869 if (len > UINT32_MAX)
870 return CHANNEL_RC_OK;
871
872 /* if the frame is to be published in less than 10 ms, let's consider it's now */
873 status =
874 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
875 surface->data, surface->format, surface->scanline,
876 surface->alignedWidth, surface->alignedHeight, &rect, 1);
877
878 if (status < 0)
879 return CHANNEL_RC_OK;
880
881 WINPR_ASSERT(context->showSurface);
882 context->showSurface(context, presentation->surface, presentation->ScaledWidth,
883 presentation->ScaledHeight);
884
885 priv->publishedFrames++;
886
887 /* cleanup previously scheduled frames */
888 EnterCriticalSection(&priv->framesLock);
889 while (Queue_Count(priv->frames) > 0)
890 {
891 VideoFrame* frame = Queue_Dequeue(priv->frames);
892 if (frame)
893 {
894 priv->droppedFrames++;
895 VideoFrame_free(&frame);
896 dropped++;
897 }
898 }
899 LeaveCriticalSection(&priv->framesLock);
900
901 if (dropped)
902 WLog_DBG(TAG, "showing frame (%d dropped)", dropped);
903 }
904 else
905 {
906 const size_t len = Stream_Length(presentation->currentSample);
907 if (len > UINT32_MAX)
908 return CHANNEL_RC_OK;
909
910 BOOL enqueueResult = 0;
911 VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
912 if (!frame)
913 {
914 WLog_ERR(TAG, "unable to create frame");
915 return CHANNEL_RC_NO_MEMORY;
916 }
917
918 status =
919 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
920 frame->surfaceData, surface->format, surface->scanline,
921 surface->alignedWidth, surface->alignedHeight, &rect, 1);
922 if (status < 0)
923 {
924 VideoFrame_free(&frame);
925 return CHANNEL_RC_OK;
926 }
927
928 EnterCriticalSection(&priv->framesLock);
929 enqueueResult = Queue_Enqueue(priv->frames, frame);
930 LeaveCriticalSection(&priv->framesLock);
931
932 if (!enqueueResult)
933 {
934 WLog_ERR(TAG, "unable to enqueue frame");
935 VideoFrame_free(&frame);
936 return CHANNEL_RC_NO_MEMORY;
937 }
938
939 // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
940 WLog_DBG(TAG, "scheduling frame in %" PRIu32 " ms", (frame->publishTime - startTime));
941 }
942 }
943
944 return CHANNEL_RC_OK;
945}
946
947static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
948{
949 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
950 VIDEO_PLUGIN* video = NULL;
951 VideoClientContext* context = NULL;
952 UINT32 cbSize = 0;
953 UINT32 packetType = 0;
954 TSMM_VIDEO_DATA data;
955
956 video = (VIDEO_PLUGIN*)callback->plugin;
957 context = (VideoClientContext*)video->wtsPlugin.pInterface;
958
959 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
960 return ERROR_INVALID_DATA;
961
962 Stream_Read_UINT32(s, cbSize);
963 if (cbSize < 8)
964 {
965 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
966 return ERROR_INVALID_DATA;
967 }
968
969 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
970 return ERROR_INVALID_DATA;
971
972 Stream_Read_UINT32(s, packetType);
973 if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
974 {
975 WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
976 return ERROR_INVALID_DATA;
977 }
978
979 if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
980 return ERROR_INVALID_DATA;
981
982 Stream_Read_UINT8(s, data.PresentationId);
983 Stream_Read_UINT8(s, data.Version);
984 Stream_Read_UINT8(s, data.Flags);
985 Stream_Seek_UINT8(s); /* reserved */
986 Stream_Read_UINT64(s, data.hnsTimestamp);
987 Stream_Read_UINT64(s, data.hnsDuration);
988 Stream_Read_UINT16(s, data.CurrentPacketIndex);
989 Stream_Read_UINT16(s, data.PacketsInSample);
990 Stream_Read_UINT32(s, data.SampleNumber);
991 Stream_Read_UINT32(s, data.cbSample);
992 if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
993 return ERROR_INVALID_DATA;
994 data.pSample = Stream_Pointer(s);
995
996 /*
997 WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
998 duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
999 cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
1000 data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
1001 data.cbSample);
1002 */
1003
1004 return video_VideoData(context, &data);
1005}
1006
1012static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1013{
1014 free(pChannelCallback);
1015 return CHANNEL_RC_OK;
1016}
1017
1018static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1019{
1020 free(pChannelCallback);
1021 return CHANNEL_RC_OK;
1022}
1023
1029// NOLINTBEGIN(readability-non-const-parameter)
1030static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1031 IWTSVirtualChannel* channel, BYTE* Data,
1032 BOOL* pbAccept,
1033 IWTSVirtualChannelCallback** ppCallback)
1034// NOLINTEND(readability-non-const-parameter)
1035{
1036 GENERIC_CHANNEL_CALLBACK* callback = NULL;
1037 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1038
1039 WINPR_UNUSED(Data);
1040 WINPR_UNUSED(pbAccept);
1041
1042 callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1043 if (!callback)
1044 {
1045 WLog_ERR(TAG, "calloc failed!");
1046 return CHANNEL_RC_NO_MEMORY;
1047 }
1048
1049 callback->iface.OnDataReceived = video_control_on_data_received;
1050 callback->iface.OnClose = video_control_on_close;
1051 callback->plugin = listener_callback->plugin;
1052 callback->channel_mgr = listener_callback->channel_mgr;
1053 callback->channel = channel;
1054 listener_callback->channel_callback = callback;
1055
1056 *ppCallback = (IWTSVirtualChannelCallback*)callback;
1057
1058 return CHANNEL_RC_OK;
1059}
1060
1061// NOLINTBEGIN(readability-non-const-parameter)
1062static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1063 IWTSVirtualChannel* pChannel, BYTE* Data,
1064 BOOL* pbAccept,
1065 IWTSVirtualChannelCallback** ppCallback)
1066// NOLINTEND(readability-non-const-parameter)
1067{
1068 GENERIC_CHANNEL_CALLBACK* callback = NULL;
1069 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1070
1071 WINPR_UNUSED(Data);
1072 WINPR_UNUSED(pbAccept);
1073
1074 callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1075 if (!callback)
1076 {
1077 WLog_ERR(TAG, "calloc failed!");
1078 return CHANNEL_RC_NO_MEMORY;
1079 }
1080
1081 callback->iface.OnDataReceived = video_data_on_data_received;
1082 callback->iface.OnClose = video_data_on_close;
1083 callback->plugin = listener_callback->plugin;
1084 callback->channel_mgr = listener_callback->channel_mgr;
1085 callback->channel = pChannel;
1086 listener_callback->channel_callback = callback;
1087
1088 *ppCallback = (IWTSVirtualChannelCallback*)callback;
1089
1090 return CHANNEL_RC_OK;
1091}
1092
1093static uint64_t timer_cb(WINPR_ATTR_UNUSED rdpContext* context, void* userdata,
1094 WINPR_ATTR_UNUSED FreeRDP_TimerID timerID, uint64_t timestamp,
1095 uint64_t interval)
1096{
1097 VideoClientContext* video = userdata;
1098 if (!video)
1099 return 0;
1100 if (!video->timer)
1101 return 0;
1102
1103 video->timer(video, timestamp);
1104
1105 return interval;
1106}
1107
1113static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1114{
1115 UINT status = 0;
1116 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1117 GENERIC_LISTENER_CALLBACK* callback = NULL;
1118
1119 if (video->initialized)
1120 {
1121 WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1122 return ERROR_INVALID_DATA;
1123 }
1124 video->control_callback = callback =
1125 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1126 if (!callback)
1127 {
1128 WLog_ERR(TAG, "calloc for control callback failed!");
1129 return CHANNEL_RC_NO_MEMORY;
1130 }
1131
1132 callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1133 callback->plugin = plugin;
1134 callback->channel_mgr = channelMgr;
1135
1136 status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1137 &callback->iface, &(video->controlListener));
1138
1139 if (status != CHANNEL_RC_OK)
1140 return status;
1141 video->controlListener->pInterface = video->wtsPlugin.pInterface;
1142
1143 video->data_callback = callback =
1144 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1145 if (!callback)
1146 {
1147 WLog_ERR(TAG, "calloc for data callback failed!");
1148 return CHANNEL_RC_NO_MEMORY;
1149 }
1150
1151 callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1152 callback->plugin = plugin;
1153 callback->channel_mgr = channelMgr;
1154
1155 status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1156 &callback->iface, &(video->dataListener));
1157
1158 if (status == CHANNEL_RC_OK)
1159 video->dataListener->pInterface = video->wtsPlugin.pInterface;
1160
1161 if (status == CHANNEL_RC_OK)
1162 video->context->priv->timerID =
1163 freerdp_timer_add(video->rdpcontext, 20000000, timer_cb, video->context, true);
1164 video->initialized = video->context->priv->timerID != 0;
1165 if (!video->initialized)
1166 status = ERROR_INTERNAL_ERROR;
1167 return status;
1168}
1169
1175static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1176{
1177 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1178 if (!video)
1179 return CHANNEL_RC_INVALID_INSTANCE;
1180
1181 if (video->context && video->context->priv)
1182 freerdp_timer_remove(video->rdpcontext, video->context->priv->timerID);
1183
1184 if (video->control_callback)
1185 {
1186 IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1187 if (mgr)
1188 IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1189 }
1190 if (video->data_callback)
1191 {
1192 IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1193 if (mgr)
1194 IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1195 }
1196
1197 if (video->context)
1198 VideoClientContextPriv_free(video->context->priv);
1199
1200 free(video->control_callback);
1201 free(video->data_callback);
1202 free(video->wtsPlugin.pInterface);
1203 free(pPlugin);
1204 return CHANNEL_RC_OK;
1205}
1206
1215FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1216{
1217 UINT error = ERROR_INTERNAL_ERROR;
1218 VIDEO_PLUGIN* videoPlugin = NULL;
1219 VideoClientContext* videoContext = NULL;
1220 VideoClientContextPriv* priv = NULL;
1221
1222 videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1223 if (!videoPlugin)
1224 {
1225 videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1226 if (!videoPlugin)
1227 {
1228 WLog_ERR(TAG, "calloc failed!");
1229 return CHANNEL_RC_NO_MEMORY;
1230 }
1231
1232 videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1233 videoPlugin->wtsPlugin.Connected = NULL;
1234 videoPlugin->wtsPlugin.Disconnected = NULL;
1235 videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1236
1237 videoContext = (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1238 if (!videoContext)
1239 {
1240 WLog_ERR(TAG, "calloc failed!");
1241 free(videoPlugin);
1242 return CHANNEL_RC_NO_MEMORY;
1243 }
1244
1245 priv = VideoClientContextPriv_new(videoContext);
1246 if (!priv)
1247 {
1248 WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1249 free(videoContext);
1250 free(videoPlugin);
1251 return CHANNEL_RC_NO_MEMORY;
1252 }
1253
1254 videoContext->handle = (void*)videoPlugin;
1255 videoContext->priv = priv;
1256 videoContext->timer = video_timer;
1257 videoContext->setGeometry = video_client_context_set_geometry;
1258
1259 videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1260 videoPlugin->context = videoContext;
1261 videoPlugin->rdpcontext = pEntryPoints->GetRdpContext(pEntryPoints);
1262 if (videoPlugin->rdpcontext)
1263 error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1264 }
1265 else
1266 {
1267 WLog_ERR(TAG, "could not get video Plugin.");
1268 return CHANNEL_RC_BAD_CHANNEL;
1269 }
1270
1271 return error;
1272}
GENERIC_CHANNEL_CALLBACK
Definition include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition include/freerdp/client/channels.h:43
RDP_RECT
Definition types.h:101
RECTANGLE_16
Definition types.h:109
RTL_CRITICAL_SECTION
Definition include/winpr/synch.h:158
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition client/video.h:36