FreeRDP
Loading...
Searching...
No Matches
video_main.c
1
20#include <freerdp/config.h>
21
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include <winpr/crt.h>
27#include <winpr/assert.h>
28#include <winpr/cast.h>
29#include <winpr/synch.h>
30#include <winpr/print.h>
31#include <winpr/stream.h>
32#include <winpr/cmdline.h>
33#include <winpr/collections.h>
34#include <winpr/interlocked.h>
35#include <winpr/sysinfo.h>
36
37#include <freerdp/addin.h>
38#include <freerdp/primitives.h>
39#include <freerdp/client/channels.h>
40#include <freerdp/client/geometry.h>
41#include <freerdp/client/video.h>
42#include <freerdp/channels/log.h>
43#include <freerdp/codec/h264.h>
44#include <freerdp/codec/yuv.h>
45
46#define TAG CHANNELS_TAG("video")
47
48#include "video_main.h"
49
50typedef struct
51{
52 IWTSPlugin wtsPlugin;
53
54 IWTSListener* controlListener;
55 IWTSListener* dataListener;
56 GENERIC_LISTENER_CALLBACK* control_callback;
57 GENERIC_LISTENER_CALLBACK* data_callback;
58
59 VideoClientContext* context;
60 BOOL initialized;
61} VIDEO_PLUGIN;
62
63#define XF_VIDEO_UNLIMITED_RATE 31
64
65static const BYTE MFVideoFormat_H264[] = { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00,
66 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 };
67
68typedef struct
69{
70 VideoClientContext* video;
71 BYTE PresentationId;
72 UINT32 ScaledWidth, ScaledHeight;
73 MAPPED_GEOMETRY* geometry;
74
75 UINT64 startTimeStamp;
76 UINT64 publishOffset;
77 H264_CONTEXT* h264;
78 wStream* currentSample;
79 UINT64 lastPublishTime, nextPublishTime;
80 volatile LONG refCounter;
81 VideoSurface* surface;
82} PresentationContext;
83
84typedef struct
85{
86 UINT64 publishTime;
87 UINT64 hnsDuration;
88 MAPPED_GEOMETRY* geometry;
89 UINT32 w, h;
90 UINT32 scanline;
91 BYTE* surfaceData;
92 PresentationContext* presentation;
93} VideoFrame;
94
96struct s_VideoClientContextPriv
97{
98 VideoClientContext* video;
99 GeometryClientContext* geometry;
100 wQueue* frames;
101 CRITICAL_SECTION framesLock;
102 wBufferPool* surfacePool;
103 UINT32 publishedFrames;
104 UINT32 droppedFrames;
105 UINT32 lastSentRate;
106 UINT64 nextFeedbackTime;
107 PresentationContext* currentPresentation;
108};
109
110static void PresentationContext_unref(PresentationContext** presentation);
111static void VideoClientContextPriv_free(VideoClientContextPriv* priv);
112
113static const char* video_command_name(BYTE cmd)
114{
115 switch (cmd)
116 {
117 case TSMM_START_PRESENTATION:
118 return "start";
119 case TSMM_STOP_PRESENTATION:
120 return "stop";
121 default:
122 return "<unknown>";
123 }
124}
125
126static void video_client_context_set_geometry(VideoClientContext* video,
127 GeometryClientContext* geometry)
128{
129 WINPR_ASSERT(video);
130 WINPR_ASSERT(video->priv);
131 video->priv->geometry = geometry;
132}
133
134static VideoClientContextPriv* VideoClientContextPriv_new(VideoClientContext* video)
135{
136 VideoClientContextPriv* ret = NULL;
137
138 WINPR_ASSERT(video);
139 ret = calloc(1, sizeof(*ret));
140 if (!ret)
141 return NULL;
142
143 ret->frames = Queue_New(TRUE, 10, 2);
144 if (!ret->frames)
145 {
146 WLog_ERR(TAG, "unable to allocate frames queue");
147 goto fail;
148 }
149
150 ret->surfacePool = BufferPool_New(FALSE, 0, 16);
151 if (!ret->surfacePool)
152 {
153 WLog_ERR(TAG, "unable to create surface pool");
154 goto fail;
155 }
156
157 if (!InitializeCriticalSectionAndSpinCount(&ret->framesLock, 4 * 1000))
158 {
159 WLog_ERR(TAG, "unable to initialize frames lock");
160 goto fail;
161 }
162
163 ret->video = video;
164
165 /* don't set to unlimited so that we have the chance to send a feedback in
166 * the first second (for servers that want feedback directly)
167 */
168 ret->lastSentRate = 30;
169 return ret;
170
171fail:
172 VideoClientContextPriv_free(ret);
173 return NULL;
174}
175
176static BOOL PresentationContext_ref(PresentationContext* presentation)
177{
178 WINPR_ASSERT(presentation);
179
180 InterlockedIncrement(&presentation->refCounter);
181 return TRUE;
182}
183
184static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
185 UINT32 x, UINT32 y, UINT32 width, UINT32 height)
186{
187 size_t s = 4ULL * width * height;
188 PresentationContext* ret = NULL;
189
190 WINPR_ASSERT(video);
191
192 if (s > INT32_MAX)
193 return NULL;
194
195 ret = calloc(1, sizeof(*ret));
196 if (!ret)
197 return NULL;
198
199 ret->video = video;
200 ret->PresentationId = PresentationId;
201
202 ret->h264 = h264_context_new(FALSE);
203 if (!ret->h264)
204 {
205 WLog_ERR(TAG, "unable to create a h264 context");
206 goto fail;
207 }
208 if (!h264_context_reset(ret->h264, width, height))
209 goto fail;
210
211 ret->currentSample = Stream_New(NULL, 4096);
212 if (!ret->currentSample)
213 {
214 WLog_ERR(TAG, "unable to create current packet stream");
215 goto fail;
216 }
217
218 ret->surface = video->createSurface(video, x, y, width, height);
219 if (!ret->surface)
220 {
221 WLog_ERR(TAG, "unable to create surface");
222 goto fail;
223 }
224
225 if (!PresentationContext_ref(ret))
226 goto fail;
227
228 return ret;
229
230fail:
231 PresentationContext_unref(&ret);
232 return NULL;
233}
234
235static void PresentationContext_unref(PresentationContext** ppresentation)
236{
237 PresentationContext* presentation = NULL;
238 MAPPED_GEOMETRY* geometry = NULL;
239
240 WINPR_ASSERT(ppresentation);
241
242 presentation = *ppresentation;
243 if (!presentation)
244 return;
245
246 if (InterlockedDecrement(&presentation->refCounter) > 0)
247 return;
248
249 geometry = presentation->geometry;
250 if (geometry)
251 {
252 geometry->MappedGeometryUpdate = NULL;
253 geometry->MappedGeometryClear = NULL;
254 geometry->custom = NULL;
255 mappedGeometryUnref(geometry);
256 }
257
258 h264_context_free(presentation->h264);
259 Stream_Free(presentation->currentSample, TRUE);
260 presentation->video->deleteSurface(presentation->video, presentation->surface);
261 free(presentation);
262 *ppresentation = NULL;
263}
264
265static void VideoFrame_free(VideoFrame** pframe)
266{
267 VideoFrame* frame = NULL;
268
269 WINPR_ASSERT(pframe);
270 frame = *pframe;
271 if (!frame)
272 return;
273
274 mappedGeometryUnref(frame->geometry);
275
276 WINPR_ASSERT(frame->presentation);
277 WINPR_ASSERT(frame->presentation->video);
278 WINPR_ASSERT(frame->presentation->video->priv);
279 BufferPool_Return(frame->presentation->video->priv->surfacePool, frame->surfaceData);
280 PresentationContext_unref(&frame->presentation);
281 free(frame);
282 *pframe = NULL;
283}
284
285static VideoFrame* VideoFrame_new(VideoClientContextPriv* priv, PresentationContext* presentation,
286 MAPPED_GEOMETRY* geom)
287{
288 VideoFrame* frame = NULL;
289 const VideoSurface* surface = NULL;
290
291 WINPR_ASSERT(priv);
292 WINPR_ASSERT(presentation);
293 WINPR_ASSERT(geom);
294
295 surface = presentation->surface;
296 WINPR_ASSERT(surface);
297
298 frame = calloc(1, sizeof(VideoFrame));
299 if (!frame)
300 goto fail;
301
302 mappedGeometryRef(geom);
303
304 frame->publishTime = presentation->lastPublishTime;
305 frame->geometry = geom;
306 frame->w = surface->alignedWidth;
307 frame->h = surface->alignedHeight;
308 frame->scanline = surface->scanline;
309
310 frame->surfaceData = BufferPool_Take(priv->surfacePool, 1ll * frame->scanline * frame->h);
311 if (!frame->surfaceData)
312 goto fail;
313
314 frame->presentation = presentation;
315 if (!PresentationContext_ref(frame->presentation))
316 goto fail;
317
318 return frame;
319
320fail:
321 VideoFrame_free(&frame);
322 return NULL;
323}
324
325void VideoClientContextPriv_free(VideoClientContextPriv* priv)
326{
327 if (!priv)
328 return;
329
330 EnterCriticalSection(&priv->framesLock);
331
332 if (priv->frames)
333 {
334 while (Queue_Count(priv->frames))
335 {
336 VideoFrame* frame = Queue_Dequeue(priv->frames);
337 if (frame)
338 VideoFrame_free(&frame);
339 }
340 }
341
342 Queue_Free(priv->frames);
343 LeaveCriticalSection(&priv->framesLock);
344
345 DeleteCriticalSection(&priv->framesLock);
346
347 if (priv->currentPresentation)
348 PresentationContext_unref(&priv->currentPresentation);
349
350 BufferPool_Free(priv->surfacePool);
351 free(priv);
352}
353
354static UINT video_control_send_presentation_response(VideoClientContext* context,
355 TSMM_PRESENTATION_RESPONSE* resp)
356{
357 BYTE buf[12] = { 0 };
358 wStream* s = NULL;
359 VIDEO_PLUGIN* video = NULL;
360 IWTSVirtualChannel* channel = NULL;
361 UINT ret = 0;
362
363 WINPR_ASSERT(context);
364 WINPR_ASSERT(resp);
365
366 video = (VIDEO_PLUGIN*)context->handle;
367 WINPR_ASSERT(video);
368
369 s = Stream_New(buf, 12);
370 if (!s)
371 return CHANNEL_RC_NO_MEMORY;
372
373 Stream_Write_UINT32(s, 12); /* cbSize */
374 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_PRESENTATION_RESPONSE); /* PacketType */
375 Stream_Write_UINT8(s, resp->PresentationId);
376 Stream_Zero(s, 3);
377 Stream_SealLength(s);
378
379 channel = video->control_callback->channel_callback->channel;
380 ret = channel->Write(channel, 12, buf, NULL);
381 Stream_Free(s, FALSE);
382
383 return ret;
384}
385
386static BOOL video_onMappedGeometryUpdate(MAPPED_GEOMETRY* geometry)
387{
388 PresentationContext* presentation = NULL;
389 RDP_RECT* r = NULL;
390
391 WINPR_ASSERT(geometry);
392
393 presentation = (PresentationContext*)geometry->custom;
394 WINPR_ASSERT(presentation);
395
396 r = &geometry->geometry.boundingRect;
397 WLog_DBG(TAG,
398 "geometry updated topGeom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32
399 ") geom=(%" PRId32 ",%" PRId32 "-%" PRId32 "x%" PRId32 ") rects=(%" PRId16 ",%" PRId16
400 "-%" PRId16 "x%" PRId16 ")",
401 geometry->topLevelLeft, geometry->topLevelTop,
402 geometry->topLevelRight - geometry->topLevelLeft,
403 geometry->topLevelBottom - geometry->topLevelTop,
404
405 geometry->left, geometry->top, geometry->right - geometry->left,
406 geometry->bottom - geometry->top,
407
408 r->x, r->y, r->width, r->height);
409
410 presentation->surface->x =
411 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelLeft + geometry->left);
412 presentation->surface->y =
413 WINPR_ASSERTING_INT_CAST(uint32_t, geometry->topLevelTop + geometry->top);
414
415 return TRUE;
416}
417
418static BOOL video_onMappedGeometryClear(MAPPED_GEOMETRY* geometry)
419{
420 PresentationContext* presentation = NULL;
421
422 WINPR_ASSERT(geometry);
423
424 presentation = (PresentationContext*)geometry->custom;
425 WINPR_ASSERT(presentation);
426
427 mappedGeometryUnref(presentation->geometry);
428 presentation->geometry = NULL;
429 return TRUE;
430}
431
432static UINT video_PresentationRequest(VideoClientContext* video,
433 const TSMM_PRESENTATION_REQUEST* req)
434{
435 UINT ret = CHANNEL_RC_OK;
436
437 WINPR_ASSERT(video);
438 WINPR_ASSERT(req);
439
440 VideoClientContextPriv* priv = video->priv;
441 WINPR_ASSERT(priv);
442
443 if (req->Command == TSMM_START_PRESENTATION)
444 {
445 MAPPED_GEOMETRY* geom = NULL;
446 TSMM_PRESENTATION_RESPONSE resp;
447
448 if (memcmp(req->VideoSubtypeId, MFVideoFormat_H264, 16) != 0)
449 {
450 WLog_ERR(TAG, "not a H264 video, ignoring request");
451 return CHANNEL_RC_OK;
452 }
453
454 if (priv->currentPresentation)
455 {
456 if (priv->currentPresentation->PresentationId == req->PresentationId)
457 {
458 WLog_ERR(TAG, "ignoring start request for existing presentation %" PRIu8,
459 req->PresentationId);
460 return CHANNEL_RC_OK;
461 }
462
463 WLog_ERR(TAG, "releasing current presentation %" PRIu8, req->PresentationId);
464 PresentationContext_unref(&priv->currentPresentation);
465 }
466
467 if (!priv->geometry)
468 {
469 WLog_ERR(TAG, "geometry channel not ready, ignoring request");
470 return CHANNEL_RC_OK;
471 }
472
473 geom = HashTable_GetItemValue(priv->geometry->geometries, &(req->GeometryMappingId));
474 if (!geom)
475 {
476 WLog_ERR(TAG, "geometry mapping 0x%" PRIx64 " not registered", req->GeometryMappingId);
477 return CHANNEL_RC_OK;
478 }
479
480 WLog_DBG(TAG, "creating presentation 0x%x", req->PresentationId);
481 priv->currentPresentation = PresentationContext_new(
482 video, req->PresentationId,
483 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelLeft + geom->left),
484 WINPR_ASSERTING_INT_CAST(uint32_t, geom->topLevelTop + geom->top), req->SourceWidth,
485 req->SourceHeight);
486 if (!priv->currentPresentation)
487 {
488 WLog_ERR(TAG, "unable to create presentation video");
489 return CHANNEL_RC_NO_MEMORY;
490 }
491
492 mappedGeometryRef(geom);
493 priv->currentPresentation->geometry = geom;
494
495 priv->currentPresentation->video = video;
496 priv->currentPresentation->ScaledWidth = req->ScaledWidth;
497 priv->currentPresentation->ScaledHeight = req->ScaledHeight;
498
499 geom->custom = priv->currentPresentation;
500 geom->MappedGeometryUpdate = video_onMappedGeometryUpdate;
501 geom->MappedGeometryClear = video_onMappedGeometryClear;
502
503 /* send back response */
504 resp.PresentationId = req->PresentationId;
505 ret = video_control_send_presentation_response(video, &resp);
506 }
507 else if (req->Command == TSMM_STOP_PRESENTATION)
508 {
509 WLog_DBG(TAG, "stopping presentation 0x%x", req->PresentationId);
510 if (!priv->currentPresentation)
511 {
512 WLog_ERR(TAG, "unknown presentation to stop %" PRIu8, req->PresentationId);
513 return CHANNEL_RC_OK;
514 }
515
516 priv->droppedFrames = 0;
517 priv->publishedFrames = 0;
518 PresentationContext_unref(&priv->currentPresentation);
519 }
520
521 return ret;
522}
523
524static UINT video_read_tsmm_presentation_req(VideoClientContext* context, wStream* s)
525{
526 TSMM_PRESENTATION_REQUEST req = { 0 };
527
528 WINPR_ASSERT(context);
529 WINPR_ASSERT(s);
530
531 if (!Stream_CheckAndLogRequiredLength(TAG, s, 60))
532 return ERROR_INVALID_DATA;
533
534 Stream_Read_UINT8(s, req.PresentationId);
535 Stream_Read_UINT8(s, req.Version);
536 Stream_Read_UINT8(s, req.Command);
537 Stream_Read_UINT8(s, req.FrameRate); /* FrameRate - reserved and ignored */
538
539 Stream_Seek_UINT16(s); /* AverageBitrateKbps reserved and ignored */
540 Stream_Seek_UINT16(s); /* reserved */
541
542 Stream_Read_UINT32(s, req.SourceWidth);
543 Stream_Read_UINT32(s, req.SourceHeight);
544 Stream_Read_UINT32(s, req.ScaledWidth);
545 Stream_Read_UINT32(s, req.ScaledHeight);
546 Stream_Read_UINT64(s, req.hnsTimestampOffset);
547 Stream_Read_UINT64(s, req.GeometryMappingId);
548 Stream_Read(s, req.VideoSubtypeId, 16);
549
550 Stream_Read_UINT32(s, req.cbExtra);
551
552 if (!Stream_CheckAndLogRequiredLength(TAG, s, req.cbExtra))
553 return ERROR_INVALID_DATA;
554
555 req.pExtraData = Stream_Pointer(s);
556
557 WLog_DBG(TAG,
558 "presentationReq: id:%" PRIu8 " version:%" PRIu8
559 " command:%s srcWidth/srcHeight=%" PRIu32 "x%" PRIu32 " scaled Width/Height=%" PRIu32
560 "x%" PRIu32 " timestamp=%" PRIu64 " mappingId=%" PRIx64 "",
561 req.PresentationId, req.Version, video_command_name(req.Command), req.SourceWidth,
562 req.SourceHeight, req.ScaledWidth, req.ScaledHeight, req.hnsTimestampOffset,
563 req.GeometryMappingId);
564
565 return video_PresentationRequest(context, &req);
566}
567
573static UINT video_control_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
574{
575 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
576 VIDEO_PLUGIN* video = NULL;
577 VideoClientContext* context = NULL;
578 UINT ret = CHANNEL_RC_OK;
579 UINT32 cbSize = 0;
580 UINT32 packetType = 0;
581
582 WINPR_ASSERT(callback);
583 WINPR_ASSERT(s);
584
585 video = (VIDEO_PLUGIN*)callback->plugin;
586 WINPR_ASSERT(video);
587
588 context = (VideoClientContext*)video->wtsPlugin.pInterface;
589 WINPR_ASSERT(context);
590
591 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
592 return ERROR_INVALID_DATA;
593
594 Stream_Read_UINT32(s, cbSize);
595 if (cbSize < 8)
596 {
597 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected 8", cbSize);
598 return ERROR_INVALID_DATA;
599 }
600 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
601 return ERROR_INVALID_DATA;
602
603 Stream_Read_UINT32(s, packetType);
604 switch (packetType)
605 {
606 case TSMM_PACKET_TYPE_PRESENTATION_REQUEST:
607 ret = video_read_tsmm_presentation_req(context, s);
608 break;
609 default:
610 WLog_ERR(TAG, "not expecting packet type %" PRIu32 "", packetType);
611 ret = ERROR_UNSUPPORTED_TYPE;
612 break;
613 }
614
615 return ret;
616}
617
618static UINT video_control_send_client_notification(VideoClientContext* context,
619 const TSMM_CLIENT_NOTIFICATION* notif)
620{
621 BYTE buf[100];
622 wStream* s = NULL;
623 VIDEO_PLUGIN* video = NULL;
624 IWTSVirtualChannel* channel = NULL;
625 UINT ret = 0;
626 UINT32 cbSize = 0;
627
628 WINPR_ASSERT(context);
629 WINPR_ASSERT(notif);
630
631 video = (VIDEO_PLUGIN*)context->handle;
632 WINPR_ASSERT(video);
633
634 s = Stream_New(buf, 32);
635 if (!s)
636 return CHANNEL_RC_NO_MEMORY;
637
638 cbSize = 16;
639 Stream_Seek_UINT32(s); /* cbSize */
640 Stream_Write_UINT32(s, TSMM_PACKET_TYPE_CLIENT_NOTIFICATION); /* PacketType */
641 Stream_Write_UINT8(s, notif->PresentationId);
642 Stream_Write_UINT8(s, notif->NotificationType);
643 Stream_Zero(s, 2);
644 if (notif->NotificationType == TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE)
645 {
646 Stream_Write_UINT32(s, 16); /* cbData */
647
648 /* TSMM_CLIENT_NOTIFICATION_FRAMERATE_OVERRIDE */
649 Stream_Write_UINT32(s, notif->FramerateOverride.Flags);
650 Stream_Write_UINT32(s, notif->FramerateOverride.DesiredFrameRate);
651 Stream_Zero(s, 4ULL * 2ULL);
652
653 cbSize += 4UL * 4UL;
654 }
655 else
656 {
657 Stream_Write_UINT32(s, 0); /* cbData */
658 }
659
660 Stream_SealLength(s);
661 Stream_SetPosition(s, 0);
662 Stream_Write_UINT32(s, cbSize);
663 Stream_Free(s, FALSE);
664
665 WINPR_ASSERT(video->control_callback);
666 WINPR_ASSERT(video->control_callback->channel_callback);
667
668 channel = video->control_callback->channel_callback->channel;
669 WINPR_ASSERT(channel);
670 WINPR_ASSERT(channel->Write);
671
672 ret = channel->Write(channel, cbSize, buf, NULL);
673
674 return ret;
675}
676
677static void video_timer(VideoClientContext* video, UINT64 now)
678{
679 PresentationContext* presentation = NULL;
680 VideoClientContextPriv* priv = NULL;
681 VideoFrame* peekFrame = NULL;
682 VideoFrame* frame = NULL;
683
684 WINPR_ASSERT(video);
685
686 priv = video->priv;
687 WINPR_ASSERT(priv);
688
689 EnterCriticalSection(&priv->framesLock);
690 do
691 {
692 peekFrame = (VideoFrame*)Queue_Peek(priv->frames);
693 if (!peekFrame)
694 break;
695
696 if (peekFrame->publishTime > now)
697 break;
698
699 if (frame)
700 {
701 WLog_DBG(TAG, "dropping frame @%" PRIu64, frame->publishTime);
702 priv->droppedFrames++;
703 VideoFrame_free(&frame);
704 }
705 frame = peekFrame;
706 Queue_Dequeue(priv->frames);
707 } while (1);
708 LeaveCriticalSection(&priv->framesLock);
709
710 if (!frame)
711 goto treat_feedback;
712
713 presentation = frame->presentation;
714
715 priv->publishedFrames++;
716 memcpy(presentation->surface->data, frame->surfaceData, 1ull * frame->scanline * frame->h);
717
718 WINPR_ASSERT(video->showSurface);
719 video->showSurface(video, presentation->surface, presentation->ScaledWidth,
720 presentation->ScaledHeight);
721
722 VideoFrame_free(&frame);
723
724treat_feedback:
725 if (priv->nextFeedbackTime < now)
726 {
727 /* we can compute some feedback only if we have some published frames and
728 * a current presentation
729 */
730 if (priv->publishedFrames && priv->currentPresentation)
731 {
732 UINT32 computedRate = 0;
733
734 PresentationContext_ref(priv->currentPresentation);
735
736 if (priv->droppedFrames)
737 {
743 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
744 computedRate = 24;
745 else
746 {
747 computedRate = priv->lastSentRate - 2;
748 if (!computedRate)
749 computedRate = 2;
750 }
751 }
752 else
753 {
758 if (priv->lastSentRate == XF_VIDEO_UNLIMITED_RATE)
759 computedRate = XF_VIDEO_UNLIMITED_RATE; /* stay unlimited */
760 else
761 {
762 computedRate = priv->lastSentRate + 2;
763 if (computedRate > XF_VIDEO_UNLIMITED_RATE)
764 computedRate = XF_VIDEO_UNLIMITED_RATE;
765 }
766 }
767
768 if (computedRate != priv->lastSentRate)
769 {
770 TSMM_CLIENT_NOTIFICATION notif;
771
772 WINPR_ASSERT(priv->currentPresentation);
773 notif.PresentationId = priv->currentPresentation->PresentationId;
774 notif.NotificationType = TSMM_CLIENT_NOTIFICATION_TYPE_FRAMERATE_OVERRIDE;
775 if (computedRate == XF_VIDEO_UNLIMITED_RATE)
776 {
777 notif.FramerateOverride.Flags = 0x01;
778 notif.FramerateOverride.DesiredFrameRate = 0x00;
779 }
780 else
781 {
782 notif.FramerateOverride.Flags = 0x02;
783 notif.FramerateOverride.DesiredFrameRate = computedRate;
784 }
785
786 video_control_send_client_notification(video, &notif);
787 priv->lastSentRate = computedRate;
788
789 WLog_DBG(TAG,
790 "server notified with rate %" PRIu32 " published=%" PRIu32
791 " dropped=%" PRIu32,
792 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
793 }
794
795 PresentationContext_unref(&priv->currentPresentation);
796 }
797
798 WLog_DBG(TAG, "currentRate=%" PRIu32 " published=%" PRIu32 " dropped=%" PRIu32,
799 priv->lastSentRate, priv->publishedFrames, priv->droppedFrames);
800
801 priv->droppedFrames = 0;
802 priv->publishedFrames = 0;
803 priv->nextFeedbackTime = now + 1000;
804 }
805}
806
807static UINT video_VideoData(VideoClientContext* context, const TSMM_VIDEO_DATA* data)
808{
809 VideoClientContextPriv* priv = NULL;
810 PresentationContext* presentation = NULL;
811 int status = 0;
812
813 WINPR_ASSERT(context);
814 WINPR_ASSERT(data);
815
816 priv = context->priv;
817 WINPR_ASSERT(priv);
818
819 presentation = priv->currentPresentation;
820 if (!presentation)
821 {
822 WLog_ERR(TAG, "no current presentation");
823 return CHANNEL_RC_OK;
824 }
825
826 if (presentation->PresentationId != data->PresentationId)
827 {
828 WLog_ERR(TAG, "current presentation id=%" PRIu8 " doesn't match data id=%" PRIu8,
829 presentation->PresentationId, data->PresentationId);
830 return CHANNEL_RC_OK;
831 }
832
833 if (!Stream_EnsureRemainingCapacity(presentation->currentSample, data->cbSample))
834 {
835 WLog_ERR(TAG, "unable to expand the current packet");
836 return CHANNEL_RC_NO_MEMORY;
837 }
838
839 Stream_Write(presentation->currentSample, data->pSample, data->cbSample);
840
841 if (data->CurrentPacketIndex == data->PacketsInSample)
842 {
843 VideoSurface* surface = presentation->surface;
844 H264_CONTEXT* h264 = presentation->h264;
845 UINT64 startTime = GetTickCount64();
846 UINT64 timeAfterH264 = 0;
847 MAPPED_GEOMETRY* geom = presentation->geometry;
848
849 const RECTANGLE_16 rect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedWidth),
850 WINPR_ASSERTING_INT_CAST(UINT16, surface->alignedHeight) };
851 Stream_SealLength(presentation->currentSample);
852 Stream_SetPosition(presentation->currentSample, 0);
853
854 timeAfterH264 = GetTickCount64();
855 if (data->SampleNumber == 1)
856 {
857 presentation->lastPublishTime = startTime;
858 }
859
860 presentation->lastPublishTime += (data->hnsDuration / 10000);
861 if (presentation->lastPublishTime <= timeAfterH264 + 10)
862 {
863 int dropped = 0;
864
865 const size_t len = Stream_Length(presentation->currentSample);
866 if (len > UINT32_MAX)
867 return CHANNEL_RC_OK;
868
869 /* if the frame is to be published in less than 10 ms, let's consider it's now */
870 status =
871 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
872 surface->data, surface->format, surface->scanline,
873 surface->alignedWidth, surface->alignedHeight, &rect, 1);
874
875 if (status < 0)
876 return CHANNEL_RC_OK;
877
878 WINPR_ASSERT(context->showSurface);
879 context->showSurface(context, presentation->surface, presentation->ScaledWidth,
880 presentation->ScaledHeight);
881
882 priv->publishedFrames++;
883
884 /* cleanup previously scheduled frames */
885 EnterCriticalSection(&priv->framesLock);
886 while (Queue_Count(priv->frames) > 0)
887 {
888 VideoFrame* frame = Queue_Dequeue(priv->frames);
889 if (frame)
890 {
891 priv->droppedFrames++;
892 VideoFrame_free(&frame);
893 dropped++;
894 }
895 }
896 LeaveCriticalSection(&priv->framesLock);
897
898 if (dropped)
899 WLog_DBG(TAG, "showing frame (%d dropped)", dropped);
900 }
901 else
902 {
903 const size_t len = Stream_Length(presentation->currentSample);
904 if (len > UINT32_MAX)
905 return CHANNEL_RC_OK;
906
907 BOOL enqueueResult = 0;
908 VideoFrame* frame = VideoFrame_new(priv, presentation, geom);
909 if (!frame)
910 {
911 WLog_ERR(TAG, "unable to create frame");
912 return CHANNEL_RC_NO_MEMORY;
913 }
914
915 status =
916 avc420_decompress(h264, Stream_Pointer(presentation->currentSample), (UINT32)len,
917 frame->surfaceData, surface->format, surface->scanline,
918 surface->alignedWidth, surface->alignedHeight, &rect, 1);
919 if (status < 0)
920 {
921 VideoFrame_free(&frame);
922 return CHANNEL_RC_OK;
923 }
924
925 EnterCriticalSection(&priv->framesLock);
926 enqueueResult = Queue_Enqueue(priv->frames, frame);
927 LeaveCriticalSection(&priv->framesLock);
928
929 if (!enqueueResult)
930 {
931 WLog_ERR(TAG, "unable to enqueue frame");
932 VideoFrame_free(&frame);
933 return CHANNEL_RC_NO_MEMORY;
934 }
935
936 // NOLINTNEXTLINE(clang-analyzer-unix.Malloc): Queue_Enqueue owns frame
937 WLog_DBG(TAG, "scheduling frame in %" PRIu32 " ms", (frame->publishTime - startTime));
938 }
939 }
940
941 return CHANNEL_RC_OK;
942}
943
944static UINT video_data_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, wStream* s)
945{
946 GENERIC_CHANNEL_CALLBACK* callback = (GENERIC_CHANNEL_CALLBACK*)pChannelCallback;
947 VIDEO_PLUGIN* video = NULL;
948 VideoClientContext* context = NULL;
949 UINT32 cbSize = 0;
950 UINT32 packetType = 0;
951 TSMM_VIDEO_DATA data;
952
953 video = (VIDEO_PLUGIN*)callback->plugin;
954 context = (VideoClientContext*)video->wtsPlugin.pInterface;
955
956 if (!Stream_CheckAndLogRequiredLength(TAG, s, 4))
957 return ERROR_INVALID_DATA;
958
959 Stream_Read_UINT32(s, cbSize);
960 if (cbSize < 8)
961 {
962 WLog_ERR(TAG, "invalid cbSize %" PRIu32 ", expected >= 8", cbSize);
963 return ERROR_INVALID_DATA;
964 }
965
966 if (!Stream_CheckAndLogRequiredLength(TAG, s, cbSize - 4))
967 return ERROR_INVALID_DATA;
968
969 Stream_Read_UINT32(s, packetType);
970 if (packetType != TSMM_PACKET_TYPE_VIDEO_DATA)
971 {
972 WLog_ERR(TAG, "only expecting VIDEO_DATA on the data channel");
973 return ERROR_INVALID_DATA;
974 }
975
976 if (!Stream_CheckAndLogRequiredLength(TAG, s, 32))
977 return ERROR_INVALID_DATA;
978
979 Stream_Read_UINT8(s, data.PresentationId);
980 Stream_Read_UINT8(s, data.Version);
981 Stream_Read_UINT8(s, data.Flags);
982 Stream_Seek_UINT8(s); /* reserved */
983 Stream_Read_UINT64(s, data.hnsTimestamp);
984 Stream_Read_UINT64(s, data.hnsDuration);
985 Stream_Read_UINT16(s, data.CurrentPacketIndex);
986 Stream_Read_UINT16(s, data.PacketsInSample);
987 Stream_Read_UINT32(s, data.SampleNumber);
988 Stream_Read_UINT32(s, data.cbSample);
989 if (!Stream_CheckAndLogRequiredLength(TAG, s, data.cbSample))
990 return ERROR_INVALID_DATA;
991 data.pSample = Stream_Pointer(s);
992
993 /*
994 WLog_DBG(TAG, "videoData: id:%"PRIu8" version:%"PRIu8" flags:0x%"PRIx8" timestamp=%"PRIu64"
995 duration=%"PRIu64 " curPacketIndex:%"PRIu16" packetInSample:%"PRIu16" sampleNumber:%"PRIu32"
996 cbSample:%"PRIu32"", data.PresentationId, data.Version, data.Flags, data.hnsTimestamp,
997 data.hnsDuration, data.CurrentPacketIndex, data.PacketsInSample, data.SampleNumber,
998 data.cbSample);
999 */
1000
1001 return video_VideoData(context, &data);
1002}
1003
1009static UINT video_control_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1010{
1011 free(pChannelCallback);
1012 return CHANNEL_RC_OK;
1013}
1014
1015static UINT video_data_on_close(IWTSVirtualChannelCallback* pChannelCallback)
1016{
1017 free(pChannelCallback);
1018 return CHANNEL_RC_OK;
1019}
1020
1026// NOLINTBEGIN(readability-non-const-parameter)
1027static UINT video_control_on_new_channel_connection(IWTSListenerCallback* listenerCallback,
1028 IWTSVirtualChannel* channel, BYTE* Data,
1029 BOOL* pbAccept,
1030 IWTSVirtualChannelCallback** ppCallback)
1031// NOLINTEND(readability-non-const-parameter)
1032{
1033 GENERIC_CHANNEL_CALLBACK* callback = NULL;
1034 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)listenerCallback;
1035
1036 WINPR_UNUSED(Data);
1037 WINPR_UNUSED(pbAccept);
1038
1039 callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1040 if (!callback)
1041 {
1042 WLog_ERR(TAG, "calloc failed!");
1043 return CHANNEL_RC_NO_MEMORY;
1044 }
1045
1046 callback->iface.OnDataReceived = video_control_on_data_received;
1047 callback->iface.OnClose = video_control_on_close;
1048 callback->plugin = listener_callback->plugin;
1049 callback->channel_mgr = listener_callback->channel_mgr;
1050 callback->channel = channel;
1051 listener_callback->channel_callback = callback;
1052
1053 *ppCallback = (IWTSVirtualChannelCallback*)callback;
1054
1055 return CHANNEL_RC_OK;
1056}
1057
1058// NOLINTBEGIN(readability-non-const-parameter)
1059static UINT video_data_on_new_channel_connection(IWTSListenerCallback* pListenerCallback,
1060 IWTSVirtualChannel* pChannel, BYTE* Data,
1061 BOOL* pbAccept,
1062 IWTSVirtualChannelCallback** ppCallback)
1063// NOLINTEND(readability-non-const-parameter)
1064{
1065 GENERIC_CHANNEL_CALLBACK* callback = NULL;
1066 GENERIC_LISTENER_CALLBACK* listener_callback = (GENERIC_LISTENER_CALLBACK*)pListenerCallback;
1067
1068 WINPR_UNUSED(Data);
1069 WINPR_UNUSED(pbAccept);
1070
1071 callback = (GENERIC_CHANNEL_CALLBACK*)calloc(1, sizeof(GENERIC_CHANNEL_CALLBACK));
1072 if (!callback)
1073 {
1074 WLog_ERR(TAG, "calloc failed!");
1075 return CHANNEL_RC_NO_MEMORY;
1076 }
1077
1078 callback->iface.OnDataReceived = video_data_on_data_received;
1079 callback->iface.OnClose = video_data_on_close;
1080 callback->plugin = listener_callback->plugin;
1081 callback->channel_mgr = listener_callback->channel_mgr;
1082 callback->channel = pChannel;
1083 listener_callback->channel_callback = callback;
1084
1085 *ppCallback = (IWTSVirtualChannelCallback*)callback;
1086
1087 return CHANNEL_RC_OK;
1088}
1089
1095static UINT video_plugin_initialize(IWTSPlugin* plugin, IWTSVirtualChannelManager* channelMgr)
1096{
1097 UINT status = 0;
1098 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)plugin;
1099 GENERIC_LISTENER_CALLBACK* callback = NULL;
1100
1101 if (video->initialized)
1102 {
1103 WLog_ERR(TAG, "[%s] channel initialized twice, aborting", VIDEO_CONTROL_DVC_CHANNEL_NAME);
1104 return ERROR_INVALID_DATA;
1105 }
1106 video->control_callback = callback =
1107 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1108 if (!callback)
1109 {
1110 WLog_ERR(TAG, "calloc for control callback failed!");
1111 return CHANNEL_RC_NO_MEMORY;
1112 }
1113
1114 callback->iface.OnNewChannelConnection = video_control_on_new_channel_connection;
1115 callback->plugin = plugin;
1116 callback->channel_mgr = channelMgr;
1117
1118 status = channelMgr->CreateListener(channelMgr, VIDEO_CONTROL_DVC_CHANNEL_NAME, 0,
1119 &callback->iface, &(video->controlListener));
1120
1121 if (status != CHANNEL_RC_OK)
1122 return status;
1123 video->controlListener->pInterface = video->wtsPlugin.pInterface;
1124
1125 video->data_callback = callback =
1126 (GENERIC_LISTENER_CALLBACK*)calloc(1, sizeof(GENERIC_LISTENER_CALLBACK));
1127 if (!callback)
1128 {
1129 WLog_ERR(TAG, "calloc for data callback failed!");
1130 return CHANNEL_RC_NO_MEMORY;
1131 }
1132
1133 callback->iface.OnNewChannelConnection = video_data_on_new_channel_connection;
1134 callback->plugin = plugin;
1135 callback->channel_mgr = channelMgr;
1136
1137 status = channelMgr->CreateListener(channelMgr, VIDEO_DATA_DVC_CHANNEL_NAME, 0,
1138 &callback->iface, &(video->dataListener));
1139
1140 if (status == CHANNEL_RC_OK)
1141 video->dataListener->pInterface = video->wtsPlugin.pInterface;
1142
1143 video->initialized = status == CHANNEL_RC_OK;
1144 return status;
1145}
1146
1152static UINT video_plugin_terminated(IWTSPlugin* pPlugin)
1153{
1154 VIDEO_PLUGIN* video = (VIDEO_PLUGIN*)pPlugin;
1155
1156 if (video->control_callback)
1157 {
1158 IWTSVirtualChannelManager* mgr = video->control_callback->channel_mgr;
1159 if (mgr)
1160 IFCALL(mgr->DestroyListener, mgr, video->controlListener);
1161 }
1162 if (video->data_callback)
1163 {
1164 IWTSVirtualChannelManager* mgr = video->data_callback->channel_mgr;
1165 if (mgr)
1166 IFCALL(mgr->DestroyListener, mgr, video->dataListener);
1167 }
1168
1169 if (video->context)
1170 VideoClientContextPriv_free(video->context->priv);
1171
1172 free(video->control_callback);
1173 free(video->data_callback);
1174 free(video->wtsPlugin.pInterface);
1175 free(pPlugin);
1176 return CHANNEL_RC_OK;
1177}
1178
1187FREERDP_ENTRY_POINT(UINT VCAPITYPE video_DVCPluginEntry(IDRDYNVC_ENTRY_POINTS* pEntryPoints))
1188{
1189 UINT error = CHANNEL_RC_OK;
1190 VIDEO_PLUGIN* videoPlugin = NULL;
1191 VideoClientContext* videoContext = NULL;
1192 VideoClientContextPriv* priv = NULL;
1193
1194 videoPlugin = (VIDEO_PLUGIN*)pEntryPoints->GetPlugin(pEntryPoints, "video");
1195 if (!videoPlugin)
1196 {
1197 videoPlugin = (VIDEO_PLUGIN*)calloc(1, sizeof(VIDEO_PLUGIN));
1198 if (!videoPlugin)
1199 {
1200 WLog_ERR(TAG, "calloc failed!");
1201 return CHANNEL_RC_NO_MEMORY;
1202 }
1203
1204 videoPlugin->wtsPlugin.Initialize = video_plugin_initialize;
1205 videoPlugin->wtsPlugin.Connected = NULL;
1206 videoPlugin->wtsPlugin.Disconnected = NULL;
1207 videoPlugin->wtsPlugin.Terminated = video_plugin_terminated;
1208
1209 videoContext = (VideoClientContext*)calloc(1, sizeof(VideoClientContext));
1210 if (!videoContext)
1211 {
1212 WLog_ERR(TAG, "calloc failed!");
1213 free(videoPlugin);
1214 return CHANNEL_RC_NO_MEMORY;
1215 }
1216
1217 priv = VideoClientContextPriv_new(videoContext);
1218 if (!priv)
1219 {
1220 WLog_ERR(TAG, "VideoClientContextPriv_new failed!");
1221 free(videoContext);
1222 free(videoPlugin);
1223 return CHANNEL_RC_NO_MEMORY;
1224 }
1225
1226 videoContext->handle = (void*)videoPlugin;
1227 videoContext->priv = priv;
1228 videoContext->timer = video_timer;
1229 videoContext->setGeometry = video_client_context_set_geometry;
1230
1231 videoPlugin->wtsPlugin.pInterface = (void*)videoContext;
1232 videoPlugin->context = videoContext;
1233
1234 error = pEntryPoints->RegisterPlugin(pEntryPoints, "video", &videoPlugin->wtsPlugin);
1235 }
1236 else
1237 {
1238 WLog_ERR(TAG, "could not get video Plugin.");
1239 return CHANNEL_RC_BAD_CHANNEL;
1240 }
1241
1242 return error;
1243}
GENERIC_CHANNEL_CALLBACK
Definition include/freerdp/client/channels.h:35
GENERIC_LISTENER_CALLBACK
Definition include/freerdp/client/channels.h:43
RDP_RECT
Definition types.h:101
RECTANGLE_16
Definition types.h:109
RTL_CRITICAL_SECTION
Definition include/winpr/synch.h:158
TSMM_CLIENT_NOTIFICATION
a client to server notification struct
Definition channels/video.h:116
TSMM_PRESENTATION_REQUEST
presentation request struct
Definition channels/video.h:57
TSMM_PRESENTATION_RESPONSE
response to a TSMM_PRESENTATION_REQUEST
Definition channels/video.h:73
TSMM_VIDEO_DATA
a video data packet
Definition channels/video.h:87
VideoSurface
an implementation of surface used by the video channel
Definition client/video.h:36