FreeRDP
Loading...
Searching...
No Matches
tsmf_gstreamer.c
1/*
2 * FreeRDP: A Remote Desktop Protocol Implementation
3 * Video Redirection Virtual Channel - GStreamer Decoder
4 *
5 * (C) Copyright 2012 HP Development Company, LLC
6 * (C) Copyright 2014 Thincast Technologies GmbH
7 * (C) Copyright 2014 Armin Novak <armin.novak@thincast.com>
8 *
9 * Licensed under the Apache License, Version 2.0 (the "License");
10 * you may not use this file except in compliance with the License.
11 * You may obtain a copy of the License at
12 *
13 * http://www.apache.org/licenses/LICENSE-2.0
14 *
15 * Unless required by applicable law or agreed to in writing, software
16 * distributed under the License is distributed on an "AS IS" BASIS,
17 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 * See the License for the specific language governing permissions and
19 * limitations under the License.
20 */
21
22#include <freerdp/config.h>
23
24#include <winpr/assert.h>
25
26#include <fcntl.h>
27#include <stdio.h>
28#include <stdlib.h>
29#include <string.h>
30#include <unistd.h>
31
32#include <winpr/string.h>
33#include <winpr/platform.h>
34
35#include <gst/gst.h>
36
37#include <gst/app/gstappsrc.h>
38#include <gst/app/gstappsink.h>
39
40#include "tsmf_constants.h"
41#include "tsmf_decoder.h"
42#include "tsmf_platform.h"
43
44/* 1 second = 10,000,000 100ns units*/
45#define SEEK_TOLERANCE 10 * 1000 * 1000
46
47static BOOL tsmf_gstreamer_pipeline_build(TSMFGstreamerDecoder* mdecoder);
48static void tsmf_gstreamer_clean_up(TSMFGstreamerDecoder* mdecoder);
49static int tsmf_gstreamer_pipeline_set_state(TSMFGstreamerDecoder* mdecoder,
50 GstState desired_state);
51static BOOL tsmf_gstreamer_buffer_level(ITSMFDecoder* decoder);
52
53static const char* get_type(TSMFGstreamerDecoder* mdecoder)
54{
55 if (!mdecoder)
56 return NULL;
57
58 switch (mdecoder->media_type)
59 {
60 case TSMF_MAJOR_TYPE_VIDEO:
61 return "VIDEO";
62 case TSMF_MAJOR_TYPE_AUDIO:
63 return "AUDIO";
64 default:
65 return "UNKNOWN";
66 }
67}
68
69static void cb_child_added(GstChildProxy* child_proxy, GObject* object,
70 TSMFGstreamerDecoder* mdecoder)
71{
72 DEBUG_TSMF("NAME: %s", G_OBJECT_TYPE_NAME(object));
73
74 if (!g_strcmp0(G_OBJECT_TYPE_NAME(object), "GstXvImageSink") ||
75 !g_strcmp0(G_OBJECT_TYPE_NAME(object), "GstXImageSink") ||
76 !g_strcmp0(G_OBJECT_TYPE_NAME(object), "GstFluVAAutoSink"))
77 {
78 gst_base_sink_set_max_lateness((GstBaseSink*)object, 10000000); /* nanoseconds */
79 g_object_set(G_OBJECT(object), "sync", TRUE, NULL); /* synchronize on the clock */
80 g_object_set(G_OBJECT(object), "async", TRUE, NULL); /* no async state changes */
81 }
82
83 else if (!g_strcmp0(G_OBJECT_TYPE_NAME(object), "GstAlsaSink") ||
84 !g_strcmp0(G_OBJECT_TYPE_NAME(object), "GstPulseSink"))
85 {
86 gst_base_sink_set_max_lateness((GstBaseSink*)object, 10000000); /* nanoseconds */
87 g_object_set(G_OBJECT(object), "slave-method", 1, NULL);
88 g_object_set(G_OBJECT(object), "buffer-time", (gint64)20000, NULL); /* microseconds */
89 g_object_set(G_OBJECT(object), "drift-tolerance", (gint64)20000, NULL); /* microseconds */
90 g_object_set(G_OBJECT(object), "latency-time", (gint64)10000, NULL); /* microseconds */
91 g_object_set(G_OBJECT(object), "sync", TRUE, NULL); /* synchronize on the clock */
92 g_object_set(G_OBJECT(object), "async", TRUE, NULL); /* no async state changes */
93 }
94}
95
96static void tsmf_gstreamer_enough_data(GstAppSrc* src, gpointer user_data)
97{
98 TSMFGstreamerDecoder* mdecoder = user_data;
99 (void)mdecoder;
100 DEBUG_TSMF("%s", get_type(mdecoder));
101}
102
103static void tsmf_gstreamer_need_data(GstAppSrc* src, guint length, gpointer user_data)
104{
105 TSMFGstreamerDecoder* mdecoder = user_data;
106 (void)mdecoder;
107 DEBUG_TSMF("%s length=%u", get_type(mdecoder), length);
108}
109
110static gboolean tsmf_gstreamer_seek_data(GstAppSrc* src, guint64 offset, gpointer user_data)
111{
112 TSMFGstreamerDecoder* mdecoder = user_data;
113 (void)mdecoder;
114 DEBUG_TSMF("%s offset=%" PRIu64 "", get_type(mdecoder), offset);
115
116 return TRUE;
117}
118
119static BOOL tsmf_gstreamer_change_volume(ITSMFDecoder* decoder, UINT32 newVolume, UINT32 muted)
120{
121 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
122
123 if (!mdecoder || !mdecoder->pipe)
124 return TRUE;
125
126 if (mdecoder->media_type == TSMF_MAJOR_TYPE_VIDEO)
127 return TRUE;
128
129 mdecoder->gstMuted = (BOOL)muted;
130 DEBUG_TSMF("mute=[%" PRId32 "]", mdecoder->gstMuted);
131 mdecoder->gstVolume = (double)newVolume / (double)10000;
132 DEBUG_TSMF("gst_new_vol=[%f]", mdecoder->gstVolume);
133
134 if (!mdecoder->volume)
135 return TRUE;
136
137 if (!G_IS_OBJECT(mdecoder->volume))
138 return TRUE;
139
140 g_object_set(mdecoder->volume, "mute", mdecoder->gstMuted, NULL);
141 g_object_set(mdecoder->volume, "volume", mdecoder->gstVolume, NULL);
142
143 return TRUE;
144}
145
146static inline GstClockTime tsmf_gstreamer_timestamp_ms_to_gst(UINT64 ms_timestamp)
147{
148 /*
149 * Convert Microsoft 100ns timestamps to Gstreamer 1ns units.
150 */
151 return (GstClockTime)(ms_timestamp * 100);
152}
153
154int tsmf_gstreamer_pipeline_set_state(TSMFGstreamerDecoder* mdecoder, GstState desired_state)
155{
156 GstStateChangeReturn state_change;
157 const char* name;
158 const char* sname = get_type(mdecoder);
159
160 if (!mdecoder)
161 return 0;
162
163 if (!mdecoder->pipe)
164 return 0; /* Just in case this is called during startup or shutdown when we don't expect it
165 */
166
167 if (desired_state == mdecoder->state)
168 return 0; /* Redundant request - Nothing to do */
169
170 name = gst_element_state_get_name(desired_state); /* For debug */
171 DEBUG_TSMF("%s to %s", sname, name);
172 state_change = gst_element_set_state(mdecoder->pipe, desired_state);
173
174 if (state_change == GST_STATE_CHANGE_FAILURE)
175 {
176 WLog_ERR(TAG, "%s: (%s) GST_STATE_CHANGE_FAILURE.", sname, name);
177 }
178 else if (state_change == GST_STATE_CHANGE_ASYNC)
179 {
180 WLog_ERR(TAG, "%s: (%s) GST_STATE_CHANGE_ASYNC.", sname, name);
181 mdecoder->state = desired_state;
182 }
183 else
184 {
185 mdecoder->state = desired_state;
186 }
187
188 return 0;
189}
190
191static GstBuffer* tsmf_get_buffer_from_data(const void* raw_data, gsize size)
192{
193 GstBuffer* buffer;
194 gpointer data;
195
196 if (!raw_data)
197 return NULL;
198
199 if (size < 1)
200 return NULL;
201
202 data = g_malloc(size);
203
204 if (!data)
205 {
206 WLog_ERR(TAG, "Could not allocate %" G_GSIZE_FORMAT " bytes of data.", size);
207 return NULL;
208 }
209
210 CopyMemory(data, raw_data, size);
211
212#if GST_VERSION_MAJOR > 0
213 buffer = gst_buffer_new_wrapped(data, size);
214#else
215 buffer = gst_buffer_new();
216
217 if (!buffer)
218 {
219 WLog_ERR(TAG, "Could not create GstBuffer");
220 free(data);
221 return NULL;
222 }
223
224 GST_BUFFER_MALLOCDATA(buffer) = data;
225 GST_BUFFER_SIZE(buffer) = size;
226 GST_BUFFER_DATA(buffer) = GST_BUFFER_MALLOCDATA(buffer);
227#endif
228
229 return buffer;
230}
231
232static BOOL tsmf_gstreamer_set_format(ITSMFDecoder* decoder, TS_AM_MEDIA_TYPE* media_type)
233{
234 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
235
236 if (!mdecoder)
237 return FALSE;
238
239 DEBUG_TSMF("");
240
241 switch (media_type->MajorType)
242 {
243 case TSMF_MAJOR_TYPE_VIDEO:
244 mdecoder->media_type = TSMF_MAJOR_TYPE_VIDEO;
245 break;
246 case TSMF_MAJOR_TYPE_AUDIO:
247 mdecoder->media_type = TSMF_MAJOR_TYPE_AUDIO;
248 break;
249 default:
250 return FALSE;
251 }
252
253 switch (media_type->SubType)
254 {
255 case TSMF_SUB_TYPE_WVC1:
256 mdecoder->gst_caps = gst_caps_new_simple(
257 "video/x-wmv", "bitrate", G_TYPE_UINT, media_type->BitRate, "width", G_TYPE_INT,
258 media_type->Width, "height", G_TYPE_INT, media_type->Height, "wmvversion",
259 G_TYPE_INT, 3,
260#if GST_VERSION_MAJOR > 0
261 "format", G_TYPE_STRING, "WVC1",
262#else
263 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('W', 'V', 'C', '1'),
264#endif
265 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
266 media_type->SamplesPerSecond.Denominator, "pixel-aspect-ratio", GST_TYPE_FRACTION,
267 1, 1, NULL);
268 break;
269 case TSMF_SUB_TYPE_MP4S:
270 mdecoder->gst_caps = gst_caps_new_simple(
271 "video/x-divx", "divxversion", G_TYPE_INT, 5, "bitrate", G_TYPE_UINT,
272 media_type->BitRate, "width", G_TYPE_INT, media_type->Width, "height", G_TYPE_INT,
273 media_type->Height,
274#if GST_VERSION_MAJOR > 0
275 "format", G_TYPE_STRING, "MP42",
276#else
277 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('M', 'P', '4', '2'),
278#endif
279 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
280 media_type->SamplesPerSecond.Denominator, NULL);
281 break;
282 case TSMF_SUB_TYPE_MP42:
283 mdecoder->gst_caps = gst_caps_new_simple(
284 "video/x-msmpeg", "msmpegversion", G_TYPE_INT, 42, "bitrate", G_TYPE_UINT,
285 media_type->BitRate, "width", G_TYPE_INT, media_type->Width, "height", G_TYPE_INT,
286 media_type->Height,
287#if GST_VERSION_MAJOR > 0
288 "format", G_TYPE_STRING, "MP42",
289#else
290 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('M', 'P', '4', '2'),
291#endif
292 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
293 media_type->SamplesPerSecond.Denominator, NULL);
294 break;
295 case TSMF_SUB_TYPE_MP43:
296 mdecoder->gst_caps = gst_caps_new_simple(
297 "video/x-msmpeg", "msmpegversion", G_TYPE_INT, 43, "bitrate", G_TYPE_UINT,
298 media_type->BitRate, "width", G_TYPE_INT, media_type->Width, "height", G_TYPE_INT,
299 media_type->Height,
300#if GST_VERSION_MAJOR > 0
301 "format", G_TYPE_STRING, "MP43",
302#else
303 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('M', 'P', '4', '3'),
304#endif
305 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
306 media_type->SamplesPerSecond.Denominator, NULL);
307 break;
308 case TSMF_SUB_TYPE_M4S2:
309 mdecoder->gst_caps = gst_caps_new_simple(
310 "video/mpeg", "mpegversion", G_TYPE_INT, 4, "width", G_TYPE_INT, media_type->Width,
311 "height", G_TYPE_INT, media_type->Height,
312#if GST_VERSION_MAJOR > 0
313 "format", G_TYPE_STRING, "M4S2",
314#else
315 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('M', '4', 'S', '2'),
316#endif
317 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
318 media_type->SamplesPerSecond.Denominator, NULL);
319 break;
320 case TSMF_SUB_TYPE_WMA9:
321 mdecoder->gst_caps = gst_caps_new_simple(
322 "audio/x-wma", "wmaversion", G_TYPE_INT, 3, "rate", G_TYPE_INT,
323 media_type->SamplesPerSecond.Numerator, "channels", G_TYPE_INT,
324 media_type->Channels, "bitrate", G_TYPE_INT, media_type->BitRate, "depth",
325 G_TYPE_INT, media_type->BitsPerSample, "width", G_TYPE_INT,
326 media_type->BitsPerSample, "block_align", G_TYPE_INT, media_type->BlockAlign, NULL);
327 break;
328 case TSMF_SUB_TYPE_WMA1:
329 mdecoder->gst_caps = gst_caps_new_simple(
330 "audio/x-wma", "wmaversion", G_TYPE_INT, 1, "rate", G_TYPE_INT,
331 media_type->SamplesPerSecond.Numerator, "channels", G_TYPE_INT,
332 media_type->Channels, "bitrate", G_TYPE_INT, media_type->BitRate, "depth",
333 G_TYPE_INT, media_type->BitsPerSample, "width", G_TYPE_INT,
334 media_type->BitsPerSample, "block_align", G_TYPE_INT, media_type->BlockAlign, NULL);
335 break;
336 case TSMF_SUB_TYPE_WMA2:
337 mdecoder->gst_caps = gst_caps_new_simple(
338 "audio/x-wma", "wmaversion", G_TYPE_INT, 2, "rate", G_TYPE_INT,
339 media_type->SamplesPerSecond.Numerator, "channels", G_TYPE_INT,
340 media_type->Channels, "bitrate", G_TYPE_INT, media_type->BitRate, "depth",
341 G_TYPE_INT, media_type->BitsPerSample, "width", G_TYPE_INT,
342 media_type->BitsPerSample, "block_align", G_TYPE_INT, media_type->BlockAlign, NULL);
343 break;
344 case TSMF_SUB_TYPE_MP3:
345 mdecoder->gst_caps =
346 gst_caps_new_simple("audio/mpeg", "mpegversion", G_TYPE_INT, 1, "layer", G_TYPE_INT,
347 3, "rate", G_TYPE_INT, media_type->SamplesPerSecond.Numerator,
348 "channels", G_TYPE_INT, media_type->Channels, NULL);
349 break;
350 case TSMF_SUB_TYPE_WMV1:
351 mdecoder->gst_caps = gst_caps_new_simple(
352 "video/x-wmv", "bitrate", G_TYPE_UINT, media_type->BitRate, "width", G_TYPE_INT,
353 media_type->Width, "height", G_TYPE_INT, media_type->Height, "wmvversion",
354 G_TYPE_INT, 1,
355#if GST_VERSION_MAJOR > 0
356 "format", G_TYPE_STRING, "WMV1",
357#else
358 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('W', 'M', 'V', '1'),
359#endif
360 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
361 media_type->SamplesPerSecond.Denominator, NULL);
362 break;
363 case TSMF_SUB_TYPE_WMV2:
364 mdecoder->gst_caps = gst_caps_new_simple(
365 "video/x-wmv", "width", G_TYPE_INT, media_type->Width, "height", G_TYPE_INT,
366 media_type->Height, "wmvversion", G_TYPE_INT, 2,
367#if GST_VERSION_MAJOR > 0
368 "format", G_TYPE_STRING, "WMV2",
369#else
370 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('W', 'M', 'V', '2'),
371#endif
372 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
373 media_type->SamplesPerSecond.Denominator, "pixel-aspect-ratio", GST_TYPE_FRACTION,
374 1, 1, NULL);
375 break;
376 case TSMF_SUB_TYPE_WMV3:
377 mdecoder->gst_caps = gst_caps_new_simple(
378 "video/x-wmv", "bitrate", G_TYPE_UINT, media_type->BitRate, "width", G_TYPE_INT,
379 media_type->Width, "height", G_TYPE_INT, media_type->Height, "wmvversion",
380 G_TYPE_INT, 3,
381#if GST_VERSION_MAJOR > 0
382 "format", G_TYPE_STRING, "WMV3",
383#else
384 "format", GST_TYPE_FOURCC, GST_MAKE_FOURCC('W', 'M', 'V', '3'),
385#endif
386 "framerate", GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
387 media_type->SamplesPerSecond.Denominator, "pixel-aspect-ratio", GST_TYPE_FRACTION,
388 1, 1, NULL);
389 break;
390 case TSMF_SUB_TYPE_AVC1:
391 case TSMF_SUB_TYPE_H264:
392 mdecoder->gst_caps = gst_caps_new_simple(
393 "video/x-h264", "width", G_TYPE_INT, media_type->Width, "height", G_TYPE_INT,
394 media_type->Height, "framerate", GST_TYPE_FRACTION,
395 media_type->SamplesPerSecond.Numerator, media_type->SamplesPerSecond.Denominator,
396 "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, "stream-format", G_TYPE_STRING,
397 "byte-stream", "alignment", G_TYPE_STRING, "nal", NULL);
398 break;
399 case TSMF_SUB_TYPE_AC3:
400 mdecoder->gst_caps = gst_caps_new_simple(
401 "audio/x-ac3", "rate", G_TYPE_INT, media_type->SamplesPerSecond.Numerator,
402 "channels", G_TYPE_INT, media_type->Channels, NULL);
403 break;
404 case TSMF_SUB_TYPE_AAC:
405
406 /* For AAC the pFormat is a HEAACWAVEINFO struct, and the codec data
407 is at the end of it. See
408 http://msdn.microsoft.com/en-us/library/dd757806.aspx */
409 if (media_type->ExtraData)
410 {
411 if (media_type->ExtraDataSize < 12)
412 return FALSE;
413 media_type->ExtraData += 12;
414 media_type->ExtraDataSize -= 12;
415 }
416
417 mdecoder->gst_caps = gst_caps_new_simple(
418 "audio/mpeg", "rate", G_TYPE_INT, media_type->SamplesPerSecond.Numerator,
419 "channels", G_TYPE_INT, media_type->Channels, "mpegversion", G_TYPE_INT, 4,
420 "framed", G_TYPE_BOOLEAN, TRUE, "stream-format", G_TYPE_STRING, "raw", NULL);
421 break;
422 case TSMF_SUB_TYPE_MP1A:
423 mdecoder->gst_caps =
424 gst_caps_new_simple("audio/mpeg", "mpegversion", G_TYPE_INT, 1, "channels",
425 G_TYPE_INT, media_type->Channels, NULL);
426 break;
427 case TSMF_SUB_TYPE_MP1V:
428 mdecoder->gst_caps =
429 gst_caps_new_simple("video/mpeg", "mpegversion", G_TYPE_INT, 1, "width", G_TYPE_INT,
430 media_type->Width, "height", G_TYPE_INT, media_type->Height,
431 "systemstream", G_TYPE_BOOLEAN, FALSE, NULL);
432 break;
433 case TSMF_SUB_TYPE_YUY2:
434#if GST_VERSION_MAJOR > 0
435 mdecoder->gst_caps = gst_caps_new_simple(
436 "video/x-raw", "format", G_TYPE_STRING, "YUY2", "width", G_TYPE_INT,
437 media_type->Width, "height", G_TYPE_INT, media_type->Height, NULL);
438#else
439 mdecoder->gst_caps = gst_caps_new_simple(
440 "video/x-raw-yuv", "format", G_TYPE_STRING, "YUY2", "width", G_TYPE_INT,
441 media_type->Width, "height", G_TYPE_INT, media_type->Height, "framerate",
442 GST_TYPE_FRACTION, media_type->SamplesPerSecond.Numerator,
443 media_type->SamplesPerSecond.Denominator, NULL);
444#endif
445 break;
446 case TSMF_SUB_TYPE_MP2V:
447 mdecoder->gst_caps = gst_caps_new_simple("video/mpeg", "mpegversion", G_TYPE_INT, 2,
448 "systemstream", G_TYPE_BOOLEAN, FALSE, NULL);
449 break;
450 case TSMF_SUB_TYPE_MP2A:
451 mdecoder->gst_caps =
452 gst_caps_new_simple("audio/mpeg", "mpegversion", G_TYPE_INT, 1, "rate", G_TYPE_INT,
453 media_type->SamplesPerSecond.Numerator, "channels", G_TYPE_INT,
454 media_type->Channels, NULL);
455 break;
456 case TSMF_SUB_TYPE_FLAC:
457 mdecoder->gst_caps = gst_caps_new_simple("audio/x-flac", "", NULL);
458 break;
459 default:
460 WLog_ERR(TAG, "unknown format:(%d).", media_type->SubType);
461 return FALSE;
462 }
463
464 if (media_type->ExtraDataSize > 0)
465 {
466 GstBuffer* buffer;
467 DEBUG_TSMF("Extra data available (%" PRIu32 ")", media_type->ExtraDataSize);
468 buffer = tsmf_get_buffer_from_data(media_type->ExtraData, media_type->ExtraDataSize);
469
470 if (!buffer)
471 {
472 WLog_ERR(TAG, "could not allocate GstBuffer!");
473 return FALSE;
474 }
475
476 gst_caps_set_simple(mdecoder->gst_caps, "codec_data", GST_TYPE_BUFFER, buffer, NULL);
477 }
478
479 DEBUG_TSMF("%p format '%s'", (void*)mdecoder, gst_caps_to_string(mdecoder->gst_caps));
480 tsmf_platform_set_format(mdecoder);
481
482 /* Create the pipeline... */
483 if (!tsmf_gstreamer_pipeline_build(mdecoder))
484 return FALSE;
485
486 return TRUE;
487}
488
489void tsmf_gstreamer_clean_up(TSMFGstreamerDecoder* mdecoder)
490{
491 if (!mdecoder || !mdecoder->pipe)
492 return;
493
494 if (mdecoder->pipe && GST_OBJECT_REFCOUNT_VALUE(mdecoder->pipe) > 0)
495 {
496 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_NULL);
497 gst_object_unref(mdecoder->pipe);
498 }
499
500 mdecoder->ready = FALSE;
501 mdecoder->paused = FALSE;
502
503 mdecoder->pipe = NULL;
504 mdecoder->src = NULL;
505 mdecoder->queue = NULL;
506}
507
508BOOL tsmf_gstreamer_pipeline_build(TSMFGstreamerDecoder* mdecoder)
509{
510#if GST_VERSION_MAJOR > 0
511 const char* video =
512 "appsrc name=videosource ! queue2 name=videoqueue ! decodebin name=videodecoder !";
513 const char* audio =
514 "appsrc name=audiosource ! queue2 name=audioqueue ! decodebin name=audiodecoder ! "
515 "audioconvert ! audiorate ! audioresample ! volume name=audiovolume !";
516#else
517 const char* video =
518 "appsrc name=videosource ! queue2 name=videoqueue ! decodebin2 name=videodecoder !";
519 const char* audio =
520 "appsrc name=audiosource ! queue2 name=audioqueue ! decodebin2 name=audiodecoder ! "
521 "audioconvert ! audiorate ! audioresample ! volume name=audiovolume !";
522#endif
523 char pipeline[1024];
524
525 if (!mdecoder)
526 return FALSE;
527
528 /* TODO: Construction of the pipeline from a string allows easy overwrite with arguments.
529 * The only fixed elements necessary are appsrc and the volume element for audio streams.
530 * The rest could easily be provided in gstreamer pipeline notation from command line. */
531 if (mdecoder->media_type == TSMF_MAJOR_TYPE_VIDEO)
532 sprintf_s(pipeline, sizeof(pipeline), "%s %s name=videosink", video,
533 tsmf_platform_get_video_sink());
534 else
535 sprintf_s(pipeline, sizeof(pipeline), "%s %s name=audiosink", audio,
536 tsmf_platform_get_audio_sink());
537
538 DEBUG_TSMF("pipeline=%s", pipeline);
539 mdecoder->pipe = gst_parse_launch(pipeline, NULL);
540
541 if (!mdecoder->pipe)
542 {
543 WLog_ERR(TAG, "Failed to create new pipe");
544 return FALSE;
545 }
546
547 if (mdecoder->media_type == TSMF_MAJOR_TYPE_VIDEO)
548 mdecoder->src = gst_bin_get_by_name(GST_BIN(mdecoder->pipe), "videosource");
549 else
550 mdecoder->src = gst_bin_get_by_name(GST_BIN(mdecoder->pipe), "audiosource");
551
552 if (!mdecoder->src)
553 {
554 WLog_ERR(TAG, "Failed to get appsrc");
555 return FALSE;
556 }
557
558 if (mdecoder->media_type == TSMF_MAJOR_TYPE_VIDEO)
559 mdecoder->queue = gst_bin_get_by_name(GST_BIN(mdecoder->pipe), "videoqueue");
560 else
561 mdecoder->queue = gst_bin_get_by_name(GST_BIN(mdecoder->pipe), "audioqueue");
562
563 if (!mdecoder->queue)
564 {
565 WLog_ERR(TAG, "Failed to get queue");
566 return FALSE;
567 }
568
569 if (mdecoder->media_type == TSMF_MAJOR_TYPE_VIDEO)
570 mdecoder->outsink = gst_bin_get_by_name(GST_BIN(mdecoder->pipe), "videosink");
571 else
572 mdecoder->outsink = gst_bin_get_by_name(GST_BIN(mdecoder->pipe), "audiosink");
573
574 if (!mdecoder->outsink)
575 {
576 WLog_ERR(TAG, "Failed to get sink");
577 return FALSE;
578 }
579
580 g_signal_connect(mdecoder->outsink, "child-added", G_CALLBACK(cb_child_added), mdecoder);
581
582 if (mdecoder->media_type == TSMF_MAJOR_TYPE_AUDIO)
583 {
584 mdecoder->volume = gst_bin_get_by_name(GST_BIN(mdecoder->pipe), "audiovolume");
585
586 if (!mdecoder->volume)
587 {
588 WLog_ERR(TAG, "Failed to get volume");
589 return FALSE;
590 }
591
592 tsmf_gstreamer_change_volume((ITSMFDecoder*)mdecoder, mdecoder->gstVolume * ((double)10000),
593 mdecoder->gstMuted);
594 }
595
596 tsmf_platform_register_handler(mdecoder);
597 /* AppSrc settings */
598 GstAppSrcCallbacks callbacks = {
599 tsmf_gstreamer_need_data, tsmf_gstreamer_enough_data, tsmf_gstreamer_seek_data, { NULL }
600 };
601 g_object_set(mdecoder->src, "format", GST_FORMAT_TIME, NULL);
602 g_object_set(mdecoder->src, "is-live", FALSE, NULL);
603 g_object_set(mdecoder->src, "block", FALSE, NULL);
604 g_object_set(mdecoder->src, "blocksize", 1024, NULL);
605 gst_app_src_set_caps((GstAppSrc*)mdecoder->src, mdecoder->gst_caps);
606 gst_app_src_set_callbacks((GstAppSrc*)mdecoder->src, &callbacks, mdecoder, NULL);
607 gst_app_src_set_stream_type((GstAppSrc*)mdecoder->src, GST_APP_STREAM_TYPE_SEEKABLE);
608 gst_app_src_set_latency((GstAppSrc*)mdecoder->src, 0, -1);
609 gst_app_src_set_max_bytes((GstAppSrc*)mdecoder->src, (guint64)0); // unlimited
610 g_object_set(G_OBJECT(mdecoder->queue), "use-buffering", FALSE, NULL);
611 g_object_set(G_OBJECT(mdecoder->queue), "use-rate-estimate", FALSE, NULL);
612 g_object_set(G_OBJECT(mdecoder->queue), "max-size-buffers", 0, NULL);
613 g_object_set(G_OBJECT(mdecoder->queue), "max-size-bytes", 0, NULL);
614 g_object_set(G_OBJECT(mdecoder->queue), "max-size-time", (guint64)0, NULL);
615
616 /* Only set these properties if not an autosink, otherwise we will set properties when real
617 * sinks are added */
618 if (!g_strcmp0(G_OBJECT_TYPE_NAME(mdecoder->outsink), "GstAutoVideoSink") &&
619 !g_strcmp0(G_OBJECT_TYPE_NAME(mdecoder->outsink), "GstAutoAudioSink"))
620 {
621 if (mdecoder->media_type == TSMF_MAJOR_TYPE_VIDEO)
622 {
623 gst_base_sink_set_max_lateness((GstBaseSink*)mdecoder->outsink,
624 10000000); /* nanoseconds */
625 }
626 else
627 {
628 gst_base_sink_set_max_lateness((GstBaseSink*)mdecoder->outsink,
629 10000000); /* nanoseconds */
630 g_object_set(G_OBJECT(mdecoder->outsink), "buffer-time", (gint64)20000,
631 NULL); /* microseconds */
632 g_object_set(G_OBJECT(mdecoder->outsink), "drift-tolerance", (gint64)20000,
633 NULL); /* microseconds */
634 g_object_set(G_OBJECT(mdecoder->outsink), "latency-time", (gint64)10000,
635 NULL); /* microseconds */
636 g_object_set(G_OBJECT(mdecoder->outsink), "slave-method", 1, NULL);
637 }
638 g_object_set(G_OBJECT(mdecoder->outsink), "sync", TRUE,
639 NULL); /* synchronize on the clock */
640 g_object_set(G_OBJECT(mdecoder->outsink), "async", TRUE, NULL); /* no async state changes */
641 }
642
643 tsmf_window_create(mdecoder);
644 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_READY);
645 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_PLAYING);
646 mdecoder->pipeline_start_time_valid = 0;
647 mdecoder->shutdown = 0;
648 mdecoder->paused = FALSE;
649
650 GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(mdecoder->pipe), GST_DEBUG_GRAPH_SHOW_ALL,
651 get_type(mdecoder));
652
653 return TRUE;
654}
655
656static BOOL tsmf_gstreamer_decodeEx(ITSMFDecoder* decoder, const BYTE* data, UINT32 data_size,
657 UINT32 extensions, UINT64 start_time, UINT64 end_time,
658 UINT64 duration)
659{
660 GstBuffer* gst_buf;
661 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
662 UINT64 sample_time = tsmf_gstreamer_timestamp_ms_to_gst(start_time);
663 BOOL useTimestamps = TRUE;
664
665 if (!mdecoder)
666 {
667 WLog_ERR(TAG, "Decoder not initialized!");
668 return FALSE;
669 }
670
671 /*
672 * This function is always called from a stream-specific thread.
673 * It should be alright to block here if necessary.
674 * We don't expect to block here often, since the pipeline should
675 * have more than enough buffering.
676 */
677 DEBUG_TSMF(
678 "%s. Start:(%" PRIu64 ") End:(%" PRIu64 ") Duration:(%" PRIu64 ") Last Start:(%" PRIu64 ")",
679 get_type(mdecoder), start_time, end_time, duration, mdecoder->last_sample_start_time);
680
681 if (mdecoder->shutdown)
682 {
683 WLog_ERR(TAG, "decodeEx called on shutdown decoder");
684 return TRUE;
685 }
686
687 if (mdecoder->gst_caps == NULL)
688 {
689 WLog_ERR(TAG, "tsmf_gstreamer_set_format not called or invalid format.");
690 return FALSE;
691 }
692
693 if (!mdecoder->pipe)
694 tsmf_gstreamer_pipeline_build(mdecoder);
695
696 if (!mdecoder->src)
697 {
698 WLog_ERR(
699 TAG,
700 "failed to construct pipeline correctly. Unable to push buffer to source element.");
701 return FALSE;
702 }
703
704 gst_buf = tsmf_get_buffer_from_data(data, data_size);
705
706 if (gst_buf == NULL)
707 {
708 WLog_ERR(TAG, "tsmf_get_buffer_from_data(%p, %" PRIu32 ") failed.", (void*)data, data_size);
709 return FALSE;
710 }
711
712 /* Relative timestamping will sometimes be set to 0
713 * so we ignore these timestamps just to be safe(bit 8)
714 */
715 if (extensions & 0x00000080)
716 {
717 DEBUG_TSMF("Ignoring the timestamps - relative - bit 8");
718 useTimestamps = FALSE;
719 }
720
721 /* If no timestamps exist then we don't want to look at the timestamp values (bit 7) */
722 if (extensions & 0x00000040)
723 {
724 DEBUG_TSMF("Ignoring the timestamps - none - bit 7");
725 useTimestamps = FALSE;
726 }
727
728 /* If performing a seek */
729 if (mdecoder->seeking)
730 {
731 mdecoder->seeking = FALSE;
732 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_PAUSED);
733 mdecoder->pipeline_start_time_valid = 0;
734 }
735
736 if (mdecoder->pipeline_start_time_valid)
737 {
738 DEBUG_TSMF("%s start time %" PRIu64 "", get_type(mdecoder), start_time);
739
740 /* Adjusted the condition for a seek to be based on start time only
741 * WMV1 and WMV2 files in particular have bad end time and duration values
742 * there seems to be no real side effects of just using the start time instead
743 */
744 UINT64 minTime = mdecoder->last_sample_start_time - (UINT64)SEEK_TOLERANCE;
745 UINT64 maxTime = mdecoder->last_sample_start_time + (UINT64)SEEK_TOLERANCE;
746
747 /* Make sure the minTime stops at 0 , should we be at the beginning of the stream */
748 if (mdecoder->last_sample_start_time < (UINT64)SEEK_TOLERANCE)
749 minTime = 0;
750
751 /* If the start_time is valid and different from the previous start time by more than the
752 * seek tolerance, then we have a seek condition */
753 if (((start_time > maxTime) || (start_time < minTime)) && useTimestamps)
754 {
755 DEBUG_TSMF("tsmf_gstreamer_decodeEx: start_time=[%" PRIu64
756 "] > last_sample_start_time=[%" PRIu64 "] OR ",
757 start_time, mdecoder->last_sample_start_time);
758 DEBUG_TSMF("tsmf_gstreamer_decodeEx: start_time=[%" PRIu64
759 "] < last_sample_start_time=[%" PRIu64 "] with",
760 start_time, mdecoder->last_sample_start_time);
761 DEBUG_TSMF(
762 "tsmf_gstreamer_decodeEX: a tolerance of more than [%lu] from the last sample",
763 SEEK_TOLERANCE);
764 DEBUG_TSMF("tsmf_gstreamer_decodeEX: minTime=[%" PRIu64 "] maxTime=[%" PRIu64 "]",
765 minTime, maxTime);
766
767 mdecoder->seeking = TRUE;
768
769 /* since we can't make the gstreamer pipeline jump to the new start time after a seek -
770 * we just maintain an offset between realtime and gstreamer time
771 */
772 mdecoder->seek_offset = start_time;
773 }
774 }
775 else
776 {
777 DEBUG_TSMF("%s start time %" PRIu64 "", get_type(mdecoder), start_time);
778 /* Always set base/start time to 0. Will use seek offset to translate real buffer times
779 * back to 0. This allows the video to be started from anywhere and the ability to handle
780 * seeks without rebuilding the pipeline, etc. since that is costly
781 */
782 gst_element_set_base_time(mdecoder->pipe, tsmf_gstreamer_timestamp_ms_to_gst(0));
783 gst_element_set_start_time(mdecoder->pipe, tsmf_gstreamer_timestamp_ms_to_gst(0));
784 mdecoder->pipeline_start_time_valid = 1;
785
786 /* Set the seek offset if buffer has valid timestamps. */
787 if (useTimestamps)
788 mdecoder->seek_offset = start_time;
789
790 if (!gst_element_seek(mdecoder->pipe, 1.0, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH,
791 GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE))
792 {
793 WLog_ERR(TAG, "seek failed");
794 }
795 }
796
797#if GST_VERSION_MAJOR > 0
798 if (useTimestamps)
799 GST_BUFFER_PTS(gst_buf) =
800 sample_time - tsmf_gstreamer_timestamp_ms_to_gst(mdecoder->seek_offset);
801 else
802 GST_BUFFER_PTS(gst_buf) = GST_CLOCK_TIME_NONE;
803#else
804 if (useTimestamps)
805 GST_BUFFER_TIMESTAMP(gst_buf) =
806 sample_time - tsmf_gstreamer_timestamp_ms_to_gst(mdecoder->seek_offset);
807 else
808 GST_BUFFER_TIMESTAMP(gst_buf) = GST_CLOCK_TIME_NONE;
809#endif
810 GST_BUFFER_DURATION(gst_buf) = GST_CLOCK_TIME_NONE;
811 GST_BUFFER_OFFSET(gst_buf) = GST_BUFFER_OFFSET_NONE;
812#if GST_VERSION_MAJOR > 0
813#else
814 gst_buffer_set_caps(gst_buf, mdecoder->gst_caps);
815#endif
816 gst_app_src_push_buffer(GST_APP_SRC(mdecoder->src), gst_buf);
817
818 /* Should only update the last timestamps if the current ones are valid */
819 if (useTimestamps)
820 {
821 mdecoder->last_sample_start_time = start_time;
822 mdecoder->last_sample_end_time = end_time;
823 }
824
825 if (mdecoder->pipe && (GST_STATE(mdecoder->pipe) != GST_STATE_PLAYING))
826 {
827 DEBUG_TSMF("%s: state=%s", get_type(mdecoder),
828 gst_element_state_get_name(GST_STATE(mdecoder->pipe)));
829
830 DEBUG_TSMF("%s Paused: %" PRIi32 " Shutdown: %i Ready: %" PRIi32 "", get_type(mdecoder),
831 mdecoder->paused, mdecoder->shutdown, mdecoder->ready);
832 if (!mdecoder->paused && !mdecoder->shutdown && mdecoder->ready)
833 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_PLAYING);
834 }
835
836 return TRUE;
837}
838
839static BOOL tsmf_gstreamer_control(ITSMFDecoder* decoder, ITSMFControlMsg control_msg, UINT32* arg)
840{
841 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
842
843 if (!mdecoder)
844 {
845 WLog_ERR(TAG, "Control called with no decoder!");
846 return TRUE;
847 }
848
849 if (control_msg == Control_Pause)
850 {
851 DEBUG_TSMF("Control_Pause %s", get_type(mdecoder));
852
853 if (mdecoder->paused)
854 {
855 WLog_ERR(TAG, "%s: Ignoring Control_Pause, already received!", get_type(mdecoder));
856 return TRUE;
857 }
858
859 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_PAUSED);
860 mdecoder->shutdown = 0;
861 mdecoder->paused = TRUE;
862 }
863 else if (control_msg == Control_Resume)
864 {
865 DEBUG_TSMF("Control_Resume %s", get_type(mdecoder));
866
867 if (!mdecoder->paused && !mdecoder->shutdown)
868 {
869 WLog_ERR(TAG, "%s: Ignoring Control_Resume, already received!", get_type(mdecoder));
870 return TRUE;
871 }
872
873 mdecoder->shutdown = 0;
874 mdecoder->paused = FALSE;
875 }
876 else if (control_msg == Control_Stop)
877 {
878 DEBUG_TSMF("Control_Stop %s", get_type(mdecoder));
879
880 if (mdecoder->shutdown)
881 {
882 WLog_ERR(TAG, "%s: Ignoring Control_Stop, already received!", get_type(mdecoder));
883 return TRUE;
884 }
885
886 /* Reset stamps, flush buffers, etc */
887 if (mdecoder->pipe)
888 {
889 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_NULL);
890 tsmf_window_destroy(mdecoder);
891 tsmf_gstreamer_clean_up(mdecoder);
892 }
893 mdecoder->seek_offset = 0;
894 mdecoder->pipeline_start_time_valid = 0;
895 mdecoder->shutdown = 1;
896 }
897 else if (control_msg == Control_Restart)
898 {
899 DEBUG_TSMF("Control_Restart %s", get_type(mdecoder));
900 mdecoder->shutdown = 0;
901 mdecoder->paused = FALSE;
902
903 if (mdecoder->pipeline_start_time_valid)
904 tsmf_gstreamer_pipeline_set_state(mdecoder, GST_STATE_PLAYING);
905 }
906 else
907 WLog_ERR(TAG, "Unknown control message %08x", control_msg);
908
909 return TRUE;
910}
911
912static BOOL tsmf_gstreamer_buffer_level(ITSMFDecoder* decoder)
913{
914 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
915 DEBUG_TSMF("");
916
917 if (!mdecoder)
918 return FALSE;
919
920 guint clbuff = 0;
921
922 if (G_IS_OBJECT(mdecoder->queue))
923 g_object_get(mdecoder->queue, "current-level-buffers", &clbuff, NULL);
924
925 DEBUG_TSMF("%s buffer level %u", get_type(mdecoder), clbuff);
926 return clbuff;
927}
928
929static void tsmf_gstreamer_free(ITSMFDecoder* decoder)
930{
931 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
932 DEBUG_TSMF("%s", get_type(mdecoder));
933
934 if (mdecoder)
935 {
936 tsmf_window_destroy(mdecoder);
937 tsmf_gstreamer_clean_up(mdecoder);
938
939 if (mdecoder->gst_caps)
940 gst_caps_unref(mdecoder->gst_caps);
941
942 tsmf_platform_free(mdecoder);
943 ZeroMemory(mdecoder, sizeof(TSMFGstreamerDecoder));
944 free(mdecoder);
945 mdecoder = NULL;
946 }
947}
948
949static UINT64 tsmf_gstreamer_get_running_time(ITSMFDecoder* decoder)
950{
951 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
952
953 if (!mdecoder)
954 return 0;
955
956 if (!mdecoder->outsink)
957 return mdecoder->last_sample_start_time;
958
959 if (!mdecoder->pipe)
960 return 0;
961
962 GstFormat fmt = GST_FORMAT_TIME;
963 gint64 pos = 0;
964#if GST_VERSION_MAJOR > 0
965 gst_element_query_position(mdecoder->pipe, fmt, &pos);
966#else
967 gst_element_query_position(mdecoder->pipe, &fmt, &pos);
968#endif
969 return (UINT64)(pos / 100 + mdecoder->seek_offset);
970}
971
972static BOOL tsmf_gstreamer_update_rendering_area(ITSMFDecoder* decoder, int newX, int newY,
973 int newWidth, int newHeight, int numRectangles,
974 RDP_RECT* rectangles)
975{
976 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
977 DEBUG_TSMF("x=%d, y=%d, w=%d, h=%d, rect=%d", newX, newY, newWidth, newHeight, numRectangles);
978
979 if (mdecoder->media_type == TSMF_MAJOR_TYPE_VIDEO)
980 {
981 return tsmf_window_resize(mdecoder, newX, newY, newWidth, newHeight, numRectangles,
982 rectangles) == 0;
983 }
984
985 return TRUE;
986}
987
988static BOOL tsmf_gstreamer_ack(ITSMFDecoder* decoder, BOOL (*cb)(void*, BOOL), void* stream)
989{
990 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
991 DEBUG_TSMF("");
992 mdecoder->ack_cb = NULL;
993 mdecoder->stream = stream;
994 return TRUE;
995}
996
997static BOOL tsmf_gstreamer_sync(ITSMFDecoder* decoder, void (*cb)(void*), void* stream)
998{
999 TSMFGstreamerDecoder* mdecoder = (TSMFGstreamerDecoder*)decoder;
1000 DEBUG_TSMF("");
1001 mdecoder->sync_cb = NULL;
1002 mdecoder->stream = stream;
1003 return TRUE;
1004}
1005
1006FREERDP_ENTRY_POINT(UINT VCAPITYPE gstreamer_freerdp_tsmf_client_decoder_subsystem_entry(void* ptr))
1007{
1008 ITSMFDecoder** sptr = (ITSMFDecoder**)ptr;
1009 WINPR_ASSERT(sptr);
1010 *sptr = NULL;
1011
1012#if GST_CHECK_VERSION(0, 10, 31)
1013 if (!gst_is_initialized())
1014 {
1015 gst_init(NULL, NULL);
1016 }
1017#else
1018 gst_init(NULL, NULL);
1019#endif
1020
1021 TSMFGstreamerDecoder* decoder;
1022 decoder = calloc(1, sizeof(TSMFGstreamerDecoder));
1023
1024 if (!decoder)
1025 return ERROR_OUTOFMEMORY;
1026
1027 decoder->iface.SetFormat = tsmf_gstreamer_set_format;
1028 decoder->iface.Decode = NULL;
1029 decoder->iface.GetDecodedData = NULL;
1030 decoder->iface.GetDecodedFormat = NULL;
1031 decoder->iface.GetDecodedDimension = NULL;
1032 decoder->iface.GetRunningTime = tsmf_gstreamer_get_running_time;
1033 decoder->iface.UpdateRenderingArea = tsmf_gstreamer_update_rendering_area;
1034 decoder->iface.Free = tsmf_gstreamer_free;
1035 decoder->iface.Control = tsmf_gstreamer_control;
1036 decoder->iface.DecodeEx = tsmf_gstreamer_decodeEx;
1037 decoder->iface.ChangeVolume = tsmf_gstreamer_change_volume;
1038 decoder->iface.BufferLevel = tsmf_gstreamer_buffer_level;
1039 decoder->iface.SetAckFunc = tsmf_gstreamer_ack;
1040 decoder->iface.SetSyncFunc = tsmf_gstreamer_sync;
1041 decoder->paused = FALSE;
1042 decoder->gstVolume = 0.5;
1043 decoder->gstMuted = FALSE;
1044 decoder->state = GST_STATE_VOID_PENDING; /* No real state yet */
1045 decoder->last_sample_start_time = 0;
1046 decoder->last_sample_end_time = 0;
1047 decoder->seek_offset = 0;
1048 decoder->seeking = FALSE;
1049
1050 if (tsmf_platform_create(decoder) < 0)
1051 {
1052 free(decoder);
1053 return ERROR_INTERNAL_ERROR;
1054 }
1055
1056 *sptr = &decoder->iface;
1057 return CHANNEL_RC_OK;
1058}
RDP_RECT
Definition types.h:101
TS_AM_MEDIA_TYPE
Definition tsmf_types.h:41