29#include <winpr/assert.h>
30#include <winpr/cast.h>
31#include <winpr/cmdline.h>
32#include <winpr/file.h>
33#include <winpr/string.h>
34#include <winpr/synch.h>
35#include <winpr/thread.h>
38#include <gst/app/gstappsink.h>
39#include <gst/video/video.h>
43#define TAG CHANNELS_TAG("rdpecam-gst.client")
46#define CAM_GST_DEVICE_ID "RDCamera_Device_0"
48#define CAM_GST_SINK_NAME "freerdpcamsink"
50#define CAM_GST_FRAMERATE_NUMERATOR_DEFAULT 30
51#define CAM_GST_FRAMERATE_DENOMINATOR_DEFAULT 1
54#define CAM_GST_START_TIMEOUT (10 * GST_SECOND)
57#define CAM_GST_PULL_TIMEOUT (100 * GST_MSECOND)
71 UINT32 frameRateNumerator;
72 UINT32 frameRateDenominator;
83 ICamHalSampleCapturedCallback sampleCallback;
90static BOOL cam_gst_log_errors(CamGstHal* hal)
95 GstBus* bus = gst_element_get_bus(hal->pipeline);
99 GstMessage* msg =
nullptr;
100 while ((msg = gst_bus_pop_filtered(bus, GST_MESSAGE_ERROR)) !=
nullptr)
102 GError* error =
nullptr;
103 gchar* debug =
nullptr;
104 gst_message_parse_error(msg, &error, &debug);
105 WLog_ERR(TAG,
"Pipeline error from %s: %s (%s)", GST_OBJECT_NAME(GST_MESSAGE_SRC(msg)),
106 error ? error->message :
"unknown", debug ? debug :
"no details");
107 g_clear_error(&error);
109 gst_message_unref(msg);
112 gst_object_unref(bus);
116static void cam_gst_pipeline_destroy(CamGstHal* hal)
122 gst_object_unref(hal->sink);
127 (void)gst_element_set_state(hal->pipeline, GST_STATE_NULL);
128 gst_object_unref(hal->pipeline);
129 hal->pipeline =
nullptr;
136static BOOL cam_gst_pipeline_create(CamGstHal* hal)
139 WINPR_ASSERT(!hal->pipeline);
141 GError* error =
nullptr;
143 gst_parse_launch_full(hal->description,
nullptr, GST_PARSE_FLAG_FATAL_ERRORS, &error);
144 if (!hal->pipeline || error)
146 WLog_ERR(TAG,
"Invalid pipeline \"%s\": %s", hal->description,
147 error ? error->message :
"unknown error");
148 g_clear_error(&error);
149 cam_gst_pipeline_destroy(hal);
153 GstElement* sink = GST_IS_BIN(hal->pipeline)
154 ? gst_bin_get_by_name(GST_BIN(hal->pipeline), CAM_GST_SINK_NAME)
158 WLog_ERR(TAG,
"The pipeline has no %s element", CAM_GST_SINK_NAME);
159 cam_gst_pipeline_destroy(hal);
162 hal->sink = GST_APP_SINK(sink);
168static BOOL cam_gst_set_state(CamGstHal* hal, GstState state, GstStateChangeReturn* pResult)
172 GstStateChangeReturn rc = gst_element_set_state(hal->pipeline, state);
173 if (rc == GST_STATE_CHANGE_ASYNC)
174 rc = gst_element_get_state(hal->pipeline,
nullptr,
nullptr, CAM_GST_START_TIMEOUT);
181 case GST_STATE_CHANGE_SUCCESS:
182 case GST_STATE_CHANGE_NO_PREROLL:
184 case GST_STATE_CHANGE_ASYNC:
185 WLog_ERR(TAG,
"Pipeline did not reach %s in time", gst_element_state_get_name(state));
187 case GST_STATE_CHANGE_FAILURE:
189 if (!cam_gst_log_errors(hal))
190 WLog_ERR(TAG,
"Pipeline failed to go to %s", gst_element_state_get_name(state));
197static BOOL cam_gst_convert_sample(CamGstHal* hal, GstSample* sample)
200 WINPR_ASSERT(sample);
203 GstCaps* caps = gst_sample_get_caps(sample);
204 GstBuffer* buffer = gst_sample_get_buffer(sample);
205 if (!caps || !buffer || !gst_video_info_from_caps(&info, caps))
207 WLog_WARN(TAG,
"Skipping a sample without video caps");
211 if ((GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_I420) ||
212 (GST_VIDEO_INFO_WIDTH(&info) < 0) || (GST_VIDEO_INFO_HEIGHT(&info) < 0) ||
213 ((UINT32)GST_VIDEO_INFO_WIDTH(&info) < hal->width) ||
214 ((UINT32)GST_VIDEO_INFO_HEIGHT(&info) < hal->height))
217 TAG,
"Skipping a %s %dx%d frame, the announced format is I420 %" PRIu32
"x%" PRIu32,
218 gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(&info)), GST_VIDEO_INFO_WIDTH(&info),
219 GST_VIDEO_INFO_HEIGHT(&info), hal->width, hal->height);
224 if (!gst_video_frame_map(&frame, &info, buffer, GST_MAP_READ))
226 WLog_WARN(TAG,
"Unable to map a frame");
231 BYTE* dst = hal->sample;
232 for (guint plane = 0; plane < 3; plane++)
234 const size_t rowSize = (plane == 0) ? hal->width : hal->width / 2;
235 const size_t rows = (plane == 0) ? hal->height : hal->height / 2;
236 const BYTE* src = GST_VIDEO_FRAME_PLANE_DATA(&frame, plane);
237 const size_t stride = (size_t)GST_VIDEO_FRAME_PLANE_STRIDE(&frame, plane);
239 for (
size_t y = 0; y < rows; y++)
241 memcpy(dst, &src[y * stride], rowSize);
246 gst_video_frame_unmap(&frame);
252static void cam_gst_release(CamGstHal* hal)
256 if (hal->live || !cam_gst_set_state(hal, GST_STATE_PAUSED,
nullptr))
257 cam_gst_pipeline_destroy(hal);
260static DWORD WINAPI cam_gst_capture_thread(LPVOID param)
262 CamGstHal* hal = (CamGstHal*)param;
265 BOOL endLogged = FALSE;
268 EnterCriticalSection(&hal->lock);
269 const BOOL streaming = hal->streaming;
270 LeaveCriticalSection(&hal->lock);
274 GstSample* sample = gst_app_sink_try_pull_sample(hal->sink, CAM_GST_PULL_TIMEOUT);
277 if (cam_gst_log_errors(hal))
280 if (!gst_app_sink_is_eos(hal->sink))
283 if (hal->loop && !hal->live)
285 WLog_DBG(TAG,
"End of stream, playing it again");
286 if (!gst_element_seek_simple(
287 hal->pipeline, GST_FORMAT_TIME,
288 (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT), 0))
290 WLog_ERR(TAG,
"Unable to rewind the pipeline");
297 WLog_INFO(TAG,
"End of stream");
303 const BOOL converted = cam_gst_convert_sample(hal, sample);
304 gst_sample_unref(sample);
309 hal->sampleCallback(hal->dev, hal->streamIndex, hal->sample, hal->sampleSize);
310 if (error == ECAM_SAMPLE_CAPTURE_DRAINED)
312 WLog_DBG(TAG,
"No more sample requests, releasing the pipeline");
313 cam_gst_release(hal);
316 else if (error != CHANNEL_RC_OK)
317 WLog_ERR(TAG,
"Failure in sampleCallback: %" PRIu32, error);
320 EnterCriticalSection(&hal->lock);
321 hal->streaming = FALSE;
322 LeaveCriticalSection(&hal->lock);
323 return CHANNEL_RC_OK;
326static void cam_gst_join(CamGstHal* hal)
330 EnterCriticalSection(&hal->lock);
331 hal->streaming = FALSE;
332 LeaveCriticalSection(&hal->lock);
336 (void)WaitForSingleObject(hal->thread, INFINITE);
337 (void)CloseHandle(hal->thread);
338 hal->thread =
nullptr;
342static BOOL cam_gst_activate(WINPR_ATTR_UNUSED ICamHal* ihal,
343 WINPR_ATTR_UNUSED
const char* deviceId, CAM_ERROR_CODE* errorCode)
345 *errorCode = CAM_ERROR_CODE_None;
349static BOOL cam_gst_deactivate(WINPR_ATTR_UNUSED ICamHal* ihal,
350 WINPR_ATTR_UNUSED
const char* deviceId, CAM_ERROR_CODE* errorCode)
352 *errorCode = CAM_ERROR_CODE_None;
356static UINT cam_gst_enumerate(ICamHal* ihal, ICamHalEnumCallback callback,
CameraPlugin* ecam,
359 CamGstHal* hal = (CamGstHal*)ihal;
362 IFCALL(callback, ecam, hchannel, CAM_GST_DEVICE_ID, hal->name);
372static INT16 cam_gst_get_media_type_descriptions(
373 ICamHal* ihal, WINPR_ATTR_UNUSED
const char* deviceId, WINPR_ATTR_UNUSED
size_t streamIndex,
377 CamGstHal* hal = (CamGstHal*)ihal;
379 WINPR_ASSERT(nMediaTypes);
381 const size_t maxMediaTypes = *nMediaTypes;
383 if (maxMediaTypes == 0)
387 for (
size_t formatIndex = 0; formatIndex < nSupportedFormats; formatIndex++)
389 if (supportedFormats[formatIndex].inputFormat != CAM_MEDIA_FORMAT_I420)
391 if (formatIndex > INT16_MAX)
395 .Format = CAM_MEDIA_FORMAT_I420,
397 .Height = hal->height,
398 .FrameRateNumerator = hal->frameRateNumerator,
399 .FrameRateDenominator = hal->frameRateDenominator,
400 .PixelAspectRatioNumerator = 1,
401 .PixelAspectRatioDenominator = 1,
404 return (INT16)formatIndex;
407 WLog_ERR(TAG,
"I420 is not a supported input format");
411static CAM_ERROR_CODE cam_gst_stream_start(ICamHal* ihal,
CameraDevice* dev,
size_t streamIndex,
413 ICamHalSampleCapturedCallback callback)
415 CamGstHal* hal = (CamGstHal*)ihal;
418 WINPR_ASSERT(mediaType);
420 if ((mediaType->Format != CAM_MEDIA_FORMAT_I420) || (mediaType->Width != hal->width) ||
421 (mediaType->Height != hal->height))
423 WLog_ERR(TAG,
"Unexpected media type %u %" PRIu32
"x%" PRIu32, mediaType->Format,
424 mediaType->Width, mediaType->Height);
425 return CAM_ERROR_CODE_InvalidMediaType;
428 EnterCriticalSection(&hal->lock);
429 const BOOL streaming = hal->streaming;
430 LeaveCriticalSection(&hal->lock);
434 WLog_ERR(TAG,
"Streaming already in progress, device %s, streamIndex %" PRIuz,
435 dev->deviceId, streamIndex);
436 return CAM_ERROR_CODE_UnexpectedError;
443 hal->streamIndex = streamIndex;
444 hal->sampleCallback = callback;
446 if (!hal->pipeline && !cam_gst_pipeline_create(hal))
447 return CAM_ERROR_CODE_UnexpectedError;
449 if (!cam_gst_set_state(hal, GST_STATE_PLAYING,
nullptr))
451 cam_gst_pipeline_destroy(hal);
452 return CAM_ERROR_CODE_UnexpectedError;
455 EnterCriticalSection(&hal->lock);
456 hal->streaming = TRUE;
457 LeaveCriticalSection(&hal->lock);
459 hal->thread = CreateThread(
nullptr, 0, cam_gst_capture_thread, hal, 0,
nullptr);
462 WLog_ERR(TAG,
"CreateThread failure");
463 EnterCriticalSection(&hal->lock);
464 hal->streaming = FALSE;
465 LeaveCriticalSection(&hal->lock);
466 cam_gst_pipeline_destroy(hal);
467 return CAM_ERROR_CODE_OutOfMemory;
470 WLog_INFO(TAG,
"Pipeline playing as camera: %" PRIu32
"x%" PRIu32
", fps: %" PRIu32
"/%" PRIu32,
471 hal->width, hal->height, hal->frameRateNumerator, hal->frameRateDenominator);
472 return CAM_ERROR_CODE_None;
475static CAM_ERROR_CODE cam_gst_stream_stop(ICamHal* ihal, WINPR_ATTR_UNUSED
const char* deviceId,
476 WINPR_ATTR_UNUSED
size_t streamIndex)
478 CamGstHal* hal = (CamGstHal*)ihal;
484 cam_gst_pipeline_destroy(hal);
485 return CAM_ERROR_CODE_None;
488static CAM_ERROR_CODE cam_gst_free(ICamHal* ihal)
490 CamGstHal* hal = (CamGstHal*)ihal;
493 return CAM_ERROR_CODE_NotInitialized;
496 cam_gst_pipeline_destroy(hal);
497 DeleteCriticalSection(&hal->lock);
500 free(hal->description);
503 return CAM_ERROR_CODE_None;
509static BOOL cam_gst_probe(CamGstHal* hal)
513 WLog_DBG(TAG,
"Pipeline: %s", hal->description);
515 if (!cam_gst_pipeline_create(hal))
520 GstSample* sample =
nullptr;
521 GstStateChangeReturn state = GST_STATE_CHANGE_FAILURE;
522 if (!cam_gst_set_state(hal, GST_STATE_PAUSED, &state))
525 hal->live = (state == GST_STATE_CHANGE_NO_PREROLL);
528 if (!cam_gst_set_state(hal, GST_STATE_PLAYING,
nullptr))
530 sample = gst_app_sink_try_pull_sample(hal->sink, CAM_GST_START_TIMEOUT);
533 sample = gst_app_sink_try_pull_preroll(hal->sink, CAM_GST_START_TIMEOUT);
537 if (!cam_gst_log_errors(hal))
538 WLog_ERR(TAG,
"The pipeline did not produce any frame");
543 GstCaps* caps = gst_sample_get_caps(sample);
544 if (!caps || !gst_video_info_from_caps(&info, caps))
546 WLog_ERR(TAG,
"The pipeline does not produce video");
551 const int width = GST_VIDEO_INFO_WIDTH(&info);
552 const int height = GST_VIDEO_INFO_HEIGHT(&info);
553 if ((width < 2) || (height < 2))
555 WLog_ERR(TAG,
"Invalid video size %dx%d", width, height);
558 hal->width = (UINT32)width & ~1u;
559 hal->height = (UINT32)height & ~1u;
562 if ((GST_VIDEO_INFO_FPS_N(&info) > 0) && (GST_VIDEO_INFO_FPS_D(&info) > 0))
564 hal->frameRateNumerator = (UINT32)GST_VIDEO_INFO_FPS_N(&info);
565 hal->frameRateDenominator = (UINT32)GST_VIDEO_INFO_FPS_D(&info);
569 WLog_WARN(TAG,
"Variable frame rate, announcing %d fps",
570 CAM_GST_FRAMERATE_NUMERATOR_DEFAULT);
571 hal->frameRateNumerator = CAM_GST_FRAMERATE_NUMERATOR_DEFAULT;
572 hal->frameRateDenominator = CAM_GST_FRAMERATE_DENOMINATOR_DEFAULT;
575 hal->sampleSize = 3ull * hal->width * hal->height / 2;
576 hal->sample = calloc(1, hal->sampleSize);
581 "%s pipeline: %dx%d, announced as %" PRIu32
"x%" PRIu32
" at %" PRIu32
"/%" PRIu32
583 hal->live ?
"Live" :
"Non live", width, height, hal->width, hal->height,
584 hal->frameRateNumerator, hal->frameRateDenominator);
589 gst_sample_unref(sample);
592 cam_gst_pipeline_destroy(hal);
596static const char* cam_gst_basename(
const char* path)
598 const char* name = path;
599 for (
const char* cur = path; *cur; cur++)
601 if ((*cur ==
'/') || (*cur ==
'\\'))
607WINPR_ATTR_MALLOC(free, 1)
608static
char* cam_gst_read_file(const
char* path)
612 char* content =
nullptr;
613 FILE* fp = winpr_fopen(path,
"rb");
616 WLog_ERR(TAG,
"Unable to open pipeline file %s", path);
620 if (_fseeki64(fp, 0, SEEK_END) != 0)
622 const INT64 size = _ftelli64(fp);
623 if ((size <= 0) || (size > 1024ll * 1024ll) || (_fseeki64(fp, 0, SEEK_SET) != 0))
626 content = calloc(1, (
size_t)size + 1);
630 if (fread(content, 1, (
size_t)size, fp) != (
size_t)size)
638 WLog_ERR(TAG,
"Unable to read pipeline file %s", path);
644static BOOL cam_gst_parse_bool(
const char* value, BOOL* result)
647 WINPR_ASSERT(result);
649 if ((_stricmp(value,
"on") == 0) || (_stricmp(value,
"true") == 0) || (strcmp(value,
"1") == 0))
651 else if ((_stricmp(value,
"off") == 0) || (_stricmp(value,
"false") == 0) ||
652 (strcmp(value,
"0") == 0))
660static BOOL cam_gst_parse_args(CamGstHal* hal,
const ADDIN_ARGV* args)
665 {
"pipeline", COMMAND_LINE_VALUE_REQUIRED,
"<pipeline>",
nullptr,
nullptr, -1,
nullptr,
666 "pipeline producing video, in gst-launch syntax" },
667 {
"pipeline-file", COMMAND_LINE_VALUE_REQUIRED,
"<path>",
nullptr,
nullptr, -1,
nullptr,
668 "file containing the pipeline" },
669 {
"file", COMMAND_LINE_VALUE_REQUIRED,
"<path>",
nullptr,
nullptr, -1,
nullptr,
670 "media file to play" },
671 {
"name", COMMAND_LINE_VALUE_REQUIRED,
"<name>",
nullptr,
nullptr, -1,
nullptr,
673 {
"loop", COMMAND_LINE_VALUE_REQUIRED,
"<on|off>",
nullptr,
nullptr, -1,
nullptr,
674 "play the pipeline again when it ends" },
675 {
nullptr, 0,
nullptr,
nullptr,
nullptr, -1,
nullptr,
nullptr }
678 char* source =
nullptr;
679 char* defaultName =
nullptr;
684 if (args && (args->argc > 1))
686 const DWORD flags = COMMAND_LINE_SIGIL_NONE | COMMAND_LINE_SEPARATOR_COLON |
687 COMMAND_LINE_IGN_UNKNOWN_KEYWORD;
688 const int status = CommandLineParseArgumentsA(args->argc, args->argv, gst_args, flags, hal,
696 if (!(arg->Flags & COMMAND_LINE_VALUE_PRESENT))
699 CommandLineSwitchStart(arg) CommandLineSwitchCase(arg,
"pipeline")
703 source = _strdup(arg->Value);
707 CommandLineSwitchCase(arg,
"pipeline-file")
711 source = cam_gst_read_file(arg->Value);
715 CommandLineSwitchCase(arg,
"file")
721 GError* error =
nullptr;
722 gchar* uri = gst_filename_to_uri(arg->Value, &error);
725 WLog_ERR(TAG,
"Invalid file %s: %s", arg->Value,
726 error ? error->message :
"unknown error");
727 g_clear_error(&error);
733 winpr_asprintf(&source, &len,
"uridecodebin uri=\"%s\" caps=video/x-raw", uri);
739 defaultName =
nullptr;
740 winpr_asprintf(&defaultName, &len,
"Movie: %s", cam_gst_basename(arg->Value));
744 CommandLineSwitchCase(arg,
"name")
747 hal->name = _strdup(arg->Value);
751 CommandLineSwitchCase(arg,
"loop")
753 if (!cam_gst_parse_bool(arg->Value, &hal->loop))
755 WLog_ERR(TAG,
"Invalid loop value %s, expecting on or off", arg->Value);
759 CommandLineSwitchDefault(arg)
762 CommandLineSwitchEnd(arg)
763 }
while ((arg = CommandLineFindNextArgumentA(arg)) !=
nullptr);
768 WLog_ERR(TAG,
"The gst backend needs one of pipeline:<pipeline>, pipeline-file:<path> or "
773 winpr_asprintf(&hal->description, &len,
774 "%s ! videoconvert ! video/x-raw,format=I420 ! appsink name=" CAM_GST_SINK_NAME
775 " sync=true max-buffers=1 drop=true",
777 if (!hal->description)
782 hal->name = defaultName ? defaultName : _strdup(
"GStreamer camera");
783 defaultName =
nullptr;
800FREERDP_ENTRY_POINT(UINT VCAPITYPE gst_freerdp_rdpecam_client_subsystem_entry(
803 UINT ret = ERROR_INVALID_PARAMETER;
804 WINPR_ASSERT(pEntryPoints);
806 GError* error =
nullptr;
807 if (!gst_init_check(
nullptr,
nullptr, &error))
809 WLog_ERR(TAG,
"Unable to initialize GStreamer: %s",
810 error ? error->message :
"unknown error");
811 g_clear_error(&error);
812 return ERROR_INTERNAL_ERROR;
815 CamGstHal* hal = (CamGstHal*)calloc(1,
sizeof(CamGstHal));
817 return CHANNEL_RC_NO_MEMORY;
819 hal->iHal.Enumerate = cam_gst_enumerate;
820 hal->iHal.RequestDrivenCapture = TRUE;
821 hal->iHal.GetMediaTypeDescriptions = cam_gst_get_media_type_descriptions;
822 hal->iHal.Activate = cam_gst_activate;
823 hal->iHal.Deactivate = cam_gst_deactivate;
824 hal->iHal.StartStream = cam_gst_stream_start;
825 hal->iHal.StopStream = cam_gst_stream_stop;
826 hal->iHal.Free = cam_gst_free;
829 if (!InitializeCriticalSectionEx(&hal->lock, 0, 0))
832 return CHANNEL_RC_NO_MEMORY;
835 if (!cam_gst_parse_args(hal, pEntryPoints->args))
839 if (!cam_gst_probe(hal))
842 ret = pEntryPoints->pRegisterCameraHal(pEntryPoints->plugin, &hal->iHal);
843 if (ret != CHANNEL_RC_OK)
845 WLog_ERR(TAG,
"RegisterCameraHal failed with error %" PRIu32
"", ret);
852 cam_gst_free(&hal->iHal);