23#include <winpr/wtypes.h>
25#include <camera/NdkCameraDevice.h>
26#include <camera/NdkCameraManager.h>
27#include <camera/NdkCameraMetadata.h>
28#include <camera/NdkCameraCaptureSession.h>
29#include <media/NdkImage.h>
30#include <media/NdkImageReader.h>
34#define TAG CHANNELS_TAG("rdpecam-android.client")
36#define ANDROID_CAMERA_PREFIX "android-camera-"
37#define ANDROID_CAMERA_PREFIX_LEN (sizeof(ANDROID_CAMERA_PREFIX) - 1)
39#define CAM_ANDROID_FPS 30
44 ACameraManager* manager;
52 ACameraDevice* device;
53 ACameraCaptureSession* session;
54 ACaptureSessionOutputContainer* outputContainer;
55 ACaptureSessionOutput* sessionOutput;
56 ACameraOutputTarget* outputTarget;
57 ACaptureRequest* captureRequest;
58 AImageReader* imageReader;
59 ANativeWindow* window;
61 ICamHalSampleCapturedCallback sampleCallback;
68 size_t nv12BufferSize;
71static const char* cam_android_extract_camera_id(
const char* deviceId)
73 WINPR_ASSERT(deviceId);
75 if (strncmp(deviceId, ANDROID_CAMERA_PREFIX, ANDROID_CAMERA_PREFIX_LEN) != 0)
77 return deviceId + ANDROID_CAMERA_PREFIX_LEN;
81static void cam_android_close_session(CamAndroidStream* stream)
87 ACameraCaptureSession_stopRepeating(stream->session);
88 ACameraCaptureSession_close(stream->session);
89 stream->session =
nullptr;
91 if (stream->captureRequest)
93 if (stream->outputTarget)
94 ACaptureRequest_removeTarget(stream->captureRequest, stream->outputTarget);
95 ACaptureRequest_free(stream->captureRequest);
96 stream->captureRequest =
nullptr;
98 if (stream->outputTarget)
100 ACameraOutputTarget_free(stream->outputTarget);
101 stream->outputTarget =
nullptr;
103 if (stream->outputContainer)
105 ACaptureSessionOutputContainer_free(stream->outputContainer);
106 stream->outputContainer =
nullptr;
108 if (stream->sessionOutput)
110 ACaptureSessionOutput_free(stream->sessionOutput);
111 stream->sessionOutput =
nullptr;
116static void cam_android_yuv420_888_to_nv12(BYTE* dst,
int width,
int height,
const uint8_t* yData,
117 int yRowStride,
const uint8_t* uData,
int uRowStride,
118 int uPixelStride,
const uint8_t* vData,
int vRowStride,
126 const size_t ySize = (size_t)width * height;
129 if (yRowStride == width)
130 memcpy(dst, yData, ySize);
133 for (
int row = 0; row < height; row++)
134 memcpy(dst + (
size_t)row * width, yData + (
size_t)row * yRowStride, (
size_t)width);
138 BYTE* uv = dst + ySize;
139 const int uvHeight = height / 2;
140 const int uvWidth = width / 2;
141 const size_t uvRowBytes = (size_t)(uvWidth * 2);
143 if (uPixelStride == 2 && vPixelStride == 2)
149 if (uRowStride == (
int)uvRowBytes)
150 memcpy(uv, uData, (
size_t)uvHeight * uvRowBytes);
153 for (
int row = 0; row < uvHeight; row++)
154 memcpy(uv + (
size_t)row * uvRowBytes, uData + (size_t)row * uRowStride,
161 for (
int row = 0; row < uvHeight; row++)
163 const uint8_t* u = uData + (size_t)row * uRowStride;
164 const uint8_t* v = vData + (size_t)row * vRowStride;
165 BYTE* d = uv + (size_t)row * uvRowBytes;
166 for (
int col = 0; col < uvWidth; col++, u += 2, v += 2, d += 2)
177 for (
int row = 0; row < uvHeight; row++)
179 const uint8_t* uRow = uData + (size_t)row * uRowStride;
180 const uint8_t* vRow = vData + (size_t)row * vRowStride;
181 BYTE* dstRow = uv + (size_t)row * uvRowBytes;
182 for (
int col = 0; col < uvWidth; col++)
184 dstRow[col * 2] = uRow[col * uPixelStride];
185 dstRow[col * 2 + 1] = vRow[col * vPixelStride];
191static void cam_android_deliver_frame(CamAndroidStream* stream, AImage* image)
193 WINPR_ASSERT(stream);
198 if (AImage_getWidth(image, &width) != AMEDIA_OK ||
199 AImage_getHeight(image, &height) != AMEDIA_OK)
202 const size_t ySize = (size_t)(width * height);
203 const size_t nv12Size = ySize + ySize / 2;
205 if (stream->nv12BufferSize < nv12Size)
207 BYTE* tmp = (BYTE*)realloc(stream->nv12Buffer, nv12Size);
210 stream->nv12Buffer = tmp;
211 stream->nv12BufferSize = nv12Size;
216 uint8_t* yData =
nullptr;
217 uint8_t* uData =
nullptr;
218 uint8_t* vData =
nullptr;
219 if (AImage_getPlaneData(image, 0, &yData, &dataLength) != AMEDIA_OK ||
220 AImage_getPlaneData(image, 1, &uData, &dataLength) != AMEDIA_OK ||
221 AImage_getPlaneData(image, 2, &vData, &dataLength) != AMEDIA_OK)
226 int uPixelStride = 0;
228 int vPixelStride = 0;
229 AImage_getPlaneRowStride(image, 0, &yRowStride);
230 AImage_getPlaneRowStride(image, 1, &uRowStride);
231 AImage_getPlanePixelStride(image, 1, &uPixelStride);
232 AImage_getPlaneRowStride(image, 2, &vRowStride);
233 AImage_getPlanePixelStride(image, 2, &vPixelStride);
235 cam_android_yuv420_888_to_nv12(stream->nv12Buffer, width, height, yData, yRowStride, uData,
236 uRowStride, uPixelStride, vData, vRowStride, vPixelStride);
238 stream->sampleCallback(stream->dev, stream->streamIndex, stream->nv12Buffer, nv12Size);
241static void cam_android_on_image_available(
void* context, AImageReader* reader)
243 CamAndroidStream* stream = (CamAndroidStream*)context;
244 WINPR_ASSERT(stream);
245 WINPR_ASSERT(reader);
247 EnterCriticalSection(&stream->lock);
248 const BOOL streaming = stream->streaming;
249 LeaveCriticalSection(&stream->lock);
254 AImage* image =
nullptr;
255 if (AImageReader_acquireLatestImage(reader, &image) == AMEDIA_OK && image)
257 cam_android_deliver_frame(stream, image);
258 AImage_delete(image);
263static void cam_android_mark_device_error(CamAndroidStream* stream)
267 EnterCriticalSection(&stream->lock);
268 stream->deviceError = TRUE;
269 stream->streaming = FALSE;
270 LeaveCriticalSection(&stream->lock);
273static void cam_android_device_disconnected(
void* context, ACameraDevice* device)
275 WINPR_UNUSED(device);
276 WLog_WARN(TAG,
"Camera device disconnected");
277 cam_android_mark_device_error((CamAndroidStream*)context);
280static void cam_android_device_error(
void* context, ACameraDevice* device,
int error)
282 WINPR_UNUSED(device);
283 WLog_ERR(TAG,
"Camera device error %d", error);
284 cam_android_mark_device_error((CamAndroidStream*)context);
287static void cam_android_session_active(
void* context, ACameraCaptureSession* session)
289 WINPR_UNUSED(context);
290 WINPR_UNUSED(session);
291 WLog_DBG(TAG,
"Capture session active");
294static void cam_android_session_closed(
void* context, ACameraCaptureSession* session)
296 WINPR_UNUSED(context);
297 WINPR_UNUSED(session);
298 WLog_DBG(TAG,
"Capture session closed");
301static void cam_android_session_ready(
void* context, ACameraCaptureSession* session)
303 WINPR_UNUSED(context);
304 WINPR_UNUSED(session);
305 WLog_DBG(TAG,
"Capture session ready");
310static BOOL cam_android_is_backward_compatible(ACameraMetadata* characteristics)
312 WINPR_ASSERT(characteristics);
314 ACameraMetadata_const_entry caps;
315 if (ACameraMetadata_getConstEntry(characteristics, ACAMERA_REQUEST_AVAILABLE_CAPABILITIES,
316 &caps) != ACAMERA_OK)
319 for (uint32_t i = 0; i < caps.count; i++)
321 if (caps.data.u8[i] == ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE)
327static UINT cam_android_enumerate(ICamHal* ihal, ICamHalEnumCallback callback,
CameraPlugin* ecam,
330 CamAndroidHal* hal = (CamAndroidHal*)ihal;
336 if (!freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT_NV12,
337 FREERDP_VIDEO_FORMAT_YUV420P))
339 WLog_WARN(TAG,
"Camera redirection unavailable: video conversion (FFmpeg/swscale) missing");
343 ACameraIdList* idList =
nullptr;
344 if (ACameraManager_getCameraIdList(hal->manager, &idList) != ACAMERA_OK || !idList)
350 BOOL seenFacing[ACAMERA_LENS_FACING_EXTERNAL + 1] = WINPR_C_ARRAY_INIT;
352 for (
int i = 0; i < idList->numCameras; i++)
354 const char* cameraId = idList->cameraIds[i];
356 ACameraMetadata* characteristics =
nullptr;
357 if (ACameraManager_getCameraCharacteristics(hal->manager, cameraId, &characteristics) !=
362 if (!cam_android_is_backward_compatible(characteristics))
364 WLog_DBG(TAG,
"Skipping camera %s: not BACKWARD_COMPATIBLE", cameraId);
365 ACameraMetadata_free(characteristics);
369 ACameraMetadata_const_entry facingEntry;
370 const char* facingName =
"Camera";
371 uint8_t facing = ACAMERA_LENS_FACING_EXTERNAL;
372 if (ACameraMetadata_getConstEntry(characteristics, ACAMERA_LENS_FACING, &facingEntry) ==
375 facing = facingEntry.data.u8[0];
378 case ACAMERA_LENS_FACING_FRONT:
379 facingName =
"Front Camera";
381 case ACAMERA_LENS_FACING_BACK:
382 facingName =
"Back Camera";
385 facingName =
"External Camera";
391 if (facing <= ACAMERA_LENS_FACING_EXTERNAL)
393 if (seenFacing[facing])
395 WLog_DBG(TAG,
"Skipping camera %s: already have a %s", cameraId, facingName);
396 ACameraMetadata_free(characteristics);
399 seenFacing[facing] = TRUE;
403 (void)_snprintf(deviceId,
sizeof(deviceId), ANDROID_CAMERA_PREFIX
"%s", cameraId);
405 IFCALL(callback, ecam, hchannel, deviceId, facingName);
408 ACameraMetadata_free(characteristics);
411 ACameraManager_deleteCameraIdList(idList);
417static void cam_android_close_device(CamAndroidStream* stream)
419 WINPR_ASSERT(stream);
421 EnterCriticalSection(&stream->lock);
422 stream->streaming = FALSE;
423 LeaveCriticalSection(&stream->lock);
425 if (stream->imageReader)
426 AImageReader_setImageListener(stream->imageReader,
nullptr);
428 cam_android_close_session(stream);
432 ACameraDevice_close(stream->device);
433 stream->device =
nullptr;
436 if (stream->imageReader)
438 AImageReader_delete(stream->imageReader);
439 stream->imageReader =
nullptr;
440 stream->window =
nullptr;
442 free(stream->nv12Buffer);
443 stream->nv12Buffer =
nullptr;
444 stream->nv12BufferSize = 0;
445 stream->deviceError = FALSE;
450static void cam_android_close_other_devices(CamAndroidHal* hal,
const char* keepDeviceId)
453 WINPR_ASSERT(keepDeviceId);
455 ULONG_PTR* keys =
nullptr;
456 const size_t count = HashTable_GetKeys(hal->streams, &keys);
457 for (
size_t i = 0; i < count; i++)
459 const char*
id = (
const char*)keys[i];
460 if (strcmp(
id, keepDeviceId) == 0)
463 CamAndroidStream* other = (CamAndroidStream*)HashTable_GetItemValue(hal->streams,
id);
464 if (!other || !other->device)
467 WLog_DBG(TAG,
"Closing camera %s to free resources for %s",
id, keepDeviceId);
468 cam_android_close_device(other);
473static CamAndroidStream* cam_android_get_or_create_stream(CamAndroidHal* hal,
const char* deviceId,
474 CAM_ERROR_CODE* errorCode)
477 WINPR_ASSERT(deviceId);
478 WINPR_ASSERT(errorCode);
480 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
484 stream = (CamAndroidStream*)calloc(1,
sizeof(CamAndroidStream));
487 *errorCode = CAM_ERROR_CODE_OutOfMemory;
490 if (!InitializeCriticalSectionEx(&stream->lock, 0, 0))
493 *errorCode = CAM_ERROR_CODE_UnexpectedError;
496 if (!HashTable_Insert(hal->streams, deviceId, stream))
498 DeleteCriticalSection(&stream->lock);
500 *errorCode = CAM_ERROR_CODE_UnexpectedError;
508static BOOL cam_android_open_device(CamAndroidHal* hal, CamAndroidStream* stream,
509 const char* deviceId,
const char* cameraId,
510 CAM_ERROR_CODE* errorCode)
513 WINPR_ASSERT(stream);
514 WINPR_ASSERT(deviceId);
515 WINPR_ASSERT(cameraId);
516 WINPR_ASSERT(errorCode);
519 EnterCriticalSection(&stream->lock);
520 const BOOL hadError = stream->deviceError;
521 LeaveCriticalSection(&stream->lock);
522 if (stream->device && hadError)
523 cam_android_close_device(stream);
528 cam_android_close_other_devices(hal, deviceId);
530 ACameraDevice_StateCallbacks deviceCallbacks = {
532 .onDisconnected = cam_android_device_disconnected,
533 .onError = cam_android_device_error,
536 camera_status_t status =
537 ACameraManager_openCamera(hal->manager, cameraId, &deviceCallbacks, &stream->device);
538 if (status != ACAMERA_OK)
540 WLog_ERR(TAG,
"ACameraManager_openCamera failed: %d", status);
541 *errorCode = CAM_ERROR_CODE_InvalidRequest;
545 WLog_DBG(TAG,
"Opened camera %s", cameraId);
549static BOOL cam_android_activate(ICamHal* ihal,
const char* deviceId, CAM_ERROR_CODE* errorCode)
551 CamAndroidHal* hal = (CamAndroidHal*)ihal;
553 WINPR_ASSERT(deviceId);
554 WINPR_ASSERT(errorCode);
556 *errorCode = CAM_ERROR_CODE_None;
558 const char* cameraId = cam_android_extract_camera_id(deviceId);
561 *errorCode = CAM_ERROR_CODE_InvalidRequest;
566 return cam_android_get_or_create_stream(hal, deviceId, errorCode) !=
nullptr;
569static BOOL cam_android_deactivate(ICamHal* ihal,
const char* deviceId, CAM_ERROR_CODE* errorCode)
571 CamAndroidHal* hal = (CamAndroidHal*)ihal;
573 WINPR_ASSERT(deviceId);
574 WINPR_ASSERT(errorCode);
576 *errorCode = CAM_ERROR_CODE_None;
578 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
579 if (!stream || !stream->device)
582 cam_android_close_device(stream);
586static INT16 cam_android_get_media_type_descriptions(ICamHal* ihal,
const char* deviceId,
589 size_t nSupportedFormats,
593 WINPR_UNUSED(streamIndex);
594 CamAndroidHal* hal = (CamAndroidHal*)ihal;
596 WINPR_ASSERT(deviceId);
597 WINPR_ASSERT(supportedFormats || (nSupportedFormats == 0));
598 WINPR_ASSERT(mediaTypes);
599 WINPR_ASSERT(nMediaTypes);
601 size_t maxMediaTypes = *nMediaTypes;
604 const char* cameraId = cam_android_extract_camera_id(deviceId);
609 INT16 formatIndex = -1;
610 for (
size_t i = 0; i < nSupportedFormats; i++)
612 if (supportedFormats[i].inputFormat == CAM_MEDIA_FORMAT_NV12)
614 formatIndex = (INT16)i;
620 WLog_WARN(TAG,
"Server does not offer NV12 format");
624 ACameraMetadata* characteristics =
nullptr;
625 if (ACameraManager_getCameraCharacteristics(hal->manager, cameraId, &characteristics) !=
630 ACameraMetadata_const_entry entry;
631 if (ACameraMetadata_getConstEntry(
632 characteristics, ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, &entry) != ACAMERA_OK)
634 ACameraMetadata_free(characteristics);
639 for (uint32_t i = 0; i + 3 < entry.count; i += 4)
641 const int32_t fmt = entry.data.i32[i];
642 const int32_t w = entry.data.i32[i + 1];
643 const int32_t h = entry.data.i32[i + 2];
644 const int32_t isInput = entry.data.i32[i + 3];
648 if (fmt != AIMAGE_FORMAT_YUV_420_888)
651 if (w > 1920 || h > 1080)
654 mediaTypes[nTypes].Format = CAM_MEDIA_FORMAT_NV12;
655 mediaTypes[nTypes].Width = (UINT32)w;
656 mediaTypes[nTypes].Height = (UINT32)h;
657 mediaTypes[nTypes].FrameRateNumerator = CAM_ANDROID_FPS;
658 mediaTypes[nTypes].FrameRateDenominator = 1;
659 mediaTypes[nTypes].PixelAspectRatioNumerator = 1;
660 mediaTypes[nTypes].PixelAspectRatioDenominator = 1;
662 WLog_DBG(TAG,
"Camera format NV12 %dx%d @ %dfps", w, h, CAM_ANDROID_FPS);
664 if (++nTypes >= maxMediaTypes)
668 *nMediaTypes = nTypes;
669 ACameraMetadata_free(characteristics);
673 WLog_ERR(TAG,
"No supported YUV resolutions found for camera %s", cameraId);
680static CAM_ERROR_CODE cam_android_start_stream(ICamHal* ihal,
CameraDevice* dev,
size_t streamIndex,
682 ICamHalSampleCapturedCallback callback)
684 CamAndroidHal* hal = (CamAndroidHal*)ihal;
687 WINPR_ASSERT(mediaType);
688 WINPR_ASSERT(callback);
690 const char* deviceId = dev->deviceId;
692 const char* cameraId = cam_android_extract_camera_id(deviceId);
694 return CAM_ERROR_CODE_InvalidRequest;
696 CAM_ERROR_CODE errorCode = CAM_ERROR_CODE_None;
697 CamAndroidStream* stream = cam_android_get_or_create_stream(hal, deviceId, &errorCode);
701 if (!cam_android_open_device(hal, stream, deviceId, cameraId, &errorCode))
705 stream->streamIndex = streamIndex;
706 stream->sampleCallback = callback;
708 const int32_t width = (int32_t)mediaType->Width;
709 const int32_t height = (int32_t)mediaType->Height;
711 if (AImageReader_new(width, height, AIMAGE_FORMAT_YUV_420_888, 4, &stream->imageReader) !=
714 WLog_ERR(TAG,
"AImageReader_new failed");
715 return CAM_ERROR_CODE_UnexpectedError;
718 AImageReader_ImageListener listener = {
720 .onImageAvailable = cam_android_on_image_available,
722 AImageReader_setImageListener(stream->imageReader, &listener);
724 if (AImageReader_getWindow(stream->imageReader, &stream->window) != AMEDIA_OK)
726 WLog_ERR(TAG,
"AImageReader_getWindow failed");
730 ACaptureSessionOutput_create(stream->window, &stream->sessionOutput);
731 ACaptureSessionOutputContainer_create(&stream->outputContainer);
732 ACaptureSessionOutputContainer_add(stream->outputContainer, stream->sessionOutput);
733 ACameraOutputTarget_create(stream->window, &stream->outputTarget);
734 ACameraDevice_createCaptureRequest(stream->device, TEMPLATE_RECORD, &stream->captureRequest);
735 ACaptureRequest_addTarget(stream->captureRequest, stream->outputTarget);
737 ACameraCaptureSession_stateCallbacks sessionCallbacks = {
739 .onActive = cam_android_session_active,
740 .onClosed = cam_android_session_closed,
741 .onReady = cam_android_session_ready,
744 if (ACameraDevice_createCaptureSession(stream->device, stream->outputContainer,
745 &sessionCallbacks, &stream->session) != ACAMERA_OK)
747 WLog_ERR(TAG,
"ACameraDevice_createCaptureSession failed");
751 if (ACameraCaptureSession_setRepeatingRequest(stream->session,
nullptr, 1,
752 &stream->captureRequest,
nullptr) != ACAMERA_OK)
754 WLog_ERR(TAG,
"ACameraCaptureSession_setRepeatingRequest failed");
758 EnterCriticalSection(&stream->lock);
759 stream->streaming = TRUE;
760 LeaveCriticalSection(&stream->lock);
762 WLog_DBG(TAG,
"Camera stream started: %dx%d", width, height);
763 return CAM_ERROR_CODE_None;
766 cam_android_close_device(stream);
767 return CAM_ERROR_CODE_UnexpectedError;
770static CAM_ERROR_CODE cam_android_stop_stream(ICamHal* ihal,
const char* deviceId,
773 WINPR_UNUSED(streamIndex);
774 CamAndroidHal* hal = (CamAndroidHal*)ihal;
776 WINPR_ASSERT(deviceId);
778 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
780 return CAM_ERROR_CODE_None;
782 cam_android_close_device(stream);
784 WLog_DBG(TAG,
"Camera stream stopped for %s", deviceId);
785 return CAM_ERROR_CODE_None;
788static void cam_android_stream_free(
void* obj)
790 CamAndroidStream* stream = (CamAndroidStream*)obj;
794 cam_android_close_device(stream);
795 DeleteCriticalSection(&stream->lock);
799static CAM_ERROR_CODE cam_android_free(ICamHal* ihal)
801 CamAndroidHal* hal = (CamAndroidHal*)ihal;
803 return CAM_ERROR_CODE_NotInitialized;
805 HashTable_Free(hal->streams);
807 ACameraManager_delete(hal->manager);
809 return CAM_ERROR_CODE_None;
812FREERDP_ENTRY_POINT(UINT VCAPITYPE android_freerdp_rdpecam_client_subsystem_entry(
815 WINPR_ASSERT(pEntryPoints);
817 CamAndroidHal* hal = (CamAndroidHal*)calloc(1,
sizeof(CamAndroidHal));
819 return CHANNEL_RC_NO_MEMORY;
821 hal->manager = ACameraManager_create();
825 return ERROR_INTERNAL_ERROR;
828 hal->streams = HashTable_New(FALSE);
832 if (!HashTable_SetupForStringData(hal->streams, FALSE))
835 wObject* obj = HashTable_ValueObject(hal->streams);
839 hal->iHal.Enumerate = cam_android_enumerate;
840 hal->iHal.Activate = cam_android_activate;
841 hal->iHal.Deactivate = cam_android_deactivate;
842 hal->iHal.GetMediaTypeDescriptions = cam_android_get_media_type_descriptions;
843 hal->iHal.StartStream = cam_android_start_stream;
844 hal->iHal.StopStream = cam_android_stop_stream;
845 hal->iHal.Free = cam_android_free;
847 UINT ret = pEntryPoints->pRegisterCameraHal(pEntryPoints->plugin, &hal->iHal);
848 if (ret != CHANNEL_RC_OK)
850 WLog_ERR(TAG,
"RegisterCameraHal failed: %" PRIu32, ret);
857 cam_android_free(&hal->iHal);
858 return ERROR_INTERNAL_ERROR;
This struct contains function pointer to initialize/free objects.
OBJECT_FREE_FN fnObjectFree