23#include <winpr/wtypes.h>
25#include <camera/NdkCameraDevice.h>
26#include <camera/NdkCameraManager.h>
27#include <camera/NdkCameraMetadata.h>
28#include <camera/NdkCameraCaptureSession.h>
29#include <media/NdkImage.h>
30#include <media/NdkImageReader.h>
34#define TAG CHANNELS_TAG("rdpecam-android.client")
36#define ANDROID_CAMERA_PREFIX "android-camera-"
37#define ANDROID_CAMERA_PREFIX_LEN (sizeof(ANDROID_CAMERA_PREFIX) - 1)
39#define CAM_ANDROID_FPS 30
44#define CAM_ANDROID_MIN_WIDTH 320
45#define CAM_ANDROID_MIN_HEIGHT 240
50 ACameraManager* manager;
58 ACameraDevice* device;
59 ACameraCaptureSession* session;
60 ACaptureSessionOutputContainer* outputContainer;
61 ACaptureSessionOutput* sessionOutput;
62 ACameraOutputTarget* outputTarget;
63 ACaptureRequest* captureRequest;
64 AImageReader* imageReader;
65 ANativeWindow* window;
67 ICamHalSampleCapturedCallback sampleCallback;
74 size_t nv12BufferSize;
77static const char* cam_android_extract_camera_id(
const char* deviceId)
79 WINPR_ASSERT(deviceId);
81 if (strncmp(deviceId, ANDROID_CAMERA_PREFIX, ANDROID_CAMERA_PREFIX_LEN) != 0)
83 return deviceId + ANDROID_CAMERA_PREFIX_LEN;
87static void cam_android_close_session(CamAndroidStream* stream)
93 ACameraCaptureSession_stopRepeating(stream->session);
94 ACameraCaptureSession_close(stream->session);
95 stream->session =
nullptr;
97 if (stream->captureRequest)
99 if (stream->outputTarget)
100 ACaptureRequest_removeTarget(stream->captureRequest, stream->outputTarget);
101 ACaptureRequest_free(stream->captureRequest);
102 stream->captureRequest =
nullptr;
104 if (stream->outputTarget)
106 ACameraOutputTarget_free(stream->outputTarget);
107 stream->outputTarget =
nullptr;
109 if (stream->outputContainer)
111 ACaptureSessionOutputContainer_free(stream->outputContainer);
112 stream->outputContainer =
nullptr;
114 if (stream->sessionOutput)
116 ACaptureSessionOutput_free(stream->sessionOutput);
117 stream->sessionOutput =
nullptr;
122static void cam_android_yuv420_888_to_nv12(BYTE* dst,
int width,
int height,
const uint8_t* yData,
123 int yRowStride,
const uint8_t* uData,
int uRowStride,
124 int uPixelStride,
const uint8_t* vData,
int vRowStride,
132 const size_t ySize = (size_t)width * height;
135 if (yRowStride == width)
136 memcpy(dst, yData, ySize);
139 for (
int row = 0; row < height; row++)
140 memcpy(dst + (
size_t)row * width, yData + (
size_t)row * yRowStride, (
size_t)width);
144 BYTE* uv = dst + ySize;
145 const int uvHeight = height / 2;
146 const int uvWidth = width / 2;
147 const size_t uvRowBytes = (size_t)(uvWidth * 2);
149 if (uPixelStride == 2 && vPixelStride == 2)
155 if (uRowStride == (
int)uvRowBytes)
156 memcpy(uv, uData, (
size_t)uvHeight * uvRowBytes);
159 for (
int row = 0; row < uvHeight; row++)
160 memcpy(uv + (
size_t)row * uvRowBytes, uData + (size_t)row * uRowStride,
167 for (
int row = 0; row < uvHeight; row++)
169 const uint8_t* u = uData + (size_t)row * uRowStride;
170 const uint8_t* v = vData + (size_t)row * vRowStride;
171 BYTE* d = uv + (size_t)row * uvRowBytes;
172 for (
int col = 0; col < uvWidth; col++, u += 2, v += 2, d += 2)
183 for (
int row = 0; row < uvHeight; row++)
185 const uint8_t* uRow = uData + (size_t)row * uRowStride;
186 const uint8_t* vRow = vData + (size_t)row * vRowStride;
187 BYTE* dstRow = uv + (size_t)row * uvRowBytes;
188 for (
int col = 0; col < uvWidth; col++)
190 dstRow[col * 2] = uRow[col * uPixelStride];
191 dstRow[col * 2 + 1] = vRow[col * vPixelStride];
197static void cam_android_deliver_frame(CamAndroidStream* stream, AImage* image)
199 WINPR_ASSERT(stream);
204 if (AImage_getWidth(image, &width) != AMEDIA_OK ||
205 AImage_getHeight(image, &height) != AMEDIA_OK)
208 const size_t ySize = (size_t)(width * height);
209 const size_t nv12Size = ySize + ySize / 2;
211 if (stream->nv12BufferSize < nv12Size)
213 BYTE* tmp = (BYTE*)realloc(stream->nv12Buffer, nv12Size);
216 stream->nv12Buffer = tmp;
217 stream->nv12BufferSize = nv12Size;
222 uint8_t* yData =
nullptr;
223 uint8_t* uData =
nullptr;
224 uint8_t* vData =
nullptr;
225 if (AImage_getPlaneData(image, 0, &yData, &dataLength) != AMEDIA_OK ||
226 AImage_getPlaneData(image, 1, &uData, &dataLength) != AMEDIA_OK ||
227 AImage_getPlaneData(image, 2, &vData, &dataLength) != AMEDIA_OK)
232 int uPixelStride = 0;
234 int vPixelStride = 0;
235 AImage_getPlaneRowStride(image, 0, &yRowStride);
236 AImage_getPlaneRowStride(image, 1, &uRowStride);
237 AImage_getPlanePixelStride(image, 1, &uPixelStride);
238 AImage_getPlaneRowStride(image, 2, &vRowStride);
239 AImage_getPlanePixelStride(image, 2, &vPixelStride);
241 cam_android_yuv420_888_to_nv12(stream->nv12Buffer, width, height, yData, yRowStride, uData,
242 uRowStride, uPixelStride, vData, vRowStride, vPixelStride);
244 stream->sampleCallback(stream->dev, stream->streamIndex, stream->nv12Buffer, nv12Size);
247static void cam_android_on_image_available(
void* context, AImageReader* reader)
249 CamAndroidStream* stream = (CamAndroidStream*)context;
250 WINPR_ASSERT(stream);
251 WINPR_ASSERT(reader);
253 EnterCriticalSection(&stream->lock);
254 const BOOL streaming = stream->streaming;
255 LeaveCriticalSection(&stream->lock);
260 AImage* image =
nullptr;
261 if (AImageReader_acquireLatestImage(reader, &image) == AMEDIA_OK && image)
263 cam_android_deliver_frame(stream, image);
264 AImage_delete(image);
269static void cam_android_mark_device_error(CamAndroidStream* stream)
273 EnterCriticalSection(&stream->lock);
274 stream->deviceError = TRUE;
275 stream->streaming = FALSE;
276 LeaveCriticalSection(&stream->lock);
279static void cam_android_device_disconnected(
void* context, ACameraDevice* device)
281 WINPR_UNUSED(device);
282 WLog_WARN(TAG,
"Camera device disconnected");
283 cam_android_mark_device_error((CamAndroidStream*)context);
286static void cam_android_device_error(
void* context, ACameraDevice* device,
int error)
288 WINPR_UNUSED(device);
289 WLog_ERR(TAG,
"Camera device error %d", error);
290 cam_android_mark_device_error((CamAndroidStream*)context);
293static void cam_android_session_active(
void* context, ACameraCaptureSession* session)
295 WINPR_UNUSED(context);
296 WINPR_UNUSED(session);
297 WLog_DBG(TAG,
"Capture session active");
300static void cam_android_session_closed(
void* context, ACameraCaptureSession* session)
302 WINPR_UNUSED(context);
303 WINPR_UNUSED(session);
304 WLog_DBG(TAG,
"Capture session closed");
307static void cam_android_session_ready(
void* context, ACameraCaptureSession* session)
309 WINPR_UNUSED(context);
310 WINPR_UNUSED(session);
311 WLog_DBG(TAG,
"Capture session ready");
316static BOOL cam_android_is_backward_compatible(ACameraMetadata* characteristics)
318 WINPR_ASSERT(characteristics);
320 ACameraMetadata_const_entry caps;
321 if (ACameraMetadata_getConstEntry(characteristics, ACAMERA_REQUEST_AVAILABLE_CAPABILITIES,
322 &caps) != ACAMERA_OK)
325 for (uint32_t i = 0; i < caps.count; i++)
327 if (caps.data.u8[i] == ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE)
333static UINT cam_android_enumerate(ICamHal* ihal, ICamHalEnumCallback callback,
CameraPlugin* ecam,
336 CamAndroidHal* hal = (CamAndroidHal*)ihal;
344 if (!freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT_NV12, FREERDP_VIDEO_FORMAT_H264))
346 WLog_WARN(TAG,
"Camera redirection unavailable: no conversion of NV12 frames to H.264");
350 ACameraIdList* idList =
nullptr;
351 if (ACameraManager_getCameraIdList(hal->manager, &idList) != ACAMERA_OK || !idList)
357 BOOL seenFacing[ACAMERA_LENS_FACING_EXTERNAL + 1] = WINPR_C_ARRAY_INIT;
359 for (
int i = 0; i < idList->numCameras; i++)
361 const char* cameraId = idList->cameraIds[i];
363 ACameraMetadata* characteristics =
nullptr;
364 if (ACameraManager_getCameraCharacteristics(hal->manager, cameraId, &characteristics) !=
369 if (!cam_android_is_backward_compatible(characteristics))
371 WLog_DBG(TAG,
"Skipping camera %s: not BACKWARD_COMPATIBLE", cameraId);
372 ACameraMetadata_free(characteristics);
376 ACameraMetadata_const_entry facingEntry;
377 const char* facingName =
"Camera";
378 uint8_t facing = ACAMERA_LENS_FACING_EXTERNAL;
379 if (ACameraMetadata_getConstEntry(characteristics, ACAMERA_LENS_FACING, &facingEntry) ==
382 facing = facingEntry.data.u8[0];
385 case ACAMERA_LENS_FACING_FRONT:
386 facingName =
"Front Camera";
388 case ACAMERA_LENS_FACING_BACK:
389 facingName =
"Back Camera";
392 facingName =
"External Camera";
398 if (facing <= ACAMERA_LENS_FACING_EXTERNAL)
400 if (seenFacing[facing])
402 WLog_DBG(TAG,
"Skipping camera %s: already have a %s", cameraId, facingName);
403 ACameraMetadata_free(characteristics);
406 seenFacing[facing] = TRUE;
410 (void)_snprintf(deviceId,
sizeof(deviceId), ANDROID_CAMERA_PREFIX
"%s", cameraId);
412 IFCALL(callback, ecam, hchannel, deviceId, facingName);
415 ACameraMetadata_free(characteristics);
418 ACameraManager_deleteCameraIdList(idList);
424static void cam_android_close_device(CamAndroidStream* stream)
426 WINPR_ASSERT(stream);
428 EnterCriticalSection(&stream->lock);
429 stream->streaming = FALSE;
430 LeaveCriticalSection(&stream->lock);
432 if (stream->imageReader)
433 AImageReader_setImageListener(stream->imageReader,
nullptr);
435 cam_android_close_session(stream);
439 ACameraDevice_close(stream->device);
440 stream->device =
nullptr;
443 if (stream->imageReader)
445 AImageReader_delete(stream->imageReader);
446 stream->imageReader =
nullptr;
447 stream->window =
nullptr;
449 free(stream->nv12Buffer);
450 stream->nv12Buffer =
nullptr;
451 stream->nv12BufferSize = 0;
452 stream->deviceError = FALSE;
457static void cam_android_close_other_devices(CamAndroidHal* hal,
const char* keepDeviceId)
460 WINPR_ASSERT(keepDeviceId);
462 ULONG_PTR* keys =
nullptr;
463 const size_t count = HashTable_GetKeys(hal->streams, &keys);
464 for (
size_t i = 0; i < count; i++)
466 const char*
id = (
const char*)keys[i];
467 if (strcmp(
id, keepDeviceId) == 0)
470 CamAndroidStream* other = (CamAndroidStream*)HashTable_GetItemValue(hal->streams,
id);
471 if (!other || !other->device)
474 WLog_DBG(TAG,
"Closing camera %s to free resources for %s",
id, keepDeviceId);
475 cam_android_close_device(other);
480static CamAndroidStream* cam_android_get_or_create_stream(CamAndroidHal* hal,
const char* deviceId,
481 CAM_ERROR_CODE* errorCode)
484 WINPR_ASSERT(deviceId);
485 WINPR_ASSERT(errorCode);
487 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
491 stream = (CamAndroidStream*)calloc(1,
sizeof(CamAndroidStream));
494 *errorCode = CAM_ERROR_CODE_OutOfMemory;
497 if (!InitializeCriticalSectionEx(&stream->lock, 0, 0))
500 *errorCode = CAM_ERROR_CODE_UnexpectedError;
503 if (!HashTable_Insert(hal->streams, deviceId, stream))
505 DeleteCriticalSection(&stream->lock);
507 *errorCode = CAM_ERROR_CODE_UnexpectedError;
515static BOOL cam_android_open_device(CamAndroidHal* hal, CamAndroidStream* stream,
516 const char* deviceId,
const char* cameraId,
517 CAM_ERROR_CODE* errorCode)
520 WINPR_ASSERT(stream);
521 WINPR_ASSERT(deviceId);
522 WINPR_ASSERT(cameraId);
523 WINPR_ASSERT(errorCode);
526 EnterCriticalSection(&stream->lock);
527 const BOOL hadError = stream->deviceError;
528 LeaveCriticalSection(&stream->lock);
529 if (stream->device && hadError)
530 cam_android_close_device(stream);
535 cam_android_close_other_devices(hal, deviceId);
537 ACameraDevice_StateCallbacks deviceCallbacks = {
539 .onDisconnected = cam_android_device_disconnected,
540 .onError = cam_android_device_error,
543 camera_status_t status =
544 ACameraManager_openCamera(hal->manager, cameraId, &deviceCallbacks, &stream->device);
545 if (status != ACAMERA_OK)
547 WLog_ERR(TAG,
"ACameraManager_openCamera failed: %d", status);
548 *errorCode = CAM_ERROR_CODE_InvalidRequest;
552 WLog_DBG(TAG,
"Opened camera %s", cameraId);
556static BOOL cam_android_activate(ICamHal* ihal,
const char* deviceId, CAM_ERROR_CODE* errorCode)
558 CamAndroidHal* hal = (CamAndroidHal*)ihal;
560 WINPR_ASSERT(deviceId);
561 WINPR_ASSERT(errorCode);
563 *errorCode = CAM_ERROR_CODE_None;
565 const char* cameraId = cam_android_extract_camera_id(deviceId);
568 *errorCode = CAM_ERROR_CODE_InvalidRequest;
573 return cam_android_get_or_create_stream(hal, deviceId, errorCode) !=
nullptr;
576static BOOL cam_android_deactivate(ICamHal* ihal,
const char* deviceId, CAM_ERROR_CODE* errorCode)
578 CamAndroidHal* hal = (CamAndroidHal*)ihal;
580 WINPR_ASSERT(deviceId);
581 WINPR_ASSERT(errorCode);
583 *errorCode = CAM_ERROR_CODE_None;
585 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
586 if (!stream || !stream->device)
589 cam_android_close_device(stream);
593static INT16 cam_android_get_media_type_descriptions(ICamHal* ihal,
const char* deviceId,
596 size_t nSupportedFormats,
600 WINPR_UNUSED(streamIndex);
601 CamAndroidHal* hal = (CamAndroidHal*)ihal;
603 WINPR_ASSERT(deviceId);
604 WINPR_ASSERT(supportedFormats || (nSupportedFormats == 0));
605 WINPR_ASSERT(mediaTypes);
606 WINPR_ASSERT(nMediaTypes);
608 size_t maxMediaTypes = *nMediaTypes;
611 const char* cameraId = cam_android_extract_camera_id(deviceId);
616 INT16 formatIndex = -1;
617 for (
size_t i = 0; i < nSupportedFormats; i++)
619 if (supportedFormats[i].inputFormat == CAM_MEDIA_FORMAT_NV12)
621 formatIndex = (INT16)i;
627 WLog_WARN(TAG,
"Server does not offer NV12 format");
631 ACameraMetadata* characteristics =
nullptr;
632 if (ACameraManager_getCameraCharacteristics(hal->manager, cameraId, &characteristics) !=
637 ACameraMetadata_const_entry entry;
638 if (ACameraMetadata_getConstEntry(
639 characteristics, ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, &entry) != ACAMERA_OK)
641 ACameraMetadata_free(characteristics);
646 for (uint32_t i = 0; i + 3 < entry.count; i += 4)
648 const int32_t fmt = entry.data.i32[i];
649 const int32_t w = entry.data.i32[i + 1];
650 const int32_t h = entry.data.i32[i + 2];
651 const int32_t isInput = entry.data.i32[i + 3];
655 if (fmt != AIMAGE_FORMAT_YUV_420_888)
658 if (w > 1920 || h > 1080)
661 if (w < CAM_ANDROID_MIN_WIDTH || h < CAM_ANDROID_MIN_HEIGHT || (w % 2) != 0 || (h % 2) != 0)
664 mediaTypes[nTypes].Format = CAM_MEDIA_FORMAT_NV12;
665 mediaTypes[nTypes].Width = (UINT32)w;
666 mediaTypes[nTypes].Height = (UINT32)h;
667 mediaTypes[nTypes].FrameRateNumerator = CAM_ANDROID_FPS;
668 mediaTypes[nTypes].FrameRateDenominator = 1;
669 mediaTypes[nTypes].PixelAspectRatioNumerator = 1;
670 mediaTypes[nTypes].PixelAspectRatioDenominator = 1;
672 WLog_DBG(TAG,
"Camera format NV12 %dx%d @ %dfps", w, h, CAM_ANDROID_FPS);
674 if (++nTypes >= maxMediaTypes)
678 *nMediaTypes = nTypes;
679 ACameraMetadata_free(characteristics);
683 WLog_ERR(TAG,
"No supported YUV resolutions found for camera %s", cameraId);
690static CAM_ERROR_CODE cam_android_start_stream(ICamHal* ihal,
CameraDevice* dev,
size_t streamIndex,
692 ICamHalSampleCapturedCallback callback)
694 CamAndroidHal* hal = (CamAndroidHal*)ihal;
697 WINPR_ASSERT(mediaType);
698 WINPR_ASSERT(callback);
700 const char* deviceId = dev->deviceId;
702 const char* cameraId = cam_android_extract_camera_id(deviceId);
704 return CAM_ERROR_CODE_InvalidRequest;
706 CAM_ERROR_CODE errorCode = CAM_ERROR_CODE_None;
707 CamAndroidStream* stream = cam_android_get_or_create_stream(hal, deviceId, &errorCode);
711 if (!cam_android_open_device(hal, stream, deviceId, cameraId, &errorCode))
715 stream->streamIndex = streamIndex;
716 stream->sampleCallback = callback;
718 const int32_t width = (int32_t)mediaType->Width;
719 const int32_t height = (int32_t)mediaType->Height;
721 if (AImageReader_new(width, height, AIMAGE_FORMAT_YUV_420_888, 4, &stream->imageReader) !=
724 WLog_ERR(TAG,
"AImageReader_new failed");
725 return CAM_ERROR_CODE_UnexpectedError;
728 AImageReader_ImageListener listener = {
730 .onImageAvailable = cam_android_on_image_available,
732 AImageReader_setImageListener(stream->imageReader, &listener);
734 if (AImageReader_getWindow(stream->imageReader, &stream->window) != AMEDIA_OK)
736 WLog_ERR(TAG,
"AImageReader_getWindow failed");
740 ACaptureSessionOutput_create(stream->window, &stream->sessionOutput);
741 ACaptureSessionOutputContainer_create(&stream->outputContainer);
742 ACaptureSessionOutputContainer_add(stream->outputContainer, stream->sessionOutput);
743 ACameraOutputTarget_create(stream->window, &stream->outputTarget);
744 ACameraDevice_createCaptureRequest(stream->device, TEMPLATE_RECORD, &stream->captureRequest);
745 ACaptureRequest_addTarget(stream->captureRequest, stream->outputTarget);
747 ACameraCaptureSession_stateCallbacks sessionCallbacks = {
749 .onActive = cam_android_session_active,
750 .onClosed = cam_android_session_closed,
751 .onReady = cam_android_session_ready,
754 if (ACameraDevice_createCaptureSession(stream->device, stream->outputContainer,
755 &sessionCallbacks, &stream->session) != ACAMERA_OK)
757 WLog_ERR(TAG,
"ACameraDevice_createCaptureSession failed");
761 if (ACameraCaptureSession_setRepeatingRequest(stream->session,
nullptr, 1,
762 &stream->captureRequest,
nullptr) != ACAMERA_OK)
764 WLog_ERR(TAG,
"ACameraCaptureSession_setRepeatingRequest failed");
768 EnterCriticalSection(&stream->lock);
769 stream->streaming = TRUE;
770 LeaveCriticalSection(&stream->lock);
772 WLog_DBG(TAG,
"Camera stream started: %dx%d", width, height);
773 return CAM_ERROR_CODE_None;
776 cam_android_close_device(stream);
777 return CAM_ERROR_CODE_UnexpectedError;
780static CAM_ERROR_CODE cam_android_stop_stream(ICamHal* ihal,
const char* deviceId,
783 WINPR_UNUSED(streamIndex);
784 CamAndroidHal* hal = (CamAndroidHal*)ihal;
786 WINPR_ASSERT(deviceId);
788 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
790 return CAM_ERROR_CODE_None;
792 cam_android_close_device(stream);
794 WLog_DBG(TAG,
"Camera stream stopped for %s", deviceId);
795 return CAM_ERROR_CODE_None;
798static void cam_android_stream_free(
void* obj)
800 CamAndroidStream* stream = (CamAndroidStream*)obj;
804 cam_android_close_device(stream);
805 DeleteCriticalSection(&stream->lock);
809static CAM_ERROR_CODE cam_android_free(ICamHal* ihal)
811 CamAndroidHal* hal = (CamAndroidHal*)ihal;
813 return CAM_ERROR_CODE_NotInitialized;
815 HashTable_Free(hal->streams);
817 ACameraManager_delete(hal->manager);
819 return CAM_ERROR_CODE_None;
822FREERDP_ENTRY_POINT(UINT VCAPITYPE android_freerdp_rdpecam_client_subsystem_entry(
825 WINPR_ASSERT(pEntryPoints);
827 CamAndroidHal* hal = (CamAndroidHal*)calloc(1,
sizeof(CamAndroidHal));
829 return CHANNEL_RC_NO_MEMORY;
831 hal->manager = ACameraManager_create();
835 return ERROR_INTERNAL_ERROR;
838 hal->streams = HashTable_New(FALSE);
842 if (!HashTable_SetupForStringData(hal->streams, FALSE))
845 wObject* obj = HashTable_ValueObject(hal->streams);
849 hal->iHal.Enumerate = cam_android_enumerate;
850 hal->iHal.Activate = cam_android_activate;
851 hal->iHal.Deactivate = cam_android_deactivate;
852 hal->iHal.GetMediaTypeDescriptions = cam_android_get_media_type_descriptions;
853 hal->iHal.StartStream = cam_android_start_stream;
854 hal->iHal.StopStream = cam_android_stop_stream;
855 hal->iHal.Free = cam_android_free;
857 UINT ret = pEntryPoints->pRegisterCameraHal(pEntryPoints->plugin, &hal->iHal);
858 if (ret != CHANNEL_RC_OK)
860 WLog_ERR(TAG,
"RegisterCameraHal failed: %" PRIu32, ret);
867 cam_android_free(&hal->iHal);
868 return ERROR_INTERNAL_ERROR;
This struct contains function pointer to initialize/free objects.
OBJECT_FREE_FN fnObjectFree