diff --git a/include/SDL3/SDL_camera.h b/include/SDL3/SDL_camera.h index a57a66a0699678..c070666273d700 100644 --- a/include/SDL3/SDL_camera.h +++ b/include/SDL3/SDL_camera.h @@ -66,8 +66,8 @@ typedef struct SDL_CameraSpec Uint32 format; /**< Frame SDL_PixelFormatEnum format */ int width; /**< Frame width */ int height; /**< Frame height */ - int interval_numerator; /**< Frame rate numerator ((dom / num) == fps) */ - int interval_denominator; /**< Frame rate demoninator ((dom / num) == fps)*/ + int interval_numerator; /**< Frame rate numerator ((dom / num) == fps, (num / dom) == duration) */ + int interval_denominator; /**< Frame rate demoninator ((dom / num) == fps, (num / dom) == duration) */ } SDL_CameraSpec; /** diff --git a/include/build_config/SDL_build_config.h.cmake b/include/build_config/SDL_build_config.h.cmake index 2891f525bc511f..8ceac861453b4e 100644 --- a/include/build_config/SDL_build_config.h.cmake +++ b/include/build_config/SDL_build_config.h.cmake @@ -246,8 +246,6 @@ #cmakedefine USE_POSIX_SPAWN @USE_POSIX_SPAWN@ -#cmakedefine HAVE_COREMEDIA - /* SDL internal assertion support */ #if @SDL_DEFAULT_ASSERT_LEVEL_CONFIGURED@ #cmakedefine SDL_DEFAULT_ASSERT_LEVEL @SDL_DEFAULT_ASSERT_LEVEL@ diff --git a/include/build_config/SDL_build_config_ios.h b/include/build_config/SDL_build_config_ios.h index b3fb4e1eaec8fe..f935c45d6aa0ea 100644 --- a/include/build_config/SDL_build_config_ios.h +++ b/include/build_config/SDL_build_config_ios.h @@ -197,8 +197,6 @@ #define SDL_VIDEO_METAL 1 #endif -#define HAVE_COREMEDIA 1 - /* Enable system power support */ #define SDL_POWER_UIKIT 1 @@ -212,6 +210,10 @@ #define SDL_FILESYSTEM_COCOA 1 /* enable camera support */ +#ifndef SDL_PLATFORM_TVOS #define SDL_CAMERA_DRIVER_COREMEDIA 1 +#endif + +#define SDL_CAMERA_DRIVER_DUMMY 1 #endif /* SDL_build_config_ios_h_ */ diff --git a/include/build_config/SDL_build_config_macos.h b/include/build_config/SDL_build_config_macos.h index 4592c087787e6b..64c90ee387a562 100644 --- a/include/build_config/SDL_build_config_macos.h +++ b/include/build_config/SDL_build_config_macos.h @@ -260,8 +260,6 @@ #endif #endif -#define HAVE_COREMEDIA 1 - /* Enable system power support */ #define SDL_POWER_MACOSX 1 diff --git a/src/camera/SDL_camera.c b/src/camera/SDL_camera.c index 3a7854a19f31d7..6088f3d49d4ac6 100644 --- a/src/camera/SDL_camera.c +++ b/src/camera/SDL_camera.c @@ -73,6 +73,12 @@ const char *SDL_GetCurrentCameraDriver(void) return camera_driver.name; } +char *SDL_GetCameraThreadName(SDL_CameraDevice *device, char *buf, size_t buflen) +{ + (void)SDL_snprintf(buf, buflen, "SDLCamera%d", (int) device->instance_id); + return buf; +} + int SDL_AddCameraFormat(CameraFormatAddData *data, Uint32 fmt, int w, int h, int interval_numerator, int interval_denominator) { SDL_assert(data != NULL); @@ -683,7 +689,7 @@ SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device) failed = SDL_TRUE; } - // we can let go of the lock once we've tried to grab a frame of video and maybe moved the output frame from the empty to the filled list. + // we can let go of the lock once we've tried to grab a frame of video and maybe moved the output frame off the empty list. // this lets us chew up the CPU for conversion and scaling without blocking other threads. SDL_UnlockMutex(device->lock); @@ -988,7 +994,7 @@ SDL_Camera *SDL_OpenCameraDevice(SDL_CameraDeviceID instance_id, const SDL_Camer // Start the camera thread if necessary if (!camera_driver.impl.ProvidesOwnCallbackThread) { char threadname[64]; - SDL_snprintf(threadname, sizeof (threadname), "SDLCamera%d", (int) instance_id); + SDL_GetCameraThreadName(device, threadname, sizeof (threadname)); device->thread = SDL_CreateThreadInternal(CameraThread, threadname, 0, device); if (!device->thread) { ClosePhysicalCameraDevice(device); diff --git a/src/camera/SDL_syscamera.h b/src/camera/SDL_syscamera.h index b99239ec51d723..9f556523d109a5 100644 --- a/src/camera/SDL_syscamera.h +++ b/src/camera/SDL_syscamera.h @@ -28,10 +28,7 @@ #define DEBUG_CAMERA 0 -// !!! FIXME: update these drivers! -#ifdef SDL_CAMERA_DRIVER_COREMEDIA -#undef SDL_CAMERA_DRIVER_COREMEDIA -#endif +// !!! FIXME: update this driver! #ifdef SDL_CAMERA_DRIVER_ANDROID #undef SDL_CAMERA_DRIVER_ANDROID #endif @@ -53,6 +50,9 @@ extern SDL_CameraDevice *SDL_FindPhysicalCameraDeviceByCallback(SDL_bool (*callb // Backends should call this when the user has approved/denied access to a camera. extern void SDL_CameraDevicePermissionOutcome(SDL_CameraDevice *device, SDL_bool approved); +// Backends can call this to get a standardized name for a thread to power a specific camera device. +extern char *SDL_GetCameraThreadName(SDL_CameraDevice *device, char *buf, size_t buflen); + // These functions are the heart of the camera threads. Backends can call them directly if they aren't using the SDL-provided thread. extern void SDL_CameraThreadSetup(SDL_CameraDevice *device); extern SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device); diff --git a/src/camera/coremedia/SDL_camera_coremedia.m b/src/camera/coremedia/SDL_camera_coremedia.m index c37962026c205e..c3940288596173 100644 --- a/src/camera/coremedia/SDL_camera_coremedia.m +++ b/src/camera/coremedia/SDL_camera_coremedia.m @@ -26,17 +26,6 @@ #include "../SDL_camera_c.h" #include "../../thread/SDL_systhread.h" -#if defined(HAVE_COREMEDIA) && defined(SDL_PLATFORM_MACOS) && (__MAC_OS_X_VERSION_MAX_ALLOWED < 101500) -// AVCaptureDeviceTypeBuiltInWideAngleCamera requires macOS SDK 10.15 -#undef HAVE_COREMEDIA -#endif - -#ifdef SDL_PLATFORM_TVOS -#undef HAVE_COREMEDIA -#endif - -#ifdef HAVE_COREMEDIA - #import #import @@ -50,537 +39,433 @@ * MACOSX: * Add to the Code Sign Entitlement file: * com.apple.security.device.camera - * - * - * IOS: - * - * - Need to link with:: CoreMedia CoreVideo - * - Add #define SDL_CAMERA 1 - * to SDL_build_config_ios.h */ -@class MySampleBufferDelegate; - -struct SDL_PrivateCameraData -{ - dispatch_queue_t queue; - MySampleBufferDelegate *delegate; - AVCaptureSession *session; - CMSimpleQueueRef frame_queue; -}; - -static NSString *fourcc_to_nstring(Uint32 code) +static Uint32 CoreMediaFormatToSDL(FourCharCode fmt) { - Uint8 buf[4]; - *(Uint32 *)buf = code; - return [NSString stringWithFormat:@"%c%c%c%c", buf[3], buf[2], buf[1], buf[0]]; + switch (fmt) { + #define CASE(x, y) case x: return y + // the 16LE ones should use 16BE if we're on a Bigendian system like PowerPC, + // but at current time there is no bigendian Apple platform that has CoreMedia. + CASE(kCMPixelFormat_16LE555, SDL_PIXELFORMAT_RGB555); + CASE(kCMPixelFormat_16LE5551, SDL_PIXELFORMAT_RGBA5551); + CASE(kCMPixelFormat_16LE565, SDL_PIXELFORMAT_RGB565); + CASE(kCMPixelFormat_24RGB, SDL_PIXELFORMAT_RGB24); + CASE(kCMPixelFormat_32ARGB, SDL_PIXELFORMAT_ARGB32); + CASE(kCMPixelFormat_32BGRA, SDL_PIXELFORMAT_BGRA32); + CASE(kCMPixelFormat_422YpCbCr8, SDL_PIXELFORMAT_YUY2); + CASE(kCMPixelFormat_422YpCbCr8_yuvs, SDL_PIXELFORMAT_UYVY); + #undef CASE + default: + #if DEBUG_CAMERA + SDL_Log("CAMERA: Unknown format FourCharCode '%d'", (int) fmt); + #endif + break; + } + return SDL_PIXELFORMAT_UNKNOWN; } -static NSArray *DiscoverCameraDevices() -{ - NSArray *deviceType = @[AVCaptureDeviceTypeBuiltInWideAngleCamera]; +@class SDLCaptureVideoDataOutputSampleBufferDelegate; - AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession - discoverySessionWithDeviceTypes:deviceType - mediaType:AVMediaTypeVideo - position:AVCaptureDevicePositionUnspecified]; +// just a simple wrapper to help ARC manage memory... +@interface SDLPrivateCameraData : NSObject +@property(nonatomic, retain) AVCaptureSession *session; +@property(nonatomic, retain) SDLCaptureVideoDataOutputSampleBufferDelegate *delegate; +@property(nonatomic, assign) CMSampleBufferRef current_sample; +@end - NSArray *devices = discoverySession.devices; +@implementation SDLPrivateCameraData +@end - if ([devices count] > 0) { - return devices; - } else { - AVCaptureDevice *captureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; - if (captureDevice == nil) { - return devices; + +static SDL_bool CheckCameraPermissions(SDL_CameraDevice *device) +{ + if (device->permission == 0) { // still expecting a permission result. + if (@available(macOS 14, *)) { + const AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]; + if (status != AVAuthorizationStatusNotDetermined) { // NotDetermined == still waiting for an answer from the user. + SDL_CameraDevicePermissionOutcome(device, (status == AVAuthorizationStatusAuthorized) ? SDL_TRUE : SDL_FALSE); + } } else { - NSArray *default_device = @[ captureDevice ]; - return default_device; + SDL_CameraDevicePermissionOutcome(device, SDL_TRUE); // always allowed (or just unqueryable...?) on older macOS. } } - return devices; + return (device->permission > 0); } -static AVCaptureDevice *GetCameraDeviceByName(const char *dev_name) -{ - NSArray *devices = DiscoverCameraDevices(); +// this delegate just receives new video frames on a Grand Central Dispatch queue, and fires off the +// main device thread iterate function directly to consume it. +@interface SDLCaptureVideoDataOutputSampleBufferDelegate : NSObject + @property SDL_CameraDevice *device; + -(id) init:(SDL_CameraDevice *) dev; + -(void) captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection; +@end - for (AVCaptureDevice *device in devices) { - char buf[1024]; - NSString *cameraID = [device localizedName]; - const char *str = [cameraID UTF8String]; - SDL_snprintf(buf, sizeof (buf) - 1, "%s", str); - if (SDL_strcmp(buf, dev_name) == 0) { - return device; +@implementation SDLCaptureVideoDataOutputSampleBufferDelegate + + -(id) init:(SDL_CameraDevice *) dev { + if ( self = [super init] ) { + _device = dev; } + return self; } - return nil; -} -static Uint32 nsfourcc_to_sdlformat(NSString *nsfourcc) -{ - const char *str = [nsfourcc UTF8String]; - - /* FIXME - * on IOS this mode gives 2 planes, and it's NV12 - * on macos, 1 plane/ YVYU - */ - #ifdef SDL_PLATFORM_MACOS - if (SDL_strcmp("420v", str) == 0) return SDL_PIXELFORMAT_YVYU; - #else - if (SDL_strcmp("420v", str) == 0) return SDL_PIXELFORMAT_NV12; - #endif + - (void) captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection + { + SDL_CameraDevice *device = self.device; + if (!device || !device->hidden) { + return; // oh well. + } - if (SDL_strcmp("yuvs", str) == 0) return SDL_PIXELFORMAT_UYVY; - if (SDL_strcmp("420f", str) == 0) return SDL_PIXELFORMAT_UNKNOWN; + if (!CheckCameraPermissions(device)) { + return; // nothing to do right now, dump what is probably a completely black frame. + } - #if DEBUG_CAMERA - SDL_Log("CAMERA: Unknown format '%s'", str); - #endif + SDLPrivateCameraData *hidden = (__bridge SDLPrivateCameraData *) device->hidden; + hidden.current_sample = sampleBuffer; + SDL_CameraThreadIterate(device); + hidden.current_sample = NULL; + } - return SDL_PIXELFORMAT_UNKNOWN; -} + - (void)captureOutput:(AVCaptureOutput *)output didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection + { + #if DEBUG_CAMERA + SDL_Log("CAMERA: Drop frame."); + #endif + } +@end -static NSString *sdlformat_to_nsfourcc(Uint32 fmt) +static int COREMEDIA_WaitDevice(SDL_CameraDevice *device) { - const char *str = ""; - NSString *result; - -#ifdef SDL_PLATFORM_MACOS - if (fmt == SDL_PIXELFORMAT_YVYU) str = "420v"; -#else - if (fmt == SDL_PIXELFORMAT_NV12) str = "420v"; -#endif - if (fmt == SDL_PIXELFORMAT_UYVY) str = "yuvs"; - - return [[NSString alloc] initWithUTF8String: str]; + return 0; // this isn't used atm, since we run our own thread out of Grand Central Dispatch. } +static int COREMEDIA_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint64 *timestampNS) +{ + int retval = 1; + SDLPrivateCameraData *hidden = (__bridge SDLPrivateCameraData *) device->hidden; + CMSampleBufferRef sample_buffer = hidden.current_sample; + hidden.current_sample = NULL; + SDL_assert(sample_buffer != NULL); // should only have been called from our delegate with a new frame. + + CMSampleTimingInfo timinginfo; + if (CMSampleBufferGetSampleTimingInfo(sample_buffer, 0, &timinginfo) == noErr) { + *timestampNS = (Uint64) (CMTimeGetSeconds(timinginfo.presentationTimeStamp) * ((Float64) SDL_NS_PER_SECOND)); + } else { + SDL_assert(!"this shouldn't happen, I think."); + *timestampNS = 0; + } -@interface MySampleBufferDelegate : NSObject - @property struct SDL_PrivateCameraData *hidden; - - (void) set: (struct SDL_PrivateCameraData *) val; -@end + CVImageBufferRef image = CMSampleBufferGetImageBuffer(sample_buffer); // does not retain `image` (and we don't want it to). + const int numPlanes = CVPixelBufferGetPlaneCount(image); + const int planar = CVPixelBufferIsPlanar(image); -@implementation MySampleBufferDelegate + #if DEBUG_CAMERA + const int w = CVPixelBufferGetWidth(image); + const int h = CVPixelBufferGetHeight(image); + const int sz = CVPixelBufferGetDataSize(image); + const int pitch = CVPixelBufferGetBytesPerRow(image); + SDL_Log("CAMERA: buffer planar=%d numPlanes=%d %d x %d sz=%d pitch=%d", planar, numPlanes, w, h, sz, pitch); + #endif - - (void) set: (struct SDL_PrivateCameraData *) val { - _hidden = val; - } + // !!! FIXME: this currently copies the data to the surface (see FIXME about non-contiguous planar surfaces, but in theory we could just keep this locked until ReleaseFrame... + CVPixelBufferLockBaseAddress(image, 0); - - (void) captureOutput:(AVCaptureOutput *)output - didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *) connection { - CFRetain(sampleBuffer); - CMSimpleQueueEnqueue(_hidden->frame_queue, sampleBuffer); + if ((planar == 0) && (numPlanes == 0)) { + const int pitch = (int) CVPixelBufferGetBytesPerRow(image); + const size_t buflen = pitch * frame->h; + frame->pixels = SDL_aligned_alloc(SDL_SIMDGetAlignment(), buflen); + if (frame->pixels == NULL) { + retval = -1; + } else { + frame->pitch = pitch; + SDL_memcpy(frame->pixels, CVPixelBufferGetBaseAddress(image), buflen); } - - - (void)captureOutput:(AVCaptureOutput *)output - didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection { - #if DEBUG_CAMERA - SDL_Log("CAMERA: Drop frame.."); - #endif + } else { + // !!! FIXME: we have an open issue in SDL3 to allow SDL_Surface to support non-contiguous planar data, but we don't have it yet. + size_t buflen = 0; + for (int i = 0; (i < numPlanes) && (i < 3); i++) { + buflen += CVPixelBufferGetBytesPerRowOfPlane(image, i); } -@end + buflen *= frame->h; -static int COREMEDIA_OpenDevice(SDL_CameraDevice *_this) -{ - _this->hidden = (struct SDL_PrivateCameraData *) SDL_calloc(1, sizeof (struct SDL_PrivateCameraData)); - if (_this->hidden == NULL) { - return -1; + frame->pixels = SDL_aligned_alloc(SDL_SIMDGetAlignment(), buflen); + if (frame->pixels == NULL) { + retval = -1; + } else { + Uint8 *dst = frame->pixels; + frame->pitch = CVPixelBufferGetBytesPerRowOfPlane(image, 0); // this is what SDL3 currently expects, probably incorrectly. + for (int i = 0; (i < numPlanes) && (i < 3); i++) { + const void *src = CVPixelBufferGetBaseAddressOfPlane(image, i); + const size_t pitch = CVPixelBufferGetBytesPerRowOfPlane(image, i); + SDL_memcpy(dst, src, pitch * frame->h); + dst += pitch * frame->h; + } + } } - return 0; + + CVPixelBufferUnlockBaseAddress(image, 0); + + return retval; } -static void COREMEDIA_CloseDevice(SDL_CameraDevice *_this) +static void COREMEDIA_ReleaseFrame(SDL_CameraDevice *device, SDL_Surface *frame) { - if (!_this) { - return; - } + // !!! FIXME: this currently copies the data to the surface, but in theory we could just keep this locked until ReleaseFrame... + SDL_aligned_free(frame->pixels); +} - if (_this->hidden) { - AVCaptureSession *session = _this->hidden->session; +static void COREMEDIA_CloseDevice(SDL_CameraDevice *device) +{ + if (device && device->hidden) { + SDLPrivateCameraData *hidden = (SDLPrivateCameraData *) CFBridgingRelease(device->hidden); + device->hidden = NULL; + AVCaptureSession *session = hidden.session; if (session) { - AVCaptureInput *input; - AVCaptureVideoDataOutput *output; - input = [session.inputs objectAtIndex:0]; - [session removeInput:input]; - output = (AVCaptureVideoDataOutput*)[session.outputs objectAtIndex:0]; - [session removeOutput:output]; - // TODO more cleanup ? + hidden.session = nil; + [session stopRunning]; + [session removeInput:[session.inputs objectAtIndex:0]]; + [session removeOutput:(AVCaptureVideoDataOutput*)[session.outputs objectAtIndex:0]]; + session = nil; } - if (_this->hidden->frame_queue) { - CFRelease(_this->hidden->frame_queue); - } - - SDL_free(_this->hidden); - _this->hidden = NULL; + hidden.delegate = NULL; + hidden.current_sample = NULL; } } -static int COREMEDIA_InitDevice(SDL_CameraDevice *_this) +static int COREMEDIA_OpenDevice(SDL_CameraDevice *device, const SDL_CameraSpec *spec) { - // !!! FIXME: autorelease pool? - NSString *fmt = sdlformat_to_nsfourcc(_this->spec.format); - int w = _this->spec.width; - int h = _this->spec.height; - - NSError *error = nil; - AVCaptureDevice *device = nil; - AVCaptureDeviceInput *input = nil; - AVCaptureVideoDataOutput *output = nil; - - AVCaptureDeviceFormat *spec_format = nil; - -#ifdef SDL_PLATFORM_MACOS - if (@available(macOS 10.15, *)) { - // good. - } else { - return -1; - } -#endif - - device = GetCameraDeviceByName(_this->dev_name); - if (!device) { - goto error; - } - - _this->hidden->session = [[AVCaptureSession alloc] init]; - if (_this->hidden->session == nil) { - goto error; - } - - [_this->hidden->session setSessionPreset:AVCaptureSessionPresetHigh]; + AVCaptureDevice *avdevice = (__bridge AVCaptureDevice *) device->handle; // Pick format that matches the spec - NSArray *formats = [device formats]; + const Uint32 sdlfmt = spec->format; + const int w = spec->width; + const int h = spec->height; + const int rate = spec->interval_denominator; + AVCaptureDeviceFormat *spec_format = nil; + NSArray *formats = [avdevice formats]; for (AVCaptureDeviceFormat *format in formats) { CMFormatDescriptionRef formatDescription = [format formatDescription]; - FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription); - NSString *str = fourcc_to_nstring(mediaSubType); - if ([str isEqualToString:fmt]) { - CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDescription); - if (dim.width == w && dim.height == h) { - spec_format = format; - break; - } + if (CoreMediaFormatToSDL(CMFormatDescriptionGetMediaSubType(formatDescription)) != sdlfmt) { + continue; + } + + const CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDescription); + if ( ((int) dim.width != w) || (((int) dim.height) != h) ) { + continue; + } + + for (AVFrameRateRange *framerate in format.videoSupportedFrameRateRanges) { + if ((rate == (int) SDL_ceil((double) framerate.minFrameRate)) || (rate == (int) SDL_floor((double) framerate.maxFrameRate))) { + spec_format = format; + break; } } + + if (spec_format != nil) { + break; + } } if (spec_format == nil) { - return SDL_SetError("format not found"); + return SDL_SetError("camera spec format not available"); + } else if (![avdevice lockForConfiguration:NULL]) { + return SDL_SetError("Cannot lockForConfiguration"); } - // Set format - if ([device lockForConfiguration:NULL] == YES) { - device.activeFormat = spec_format; - [device unlockForConfiguration]; - } else { - return SDL_SetError("Cannot lockForConfiguration"); + avdevice.activeFormat = spec_format; + [avdevice unlockForConfiguration]; + + AVCaptureSession *session = [[AVCaptureSession alloc] init]; + if (session == nil) { + return SDL_SetError("Failed to allocate/init AVCaptureSession"); } - // Input - input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; + session.sessionPreset = AVCaptureSessionPresetHigh; + + NSError *error = nil; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:avdevice error:&error]; if (!input) { return SDL_SetError("Cannot create AVCaptureDeviceInput"); } - // Output - output = [[AVCaptureVideoDataOutput alloc] init]; - -#ifdef SDL_PLATFORM_MACOS - // FIXME this now fail on ios ... but not using anything works... - - // Specify the pixel format - output.videoSettings = - [NSDictionary dictionaryWithObject: - [NSNumber numberWithInt:kCVPixelFormatType_422YpCbCr8] - forKey:(id)kCVPixelBufferPixelFormatTypeKey]; -#endif - - _this->hidden->delegate = [[MySampleBufferDelegate alloc] init]; - [_this->hidden->delegate set:_this->hidden]; - + AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init]; + if (!output) { + return SDL_SetError("Cannot create AVCaptureVideoDataOutput"); + } - CMSimpleQueueCreate(kCFAllocatorDefault, 30 /* buffers */, &_this->hidden->frame_queue); - if (_this->hidden->frame_queue == nil) { - return SDL_SetError("CMSimpleQueueCreate() failed"); + char threadname[64]; + SDL_GetCameraThreadName(device, threadname, sizeof (threadname)); + dispatch_queue_t queue = dispatch_queue_create(threadname, NULL); + //dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); + if (!queue) { + return SDL_SetError("dispatch_queue_create() failed"); } - _this->hidden->queue = dispatch_queue_create("my_queue", NULL); - [output setSampleBufferDelegate:_this->hidden->delegate queue:_this->hidden->queue]; + SDLCaptureVideoDataOutputSampleBufferDelegate *delegate = [[SDLCaptureVideoDataOutputSampleBufferDelegate alloc] init:device]; + if (delegate == nil) { + return SDL_SetError("Cannot create SDLCaptureVideoDataOutputSampleBufferDelegate"); + } + [output setSampleBufferDelegate:delegate queue:queue]; - if ([_this->hidden->session canAddInput:input] ){ - [_this->hidden->session addInput:input]; - } else { + if (![session canAddInput:input]) { return SDL_SetError("Cannot add AVCaptureDeviceInput"); } + [session addInput:input]; - if ([_this->hidden->session canAddOutput:output] ){ - [_this->hidden->session addOutput:output]; - } else { + if (![session canAddOutput:output]) { return SDL_SetError("Cannot add AVCaptureVideoDataOutput"); } + [session addOutput:output]; - [_this->hidden->session commitConfiguration]; - - return 0; -} + [session commitConfiguration]; -static int COREMEDIA_GetDeviceSpec(SDL_CameraDevice *_this, SDL_CameraSpec *spec) -{ - // !!! FIXME: make sure higher level checks spec != NULL - if (spec) { - SDL_copyp(spec, &_this->spec); - return 0; + SDLPrivateCameraData *hidden = [[SDLPrivateCameraData alloc] init]; + if (hidden == nil) { + return SDL_SetError("Cannot create SDLPrivateCameraData"); } - return -1; -} -static int COREMEDIA_StartCamera(SDL_CameraDevice *_this) -{ - [_this->hidden->session startRunning]; - return 0; -} + hidden.session = session; + hidden.delegate = delegate; + hidden.current_sample = NULL; + device->hidden = (struct SDL_PrivateCameraData *)CFBridgingRetain(hidden); -static int COREMEDIA_StopCamera(SDL_CameraDevice *_this) -{ - [_this->hidden->session stopRunning]; - return 0; -} + [session startRunning]; // !!! FIXME: docs say this can block while camera warms up and shouldn't be done on main thread. Maybe push through `queue`? -static int COREMEDIA_AcquireFrame(SDL_CameraDevice *_this, SDL_CameraFrame *frame) -{ - if (CMSimpleQueueGetCount(_this->hidden->frame_queue) > 0) { - CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)CMSimpleQueueDequeue(_this->hidden->frame_queue); - frame->internal = (void *) sampleBuffer; - frame->timestampNS = SDL_GetTicksNS(); - - CVImageBufferRef image = CMSampleBufferGetImageBuffer(sampleBuffer); - const int numPlanes = CVPixelBufferGetPlaneCount(image); - const int planar = CVPixelBufferIsPlanar(image); - - #if DEBUG_CAMERA - const int w = CVPixelBufferGetWidth(image); - const int h = CVPixelBufferGetHeight(image); - const int sz = CVPixelBufferGetDataSize(image); - const int pitch = CVPixelBufferGetBytesPerRow(image); - SDL_Log("CAMERA: buffer planar=%d count:%d %d x %d sz=%d pitch=%d", planar, numPlanes, w, h, sz, pitch); - #endif + CheckCameraPermissions(device); // check right away, in case the process is already granted permission. - CVPixelBufferLockBaseAddress(image, 0); - - if ((planar == 0) && (numPlanes == 0)) { - frame->pitch[0] = CVPixelBufferGetBytesPerRow(image); - frame->data[0] = CVPixelBufferGetBaseAddress(image); - frame->num_planes = 1; - } else { - for (int i = 0; (i < numPlanes) && (i < 3); i++) { - frame->num_planes += 1; - frame->data[i] = CVPixelBufferGetBaseAddressOfPlane(image, i); - frame->pitch[i] = CVPixelBufferGetBytesPerRowOfPlane(image, i); - } - } - - // Unlocked when frame is released - } else { - // no frame - SDL_Delay(20); // TODO fix some delay - } return 0; } -static int COREMEDIA_ReleaseFrame(SDL_CameraDevice *_this, SDL_CameraFrame *frame) +static void COREMEDIA_FreeDeviceHandle(SDL_CameraDevice *device) { - if (frame->internal) { - CMSampleBufferRef sampleBuffer = (CMSampleBufferRef) frame->internal; - CVImageBufferRef image = CMSampleBufferGetImageBuffer(sampleBuffer); - CVPixelBufferUnlockBaseAddress(image, 0); - CFRelease(sampleBuffer); + if (device && device->handle) { + CFBridgingRelease(device->handle); } - - return 0; } -static int COREMEDIA_GetNumFormats(SDL_CameraDevice *_this) +static void GatherCameraSpecs(AVCaptureDevice *device, CameraFormatAddData *add_data) { - AVCaptureDevice *device = GetCameraDeviceByName(_this->dev_name); - if (device) { - // LIST FORMATS - NSMutableOrderedSet *array_formats = [NSMutableOrderedSet new]; - NSArray *formats = [device formats]; - for (AVCaptureDeviceFormat *format in formats) { - // NSLog(@"%@", formats); - CMFormatDescriptionRef formatDescription = [format formatDescription]; - //NSLog(@"%@", formatDescription); - FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription); - NSString *str = fourcc_to_nstring(mediaSubType); - [array_formats addObject:str]; - } - return [array_formats count]; - } - return 0; -} + SDL_zerop(add_data); -static int COREMEDIA_GetFormat(SDL_CameraDevice *_this, int index, Uint32 *format) -{ - AVCaptureDevice *device = GetCameraDeviceByName(_this->dev_name); - if (device) { - // LIST FORMATS - NSMutableOrderedSet *array_formats = [NSMutableOrderedSet new]; - NSArray *formats = [device formats]; - NSString *str; - - for (AVCaptureDeviceFormat *f in formats) { - FourCharCode mediaSubType; - CMFormatDescriptionRef formatDescription; - - formatDescription = [f formatDescription]; - mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription); - str = fourcc_to_nstring(mediaSubType); - [array_formats addObject:str]; + for (AVCaptureDeviceFormat *fmt in device.formats) { + if (CMFormatDescriptionGetMediaType(fmt.formatDescription) != kCMMediaType_Video) { + continue; } - str = array_formats[index]; - *format = nsfourcc_to_sdlformat(str); + const Uint32 sdlfmt = CoreMediaFormatToSDL(CMFormatDescriptionGetMediaSubType(fmt.formatDescription)); + if (sdlfmt == SDL_PIXELFORMAT_UNKNOWN) { + continue; + } - return 0; - } - return -1; -} + const CMVideoDimensions dims = CMVideoFormatDescriptionGetDimensions(fmt.formatDescription); + const int w = (int) dims.width; + const int h = (int) dims.height; + for (AVFrameRateRange *framerate in fmt.videoSupportedFrameRateRanges) { + int rate; -static int COREMEDIA_GetNumFrameSizes(SDL_CameraDevice *_this, Uint32 format) -{ - AVCaptureDevice *device = GetCameraDeviceByName(_this->dev_name); - if (device) { - NSString *fmt = sdlformat_to_nsfourcc(format); - int count = 0; - - NSArray *formats = [device formats]; - for (AVCaptureDeviceFormat *f in formats) { - CMFormatDescriptionRef formatDescription = [f formatDescription]; - FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription); - NSString *str = fourcc_to_nstring(mediaSubType); - - if ([str isEqualToString:fmt]) { - count++; + rate = (int) SDL_ceil((double) framerate.minFrameRate); + if (rate) { + SDL_AddCameraFormat(add_data, sdlfmt, w, h, 1, rate); + } + rate = (int) SDL_floor((double) framerate.maxFrameRate); + if (rate) { + SDL_AddCameraFormat(add_data, sdlfmt, w, h, 1, rate); } } - return count; } - return 0; } -static int COREMEDIA_GetFrameSize(SDL_CameraDevice *_this, Uint32 format, int index, int *width, int *height) +static SDL_bool FindCoreMediaCameraDeviceByUniqueID(SDL_CameraDevice *device, void *userdata) { - AVCaptureDevice *device = GetCameraDeviceByName(_this->dev_name); - if (device) { - NSString *fmt = sdlformat_to_nsfourcc(format); - int count = 0; - - NSArray *formats = [device formats]; - for (AVCaptureDeviceFormat *f in formats) { - CMFormatDescriptionRef formatDescription = [f formatDescription]; - FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription); - NSString *str = fourcc_to_nstring(mediaSubType); - - if ([str isEqualToString:fmt]) { - if (index == count) { - CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDescription); - *width = dim.width; - *height = dim.height; - return 0; - } - count++; - } - } - } - return -1; + NSString *uniqueid = (__bridge NSString *) userdata; + AVCaptureDevice *avdev = (__bridge AVCaptureDevice *) device->handle; + return ([uniqueid isEqualToString:avdev.uniqueID]) ? SDL_TRUE : SDL_FALSE; } -static int COREMEDIA_GetDeviceName(SDL_CameraDeviceID instance_id, char *buf, int size) +static void MaybeAddDevice(AVCaptureDevice *device) { - int index = instance_id - 1; - NSArray *devices = DiscoverCameraDevices(); - if (index < [devices count]) { - AVCaptureDevice *device = devices[index]; - NSString *cameraID = [device localizedName]; - const char *str = [cameraID UTF8String]; - SDL_snprintf(buf, size, "%s", str); - return 0; + if (!device.connected) { + return; // not connected. + } else if (![device hasMediaType:AVMediaTypeVideo]) { + return; // not a camera. + } else if (SDL_FindPhysicalCameraDeviceByCallback(FindCoreMediaCameraDeviceByUniqueID, (__bridge void *) device.uniqueID)) { + return; // already have this one. } - return -1; -} -static int GetNumCameraDevices(void) -{ - NSArray *devices = DiscoverCameraDevices(); - return [devices count]; + CameraFormatAddData add_data; + GatherCameraSpecs(device, &add_data); + if (add_data.num_specs > 0) { + SDL_AddCameraDevice(device.localizedName.UTF8String, add_data.num_specs, add_data.specs, (void *) CFBridgingRetain(device)); + } + SDL_free(add_data.specs); } -static SDL_CameraDeviceID *COREMEDIA_GetDevices(int *count) +static void COREMEDIA_DetectDevices(void) { - // hard-coded list of ID - const int num = GetNumCameraDevices(); - SDL_CameraDeviceID *retval = (SDL_CameraDeviceID *)SDL_calloc((num + 1), sizeof(*ret)); + NSArray *devices = nil; + + if (@available(macOS 10.15, iOS 10, *)) { + // kind of annoying that there isn't a "give me anything that looks like a camera" option, + // so this list will need to be updated when Apple decides to add + // AVCaptureDeviceTypeBuiltInQuadrupleCamera some day. + NSArray *device_types = @[ + #ifdef SDL_PLATFORM_IOS + AVCaptureDeviceTypeBuiltInTelephotoCamera, + AVCaptureDeviceTypeBuiltInDualCamera, + AVCaptureDeviceTypeBuiltInDualWideCamera, + AVCaptureDeviceTypeBuiltInTripleCamera, + AVCaptureDeviceTypeBuiltInUltraWideCamera, + #endif + AVCaptureDeviceTypeBuiltInWideAngleCamera, + AVCaptureDeviceTypeExternalUnknown + ]; - if (retval == NULL) { - *count = 0; - return NULL; - } + AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:device_types + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; - for (int i = 0; i < num; i++) { - retval[i] = i + 1; + devices = discoverySession.devices; + // !!! FIXME: this can use Key Value Observation to get hotplug events. + } else { + // this is deprecated but works back to macOS 10.7; 10.15 added AVCaptureDeviceDiscoverySession as a replacement. + devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; + // !!! FIXME: this can use AVCaptureDeviceWasConnectedNotification and AVCaptureDeviceWasDisconnectedNotification with NSNotificationCenter to get hotplug events. } - retval[num] = 0; - *count = num; - return ret; -} -static void COREMEDIA_DetectDevices(void) -{ + for (AVCaptureDevice *device in devices) { + MaybeAddDevice(device); + } } static void COREMEDIA_Deinitialize(void) { + // !!! FIXME: disable hotplug. } static SDL_bool COREMEDIA_Init(SDL_CameraDriverImpl *impl) { -#ifndef HAVE_COREMEDIA - return SDL_FALSE; -#else impl->DetectDevices = COREMEDIA_DetectDevices; impl->OpenDevice = COREMEDIA_OpenDevice; impl->CloseDevice = COREMEDIA_CloseDevice; - impl->InitDevice = COREMEDIA_InitDevice; - impl->GetDeviceSpec = COREMEDIA_GetDeviceSpec; - impl->StartCamera = COREMEDIA_StartCamera; - impl->StopCamera = COREMEDIA_StopCamera; + impl->WaitDevice = COREMEDIA_WaitDevice; impl->AcquireFrame = COREMEDIA_AcquireFrame; impl->ReleaseFrame = COREMEDIA_ReleaseFrame; - impl->GetNumFormats = COREMEDIA_GetNumFormats; - impl->GetFormat = COREMEDIA_GetFormat; - impl->GetNumFrameSizes = COREMEDIA_GetNumFrameSizes; - impl->GetFrameSize = COREMEDIA_GetFrameSize; - impl->GetDeviceName = COREMEDIA_GetDeviceName; - impl->GetDevices = COREMEDIA_GetDevices; + impl->FreeDeviceHandle = COREMEDIA_FreeDeviceHandle; impl->Deinitialize = COREMEDIA_Deinitialize; + impl->ProvidesOwnCallbackThread = SDL_TRUE; + return SDL_TRUE; -#endif } CameraBootStrap COREMEDIA_bootstrap = { "coremedia", "SDL Apple CoreMedia camera driver", COREMEDIA_Init, SDL_FALSE }; -#endif // HAVE_COREMEDIA - -#endif // SDL_CAMERA_COREMEDIA +#endif // SDL_CAMERA_DRIVER_COREMEDIA diff --git a/src/camera/mediafoundation/SDL_camera_mediafoundation.c b/src/camera/mediafoundation/SDL_camera_mediafoundation.c index 960ab1e543a021..d2d79f94cc0189 100644 --- a/src/camera/mediafoundation/SDL_camera_mediafoundation.c +++ b/src/camera/mediafoundation/SDL_camera_mediafoundation.c @@ -501,8 +501,6 @@ static int MEDIAFOUNDATION_OpenDevice(SDL_CameraDevice *device, const SDL_Camera //PROPVARIANT var; HRESULT ret; -SDL_Log("MEDIAFOUNDATION spec format: %s", SDL_GetPixelFormatName(spec->format)); - #if 0 IMFStreamDescriptor *streamdesc = NULL; IMFPresentationDescriptor *presentdesc = NULL;