Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

start

roryclearacf288034bbd
diff --git a/clearcam.xcodeproj/project.pbxproj b/clearcam.xcodeproj/project.pbxproj
index 64e7ec8..cec8661 100644
--- a/clearcam.xcodeproj/project.pbxproj
+++ b/clearcam.xcodeproj/project.pbxproj
@@ -243,7 +243,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 18.1;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.6;
LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
@@ -298,7 +298,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
- IPHONEOS_DEPLOYMENT_TARGET = 18.1;
+ IPHONEOS_DEPLOYMENT_TARGET = 15.6;
LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
MTL_ENABLE_DEBUG_INFO = NO;
MTL_FAST_MATH = YES;
diff --git a/clearcam/FileServer.h b/clearcam/FileServer.h
new file mode 100644
index 0000000..0fb14f0
--- /dev/null
+++ b/clearcam/FileServer.h
@@ -0,0 +1,7 @@
+#import <Foundation/Foundation.h>
+
+@interface FileServer : NSObject
+
+- (void)start;
+
+@end
diff --git a/clearcam/FileServer.m b/clearcam/FileServer.m
new file mode 100644
index 0000000..75f6835
--- /dev/null
+++ b/clearcam/FileServer.m
@@ -0,0 +1,167 @@
+#import "FileServer.h"
+#include <netinet/in.h>
+#include <sys/socket.h>
+
+@interface FileServer ()
+@property (nonatomic) int serverSocket;
+@end
+
+@implementation FileServer
+
+- (void)start {
+ self.serverSocket = socket(AF_INET, SOCK_STREAM, 0);
+ if (self.serverSocket < 0) {
+ NSLog(@"Error: Unable to create socket.");
+ return;
+ }
+
+ struct sockaddr_in serverAddress;
+ serverAddress.sin_family = AF_INET;
+ serverAddress.sin_port = htons(8080); // Server will run on port 8080
+ serverAddress.sin_addr.s_addr = INADDR_ANY;
+
+ if (bind(self.serverSocket, (struct sockaddr *)&serverAddress, sizeof(serverAddress)) < 0) {
+ NSLog(@"Error: Unable to bind socket.");
+ close(self.serverSocket);
+ return;
+ }
+
+ if (listen(self.serverSocket, 5) < 0) {
+ NSLog(@"Error: Unable to listen on socket.");
+ close(self.serverSocket);
+ return;
+ }
+
+ NSLog(@"FileServer started on port 8080. Access it at http://<your-ip>:8080/");
+ [self handleConnections];
+}
+
+- (void)handleConnections {
+ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+ while (YES) {
+ struct sockaddr_in clientAddress;
+ socklen_t clientAddressLen = sizeof(clientAddress);
+ int clientSocket = accept(self.serverSocket, (struct sockaddr *)&clientAddress, &clientAddressLen);
+ if (clientSocket >= 0) {
+ [self handleClient:clientSocket];
+ }
+ }
+ });
+}
+
+- (void)handleClient:(int)clientSocket {
+ char buffer[1024];
+ ssize_t receivedBytes = recv(clientSocket, buffer, sizeof(buffer) - 1, 0);
+ if (receivedBytes <= 0) {
+ close(clientSocket);
+ return;
+ }
+
+ buffer[receivedBytes] = '\0';
+ NSString *request = [NSString stringWithUTF8String:buffer];
+ NSLog(@"Request: %@", request);
+
+ if ([request containsString:@"GET /?file="]) {
+ NSString *fileName = [self extractFileNameFromRequest:request];
+ [self serveFile:fileName toSocket:clientSocket];
+ } else {
+ [self serveFileListToSocket:clientSocket];
+ }
+
+ close(clientSocket);
+}
+
+- (NSString *)extractFileNameFromRequest:(NSString *)request {
+ NSRange range = [request rangeOfString:@"GET /?file="];
+ if (range.location != NSNotFound) {
+ NSString *partialRequest = [request substringFromIndex:(range.location + range.length)];
+ NSRange endRange = [partialRequest rangeOfString:@" "];
+ if (endRange.location != NSNotFound) {
+ NSString *fileName = [partialRequest substringToIndex:endRange.location];
+ return [fileName stringByRemovingPercentEncoding];
+ }
+ }
+ return nil;
+}
+
+- (void)serveFile:(NSString *)fileName toSocket:(int)clientSocket {
+ NSString *decodedFileName = [fileName stringByRemovingPercentEncoding];
+ NSString *documentsPath = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES) firstObject];
+ NSString *filePath = [documentsPath stringByAppendingPathComponent:decodedFileName];
+
+ NSLog(@"Attempting to serve file at path: %@", filePath);
+
+ NSData *fileData = [NSData dataWithContentsOfFile:filePath];
+ if (!fileData) {
+ NSLog(@"Error: File not found at path: %@", filePath);
+ NSString *response = @"HTTP/1.1 404 Not Found\r\n\r\nFile not found.";
+ send(clientSocket, [response UTF8String], [response length], 0);
+ return;
+ }
+
+ NSString *contentType = [self contentTypeForFileAtPath:filePath];
+ NSString *header = [NSString stringWithFormat:
+ @"HTTP/1.1 200 OK\r\n"
+ "Content-Disposition: inline; filename=\"%@\"\r\n"
+ "Content-Length: %lu\r\n"
+ "Content-Type: %@\r\n"
+ "Content-Transfer-Encoding: binary\r\n"
+ "Accept-Ranges: bytes\r\n\r\n",
+ decodedFileName, (unsigned long)[fileData length], contentType];
+
+ send(clientSocket, [header UTF8String], [header length], 0);
+ send(clientSocket, [fileData bytes], [fileData length], 0);
+}
+
+- (NSString *)contentTypeForFileAtPath:(NSString *)filePath {
+ NSString *extension = [filePath pathExtension].lowercaseString;
+
+ if ([extension isEqualToString:@"mov"]) {
+ return @"video/quicktime";
+ } else if ([extension isEqualToString:@"txt"]) {
+ return @"text/plain";
+ }
+
+ return @"application/octet-stream";
+}
+
+- (void)serveFileListToSocket:(int)clientSocket {
+ NSString *html = @"<html><head><title>Documents Folder</title></head><body>";
+ html = [html stringByAppendingString:@"<h1>Files in Documents Folder:</h1><ul>"];
+
+ NSArray *documentFiles = [self listFilesInDocumentsFolder];
+ for (NSString *fileName in documentFiles) {
+ NSString *encodedFileName = [fileName stringByAddingPercentEncodingWithAllowedCharacters:[NSCharacterSet URLQueryAllowedCharacterSet]];
+ html = [html stringByAppendingFormat:@"<li><a href=\"/?file=%@\">%@</a></li>", encodedFileName, fileName];
+ }
+
+ html = [html stringByAppendingString:@"</ul></body></html>"];
+ NSString *response = [NSString stringWithFormat:
+ @"HTTP/1.1 200 OK\r\n"
+ "Content-Type: text/html\r\n"
+ "Content-Length: %lu\r\n\r\n"
+ "%@", (unsigned long)[html length], html];
+
+ send(clientSocket, [response UTF8String], [response length], 0);
+}
+
+- (NSArray *)listFilesInDocumentsFolder {
+ NSString *documentsPath = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES) firstObject];
+ NSError *error;
+ NSArray *files = [[NSFileManager defaultManager] contentsOfDirectoryAtPath:documentsPath error:&error];
+ if (error) {
+ NSLog(@"Error listing files: %@", error.localizedDescription);
+ return @[];
+ }
+ return files;
+}
+
+- (void)dealloc {
+ close(self.serverSocket);
+}
+
+@end
+
+
+
+
diff --git a/clearcam/Info.plist b/clearcam/Info.plist
index 81ed29b..9a26074 100644
--- a/clearcam/Info.plist
+++ b/clearcam/Info.plist
@@ -2,24 +2,29 @@
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
- <key>UIApplicationSceneManifest</key>
- <dict>
- <key>UIApplicationSupportsMultipleScenes</key>
- <false/>
- <key>UISceneConfigurations</key>
- <dict>
- <key>UIWindowSceneSessionRoleApplication</key>
- <array>
- <dict>
- <key>UISceneConfigurationName</key>
- <string>Default Configuration</string>
- <key>UISceneDelegateClassName</key>
- <string>SceneDelegate</string>
- <key>UISceneStoryboardFile</key>
- <string>Main</string>
- </dict>
- </array>
- </dict>
- </dict>
+ <key>UIFileSharingEnabled</key>
+ <true/>
+ <key>NSCameraUsageDescription</key>
+ <string>We need access to your camera to take photos and videos.</string>
+ <key>UIApplicationSceneManifest</key>
+ <dict>
+ <key>UIApplicationSupportsMultipleScenes</key>
+ <false/>
+ <key>UISceneConfigurations</key>
+ <dict>
+ <key>UIWindowSceneSessionRoleApplication</key>
+ <array>
+ <dict>
+ <key>UISceneConfigurationName</key>
+ <string>Default Configuration</string>
+ <key>UISceneDelegateClassName</key>
+ <string>SceneDelegate</string>
+ <key>UISceneStoryboardFile</key>
+ <string>Main</string>
+ </dict>
+ </array>
+ </dict>
+ </dict>
</dict>
</plist>
+
diff --git a/clearcam/SceneDelegate.h b/clearcam/SceneDelegate.h
index 47a20a5..4813059 100644
--- a/clearcam/SceneDelegate.h
+++ b/clearcam/SceneDelegate.h
@@ -1,10 +1,3 @@
-//
-// SceneDelegate.h
-// clearcam
-//
-// Created by Rory Clear on 06/01/2025.
-//
-
#import <UIKit/UIKit.h>
@interface SceneDelegate : UIResponder <UIWindowSceneDelegate>
@@ -13,3 +6,4 @@
@end
+
diff --git a/clearcam/SceneDelegate.m b/clearcam/SceneDelegate.m
index d34d8c3..48c214e 100644
--- a/clearcam/SceneDelegate.m
+++ b/clearcam/SceneDelegate.m
@@ -1,10 +1,3 @@
-//
-// SceneDelegate.m
-// clearcam
-//
-// Created by Rory Clear on 06/01/2025.
-//
-
#import "SceneDelegate.h"
@interface SceneDelegate ()
@@ -55,3 +48,4 @@
@end
+
diff --git a/clearcam/ViewController.h b/clearcam/ViewController.h
index aeedeee..efa26eb 100644
--- a/clearcam/ViewController.h
+++ b/clearcam/ViewController.h
@@ -1,10 +1,3 @@
-//
-// ViewController.h
-// clearcam
-//
-// Created by Rory Clear on 06/01/2025.
-//
-
#import <UIKit/UIKit.h>
@interface ViewController : UIViewController
@@ -12,3 +5,4 @@
@end
+
diff --git a/clearcam/ViewController.m b/clearcam/ViewController.m
index b9f3788..8cb3b6e 100644
--- a/clearcam/ViewController.m
+++ b/clearcam/ViewController.m
@@ -1,22 +1,370 @@
-//
-// ViewController.m
-// clearcam
-//
-// Created by Rory Clear on 06/01/2025.
-//
-
#import "ViewController.h"
+#import <AVFoundation/AVFoundation.h>
+#import <Metal/Metal.h>
+#import "Yolo.h"
+#import "FileServer.h"
@interface ViewController ()
+@property (nonatomic, strong) AVCaptureSession *captureSession;
+@property (nonatomic, strong) AVCaptureVideoPreviewLayer *previewLayer;
+@property (nonatomic, strong) UILabel *fpsLabel;
+@property (nonatomic, assign) CFTimeInterval lastFrameTime;
+@property (nonatomic, assign) NSUInteger frameCount;
+@property (nonatomic, strong) Yolo *yolo;
+@property (nonatomic, strong) CIContext *ciContext;
+
+@property (atomic, assign) BOOL isProcessing;
+@property (nonatomic, assign) BOOL isRecording;
+@property (nonatomic, assign) CMTime startTime;
+@property (nonatomic, assign) CMTime currentTime;
+@property (nonatomic, strong) AVAssetWriter *assetWriter;
+@property (nonatomic, strong) AVAssetWriterInput *videoWriterInput;
+@property (nonatomic, strong) AVAssetWriterInputPixelBufferAdaptor *adaptor;
+@property (nonatomic, strong) FileServer *fileServer;
+
@end
@implementation ViewController
+NSMutableDictionary *classColorMap;
+
- (void)viewDidLoad {
[super viewDidLoad];
- // Do any additional setup after loading the view.
+ self.ciContext = [CIContext context];
+ self.yolo = [[Yolo alloc] init];
+ [self setupCamera];
+ [self setupFPSLabel];
+ self.fileServer = [[FileServer alloc] init];
+ [self.fileServer start];
+
+ [[NSNotificationCenter defaultCenter] addObserver:self
+ selector:@selector(handleDeviceOrientationChange)
+ name:UIDeviceOrientationDidChangeNotification
+ object:nil];
+
+ [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
+ [self handleDeviceOrientationChange];
+}
+
+- (void)dealloc {
+ [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
+}
+
+- (void)handleDeviceOrientationChange {
+ UIDeviceOrientation deviceOrientation = [[UIDevice currentDevice] orientation];
+ AVCaptureVideoOrientation videoOrientation;
+
+ switch (deviceOrientation) {
+ case UIDeviceOrientationLandscapeLeft:
+ videoOrientation = AVCaptureVideoOrientationLandscapeRight;
+ break;
+ case UIDeviceOrientationLandscapeRight:
+ videoOrientation = AVCaptureVideoOrientationLandscapeLeft;
+ break;
+ case UIDeviceOrientationPortrait:
+ videoOrientation = AVCaptureVideoOrientationPortrait;
+ break;
+ case UIDeviceOrientationPortraitUpsideDown:
+ videoOrientation = AVCaptureVideoOrientationPortraitUpsideDown;
+ break;
+ default:
+ videoOrientation = AVCaptureVideoOrientationLandscapeRight;
+ break;
+ }
+
+ if (self.previewLayer.connection.isVideoOrientationSupported) {
+ self.previewLayer.connection.videoOrientation = videoOrientation;
+ }
+}
+
+- (void)setupCamera {
+ self.captureSession = [[AVCaptureSession alloc] init];
+ self.captureSession.sessionPreset = AVCaptureSessionPresetPhoto;
+ AVCaptureDevice *device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
+ NSError *error = nil;
+ AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
+ if (!input) {
+ NSLog(@"Error setting up camera input: %@", error.localizedDescription);
+ return;
+ }
+ [self.captureSession addInput:input];
+
+ AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init];
+ output.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)};
+ output.alwaysDiscardsLateVideoFrames = YES;
+ [output setSampleBufferDelegate:self queue:dispatch_get_main_queue()];
+ [self.captureSession addOutput:output];
+ self.previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:self.captureSession];
+ self.previewLayer.videoGravity = AVLayerVideoGravityResizeAspect;
+ [self.view.layer addSublayer:self.previewLayer];
+ [self.captureSession startRunning];
+ [self startNewRecording];
+}
+
+- (void)startNewRecording {
+ NSDateFormatter *formatter = [[NSDateFormatter alloc] init];
+ [formatter setDateFormat:@"yyyy-MM-dd_HH:mm:ss"];
+ NSString *timestamp = [formatter stringFromDate:[NSDate date]];
+
+ NSURL *documentsDirectory = [[[NSFileManager defaultManager] URLsForDirectory:NSDocumentDirectory inDomains:NSUserDomainMask] firstObject];
+ NSURL *outputURL = [documentsDirectory URLByAppendingPathComponent:[NSString stringWithFormat:@"output_%@.mp4", timestamp]];
+
+ NSError *error = nil;
+ self.assetWriter = [AVAssetWriter assetWriterWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error];
+ if (error) {
+ NSLog(@"Error creating asset writer: %@", error.localizedDescription);
+ return;
+ }
+
+ NSDictionary *videoSettings = @{
+ AVVideoCodecKey: AVVideoCodecTypeH264,
+ AVVideoWidthKey: @640,
+ AVVideoHeightKey: @480,
+ AVVideoScalingModeKey: AVVideoScalingModeResizeAspectFill
+ };
+ self.videoWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:videoSettings];
+ self.videoWriterInput.expectsMediaDataInRealTime = YES;
+
+ NSDictionary *sourcePixelBufferAttributes = @{
+ (NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA),
+ (NSString *)kCVPixelBufferWidthKey: @640,
+ (NSString *)kCVPixelBufferHeightKey: @480
+ };
+ self.adaptor = [AVAssetWriterInputPixelBufferAdaptor assetWriterInputPixelBufferAdaptorWithAssetWriterInput:self.videoWriterInput sourcePixelBufferAttributes:sourcePixelBufferAttributes];
+
+ if ([self.assetWriter canAddInput:self.videoWriterInput]) {
+ [self.assetWriter addInput:self.videoWriterInput];
+ } else {
+ NSLog(@"Cannot add video writer input");
+ return;
+ }
+
+ self.startTime = kCMTimeInvalid;
+ self.currentTime = kCMTimeZero;
+ self.isRecording = YES;
+
+ [self.assetWriter startWriting];
+ [self.assetWriter startSessionAtSourceTime:kCMTimeZero];
+
+ NSLog(@"Started new recording");
+}
+
+- (void)drawSquareWithTopLeftX:(CGFloat)xOrigin topLeftY:(CGFloat)yOrigin bottomRightX:(CGFloat)bottomRightX bottomRightY:(CGFloat)bottomRightY classIndex:(int)classIndex aspectRatio:(float)aspectRatio {
+ CGFloat leftEdgeX = (self.view.bounds.size.width - (self.view.bounds.size.height * aspectRatio)) / 2;
+ CGFloat scaledXOrigin, scaledYOrigin, scaledWidth, scaledHeight;
+
+ if (self.view.bounds.size.width < self.view.bounds.size.height) { //portrait
+ scaledYOrigin = (self.view.bounds.size.height / 2) - (self.view.bounds.size.width * aspectRatio / 2);
+ scaledYOrigin += (yOrigin / self.yolo.yolo_res) * (self.view.bounds.size.width * aspectRatio);
+ scaledHeight = ((bottomRightY - yOrigin) / self.yolo.yolo_res) * (self.view.bounds.size.width * aspectRatio);
+ scaledXOrigin = (xOrigin / (self.yolo.yolo_res / aspectRatio)) * self.view.bounds.size.width;
+ scaledWidth = self.view.bounds.size.width * (bottomRightX - xOrigin) / (self.yolo.yolo_res / aspectRatio);
+ } else {
+ scaledXOrigin = leftEdgeX + (xOrigin * aspectRatio / self.yolo.yolo_res) * self.view.bounds.size.height;
+ scaledYOrigin = (aspectRatio * yOrigin / self.yolo.yolo_res) * self.view.bounds.size.height;
+ scaledWidth = (bottomRightX - xOrigin) * (aspectRatio * self.view.bounds.size.height / self.yolo.yolo_res);
+ scaledHeight = (bottomRightY - yOrigin) * (aspectRatio * self.view.bounds.size.height / self.yolo.yolo_res);
+ }
+
+ UIColor *color = self.yolo.yolo_classes[classIndex][1];
+ CAShapeLayer *squareLayer = [CAShapeLayer layer];
+ squareLayer.name = @"rectangleLayer";
+ squareLayer.strokeColor = color.CGColor;
+ squareLayer.lineWidth = 2.0;
+ squareLayer.fillColor = [UIColor clearColor].CGColor;
+ squareLayer.path = [UIBezierPath bezierPathWithRect:CGRectMake(scaledXOrigin, scaledYOrigin, scaledWidth, scaledHeight)].CGPath;
+
+ [self.view.layer addSublayer:squareLayer];
+
+ NSString *className = self.yolo.yolo_classes[classIndex][0];
+ NSDictionary *textAttributes = @{NSFontAttributeName: [UIFont systemFontOfSize:12],
+ NSForegroundColorAttributeName: [UIColor whiteColor]};
+ NSString *labelText = [className lowercaseString];
+ CGSize textSize = [labelText sizeWithAttributes:textAttributes];
+
+ CGFloat labelX = scaledXOrigin - 2;
+ CGFloat labelY = scaledYOrigin - textSize.height - 2;
+
+ UILabel *label = [[UILabel alloc] initWithFrame:CGRectMake(labelX, labelY, textSize.width + 4, textSize.height + 2)];
+ label.backgroundColor = color;
+ label.textColor = [UIColor whiteColor];
+ label.font = [UIFont systemFontOfSize:12];
+ label.text = labelText;
+
+ [self.view addSubview:label];
+}
+
+- (void)resetSquares {
+ NSMutableArray *layersToRemove = [NSMutableArray array];
+ NSMutableArray *labelsToRemove = [NSMutableArray array];
+
+ for (CALayer *layer in self.view.layer.sublayers) {
+ if ([layer.name isEqualToString:@"rectangleLayer"]) {
+ [layersToRemove addObject:layer];
+ }
+ }
+
+ for (UIView *subview in self.view.subviews) {
+ if ([subview isKindOfClass:[UILabel class]] && subview != self.fpsLabel) {
+ [labelsToRemove addObject:subview];
+ }
+ }
+
+ for (CALayer *layer in layersToRemove) {
+ [layer removeFromSuperlayer];
+ }
+
+ for (UIView *label in labelsToRemove) {
+ [label removeFromSuperview];
+ }
+}
+
+- (void)setupFPSLabel {
+ self.fpsLabel = [[UILabel alloc] initWithFrame:CGRectMake(10, 30, 150, 30)];
+ self.fpsLabel.backgroundColor = [UIColor colorWithWhite:0 alpha:0.5];
+ self.fpsLabel.textColor = [UIColor whiteColor];
+ self.fpsLabel.font = [UIFont boldSystemFontOfSize:18];
+ self.fpsLabel.text = @"FPS: 0";
+ [self.view addSubview:self.fpsLabel];
}
+- (void)viewWillLayoutSubviews {
+ [super viewWillLayoutSubviews];
+ self.previewLayer.frame = [self frameForCurrentOrientation];
+}
+
+- (CGRect)frameForCurrentOrientation {
+ CGFloat width = self.view.bounds.size.width;
+ CGFloat height = self.view.bounds.size.height;
+ return CGRectMake(0, 0, width, height);
+}
+
+- (CGImageRef)cropAndResizeCGImage:(CGImageRef)image toSize:(CGSize)size {
+ CGFloat originalWidth = CGImageGetWidth(image);
+ CGFloat originalHeight = CGImageGetHeight(image);
+ CGFloat sideLength = MIN(originalWidth, originalHeight);
+ CGRect cropRect = CGRectMake((originalWidth - sideLength) / 2.0,
+ (originalHeight - sideLength) / 2.0,
+ sideLength,
+ sideLength);
+
+ CGImageRef croppedImageRef = CGImageCreateWithImageInRect(image, cropRect);
+ UIGraphicsBeginImageContextWithOptions(size, NO, 0.0);
+ [[UIImage imageWithCGImage:croppedImageRef] drawInRect:CGRectMake(0, 0, size.width, size.height)];
+ UIImage *resizedImage = UIGraphicsGetImageFromCurrentImageContext();
+ UIGraphicsEndImageContext();
+ CGImageRelease(croppedImageRef);
+
+ return resizedImage.CGImage;
+}
+
+- (void)finishRecording {
+ if (self.isRecording && self.assetWriter.status == AVAssetWriterStatusWriting) {
+ self.isRecording = NO;
+ [self.videoWriterInput markAsFinished];
+ [self.assetWriter finishWritingWithCompletionHandler:^{
+ NSLog(@"Finished recording and saving file");
+ [self startNewRecording];
+ }];
+ } else {
+ NSLog(@"Cannot finish writing. Asset writer status: %ld", (long)self.assetWriter.status);
+ }
+}
+- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {
+ if (self.isRecording && self.assetWriter.status == AVAssetWriterStatusWriting) {
+ CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
+ if (CMTIME_IS_INVALID(self.startTime)) {
+ self.startTime = timestamp;
+ self.currentTime = kCMTimeZero;
+ } else {
+ self.currentTime = CMTimeSubtract(timestamp, self.startTime);
+ }
+
+ if (self.videoWriterInput.readyForMoreMediaData) {
+ CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+
+ BOOL success = NO;
+ int retryCount = 3; //TODO, will this stop the crash?
+ while (!success && retryCount > 0) {
+ success = [self.adaptor appendPixelBuffer:pixelBuffer withPresentationTime:self.currentTime];
+ if (!success) {
+ NSLog(@"Error appending pixel buffer (remaining retries: %d): %@", retryCount - 1, self.assetWriter.error.localizedDescription);
+ retryCount--;
+ [NSThread sleepForTimeInterval:0.01];
+ }
+ }
+ }
+ NSTimeInterval elapsedTime = CMTimeGetSeconds(self.currentTime);
+ if (elapsedTime >= 60.0) {
+ [self finishRecording];
+ }
+ }
+
+ AVCaptureVideoOrientation videoOrientation = self.previewLayer.connection.videoOrientation;
+ CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+ CIImage *ciImage = [CIImage imageWithCVPixelBuffer:imageBuffer];
+
+ if (self.isProcessing) {
+ return;
+ }
+
+ self.isProcessing = YES;
+
+ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
+ size_t width = CVPixelBufferGetWidth(imageBuffer);
+ size_t height = CVPixelBufferGetHeight(imageBuffer);
+ CGFloat aspect_ratio = (CGFloat)width / (CGFloat)height;
+ CGFloat targetWidth = self.yolo.yolo_res;
+ CGFloat aspectRatio = (CGFloat)width / (CGFloat)height;
+ CGSize targetSize = CGSizeMake(targetWidth, targetWidth / aspectRatio);
+
+ CGFloat scaleX = targetSize.width / width;
+ CGFloat scaleY = targetSize.height / height;
+ CIImage *resizedImage = [ciImage imageByApplyingTransform:CGAffineTransformMakeScale(scaleX, scaleY)];
+
+ CGRect cropRect = CGRectMake(0, 0, targetSize.width, targetSize.height);
+ CIImage *croppedImage = [resizedImage imageByCroppingToRect:cropRect];
+
+ CGImageRef cgImage = [self.ciContext createCGImage:croppedImage fromRect:cropRect];
+
+ NSArray *output = [self.yolo yolo_infer:cgImage withOrientation:videoOrientation];
+ CGImageRelease(cgImage);
+
+ __weak typeof(self) weak_self = self;
+ dispatch_async(dispatch_get_main_queue(), ^{
+ [weak_self resetSquares];
+ for (int i = 0; i < output.count; i++) {
+ [weak_self drawSquareWithTopLeftX:[output[i][0] floatValue]
+ topLeftY:[output[i][1] floatValue]
+ bottomRightX:[output[i][2] floatValue]
+ bottomRightY:[output[i][3] floatValue]
+ classIndex:[output[i][4] intValue]
+ aspectRatio:aspect_ratio];
+ }
+ [weak_self updateFPS];
+ weak_self.isProcessing = NO;
+ });
+ });
+}
+
+- (void)updateFPS {
+ CFTimeInterval currentTime = CACurrentMediaTime();
+ if (self.lastFrameTime > 0) {
+ CFTimeInterval deltaTime = currentTime - self.lastFrameTime;
+ self.frameCount++;
+ if (deltaTime >= 1.0) {
+ CGFloat fps = self.frameCount / deltaTime;
+ self.fpsLabel.text = [NSString stringWithFormat:@"FPS: %.1f", fps];
+ self.frameCount = 0;
+ self.lastFrameTime = currentTime;
+ }
+ } else {
+ self.lastFrameTime = currentTime;
+ }
+}
@end
+
+
+
diff --git a/clearcam/Yolo.h b/clearcam/Yolo.h
new file mode 100644
index 0000000..36afde9
--- /dev/null
+++ b/clearcam/Yolo.h
@@ -0,0 +1,35 @@
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+#import <AVFoundation/AVFoundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface Yolo : NSObject
+
+@property (nonatomic, strong) id<MTLDevice> device;
+@property (nonatomic, strong) NSMutableDictionary<NSString *, id> *pipeline_states;
+@property (nonatomic, strong) NSMutableDictionary<NSString *, id> *buffers;
+@property (nonatomic, strong) id<MTLCommandQueue> mtl_queue;
+@property (nonatomic, strong) NSMutableArray<id<MTLCommandBuffer>> *mtl_buffers_in_flight;
+@property (nonatomic, assign) int yolo_res;
+@property (nonatomic, strong) NSArray *yolo_classes;
+@property (nonatomic, assign) CFDataRef data;
+@property (nonatomic, strong) NSMutableDictionary *_h;
+@property (nonatomic, strong) NSMutableArray *_q;
+@property (nonatomic, strong) NSString *input_buffer;
+@property (nonatomic, strong) NSString *output_buffer;
+- (NSArray *)yolo_infer:(CGImageRef)cgImage withOrientation:(AVCaptureVideoOrientation)orientation;
+
+// Initializer
+- (instancetype)init;
+
+- (CGFloat)intersectionBetweenBox:(NSArray *)box1 andBox:(NSArray *)box2;
+- (CGFloat)unionBetweenBox:(NSArray *)box1 andBox:(NSArray *)box2;
+- (CGFloat)iouBetweenBox:(NSArray *)box1 andBox:(NSArray *)box2;
+- (NSMutableDictionary<NSString *, id> *)extractValues:(NSString *)x;
+- (NSArray *)processOutput:(const float *)output outputLength:(int)outputLength imgWidth:(float)imgWidth imgHeight:(float)imgHeight;
+
+@end
+
+NS_ASSUME_NONNULL_END
+
diff --git a/clearcam/Yolo.m b/clearcam/Yolo.m
new file mode 100644
index 0000000..aaffdfd
--- /dev/null
+++ b/clearcam/Yolo.m
@@ -0,0 +1,394 @@
+#import "Yolo.h"
+#import <Metal/Metal.h>
+#import <UIKit/UIKit.h>
+#import <sys/utsname.h>
+
+@implementation Yolo
+
+id<MTLDevice> device;
+NSMutableDictionary<NSString *, id> *pipeline_states;
+NSMutableDictionary<NSString *, id> *buffers;
+id<MTLCommandQueue> mtl_queue;
+NSMutableArray<id<MTLCommandBuffer>> *mtl_buffers_in_flight;
+int yolo_res;
+NSArray *yolo_classes;
+CFDataRef data;
+NSMutableDictionary *_h;
+NSMutableArray *_q;
+NSString *input_buffer;
+NSString *output_buffer;
+
+- (instancetype)init {
+ self = [super init];
+ self.device = MTLCreateSystemDefaultDevice();
+ self.pipeline_states = [[NSMutableDictionary alloc] init];
+ self.buffers = [[NSMutableDictionary alloc] init];
+ self.mtl_queue = [self.device newCommandQueueWithMaxCommandBufferCount:1024];
+ self.mtl_buffers_in_flight = [[NSMutableArray alloc] init];
+ self.yolo_res = 640;
+ self.yolo_classes = @[
+ @[@"person", [UIColor redColor]],@[@"bicycle", [UIColor greenColor]],@[@"car", [UIColor blueColor]],
+ @[@"motorcycle", [UIColor cyanColor]],@[@"airplane", [UIColor magentaColor]],@[@"bus", [UIColor yellowColor]],
+ @[@"train", [UIColor orangeColor]],@[@"truck", [UIColor purpleColor]],@[@"boat", [UIColor brownColor]],
+ @[@"traffic light", [UIColor colorWithRed:0.5 green:0.7 blue:0.2 alpha:1.0]],@[@"fire hydrant", [UIColor colorWithRed:0.8 green:0.1 blue:0.1 alpha:1.0]],
+ @[@"stop sign", [UIColor colorWithRed:0.3 green:0.3 blue:0.8 alpha:1.0]],@[@"parking meter", [UIColor colorWithRed:0.7 green:0.5 blue:0.3 alpha:1.0]],
+ @[@"bench", [UIColor colorWithRed:0.4 green:0.4 blue:0.2 alpha:1.0]],@[@"bird", [UIColor colorWithRed:0.1 green:0.5 blue:0.9 alpha:1.0]],
+ @[@"cat", [UIColor colorWithRed:0.8 green:0.2 blue:0.6 alpha:1.0]],@[@"dog", [UIColor colorWithRed:0.9 green:0.3 blue:0.3 alpha:1.0]],
+ @[@"horse", [UIColor colorWithRed:0.2 green:0.6 blue:0.7 alpha:1.0]],@[@"sheep", [UIColor colorWithRed:0.7 green:0.3 blue:0.5 alpha:1.0]],
+ @[@"cow", [UIColor colorWithRed:0.4 green:0.8 blue:0.4 alpha:1.0]],@[@"elephant", [UIColor colorWithRed:0.3 green:0.4 blue:0.9 alpha:1.0]],
+ @[@"bear", [UIColor colorWithRed:0.6 green:0.2 blue:0.8 alpha:1.0]],@[@"zebra", [UIColor colorWithRed:0.8 green:0.5 blue:0.2 alpha:1.0]],
+ @[@"giraffe", [UIColor colorWithRed:0.5 green:0.9 blue:0.1 alpha:1.0]],@[@"backpack", [UIColor colorWithRed:0.3 green:0.7 blue:0.4 alpha:1.0]],
+ @[@"umbrella", [UIColor colorWithRed:0.4 green:0.6 blue:0.9 alpha:1.0]],@[@"handbag", [UIColor colorWithRed:0.9 green:0.2 blue:0.5 alpha:1.0]],
+ @[@"tie", [UIColor colorWithRed:0.5 green:0.3 blue:0.7 alpha:1.0]],@[@"suitcase", [UIColor colorWithRed:0.6 green:0.7 blue:0.2 alpha:1.0]],
+ @[@"frisbee", [UIColor colorWithRed:0.7 green:0.2 blue:0.4 alpha:1.0]],@[@"skis", [UIColor colorWithRed:0.3 green:0.9 blue:0.3 alpha:1.0]],
+ @[@"snowboard", [UIColor colorWithRed:0.8 green:0.1 blue:0.6 alpha:1.0]],@[@"sports ball", [UIColor colorWithRed:0.4 green:0.3 blue:0.8 alpha:1.0]],
+ @[@"kite", [UIColor colorWithRed:0.2 green:0.5 blue:0.7 alpha:1.0]],@[@"baseball bat", [UIColor colorWithRed:0.6 green:0.4 blue:0.2 alpha:1.0]],
+ @[@"baseball glove", [UIColor colorWithRed:0.7 green:0.1 blue:0.4 alpha:1.0]],@[@"skateboard", [UIColor colorWithRed:0.5 green:0.8 blue:0.5 alpha:1.0]],
+ @[@"surfboard", [UIColor colorWithRed:0.8 green:0.3 blue:0.6 alpha:1.0]],@[@"tennis racket", [UIColor colorWithRed:0.2 green:0.7 blue:0.9 alpha:1.0]],
+ @[@"bottle", [UIColor colorWithRed:0.9 green:0.2 blue:0.3 alpha:1.0]],@[@"wine glass", [UIColor colorWithRed:0.6 green:0.6 blue:0.3 alpha:1.0]],
+ @[@"cup", [UIColor colorWithRed:0.3 green:0.4 blue:0.9 alpha:1.0]],@[@"fork", [UIColor colorWithRed:0.4 green:0.7 blue:0.2 alpha:1.0]],
+ @[@"knife", [UIColor colorWithRed:0.8 green:0.2 blue:0.5 alpha:1.0]],@[@"spoon", [UIColor colorWithRed:0.6 green:0.3 blue:0.7 alpha:1.0]],
+ @[@"bowl", [UIColor colorWithRed:0.2 green:0.8 blue:0.4 alpha:1.0]],@[@"banana", [UIColor colorWithRed:0.7 green:0.7 blue:0.1 alpha:1.0]],
+ @[@"apple", [UIColor colorWithRed:0.9 green:0.1 blue:0.4 alpha:1.0]],@[@"sandwich", [UIColor colorWithRed:0.4 green:0.5 blue:0.8 alpha:1.0]],
+ @[@"orange", [UIColor colorWithRed:0.8 green:0.6 blue:0.2 alpha:1.0]],@[@"broccoli", [UIColor colorWithRed:0.3 green:0.8 blue:0.3 alpha:1.0]],
+ @[@"carrot", [UIColor colorWithRed:0.7 green:0.2 blue:0.6 alpha:1.0]],@[@"hot dog", [UIColor colorWithRed:0.9 green:0.3 blue:0.5 alpha:1.0]],
+ @[@"pizza", [UIColor colorWithRed:0.5 green:0.3 blue:0.8 alpha:1.0]],@[@"donut", [UIColor colorWithRed:0.8 green:0.1 blue:0.4 alpha:1.0]],
+ @[@"cake", [UIColor colorWithRed:0.7 green:0.5 blue:0.1 alpha:1.0]],@[@"chair", [UIColor colorWithRed:0.6 green:0.2 blue:0.4 alpha:1.0]],
+ @[@"couch", [UIColor colorWithRed:0.4 green:0.6 blue:0.2 alpha:1.0]],@[@"potted plant", [UIColor colorWithRed:0.8 green:0.4 blue:0.5 alpha:1.0]],
+ @[@"bed", [UIColor colorWithRed:0.3 green:0.7 blue:0.7 alpha:1.0]],@[@"dining table", [UIColor colorWithRed:0.5 green:0.8 blue:0.3 alpha:1.0]],
+ @[@"toilet", [UIColor colorWithRed:0.7 green:0.4 blue:0.6 alpha:1.0]],@[@"tv", [UIColor colorWithRed:0.9 green:0.5 blue:0.2 alpha:1.0]],
+ @[@"laptop", [UIColor colorWithRed:0.6 green:0.3 blue:0.7 alpha:1.0]],@[@"mouse", [UIColor colorWithRed:0.2 green:0.9 blue:0.5 alpha:1.0]],
+ @[@"remote", [UIColor colorWithRed:0.8 green:0.4 blue:0.3 alpha:1.0]],@[@"keyboard", [UIColor colorWithRed:0.3 green:0.6 blue:0.8 alpha:1.0]],
+ @[@"cell phone", [UIColor colorWithRed:0.7 green:0.3 blue:0.9 alpha:1.0]],@[@"microwave", [UIColor colorWithRed:0.4 green:0.9 blue:0.4 alpha:1.0]],
+ @[@"oven", [UIColor colorWithRed:0.5 green:0.7 blue:0.2 alpha:1.0]],@[@"toaster", [UIColor colorWithRed:0.9 green:0.2 blue:0.3 alpha:1.0]],
+ @[@"sink", [UIColor colorWithRed:0.6 green:0.8 blue:0.3 alpha:1.0]],@[@"refrigerator", [UIColor colorWithRed:0.8 green:0.4 blue:0.7 alpha:1.0]],
+ @[@"book", [UIColor colorWithRed:0.3 green:0.5 blue:0.9 alpha:1.0]],@[@"clock", [UIColor colorWithRed:0.7 green:0.7 blue:0.2 alpha:1.0]],
+ @[@"vase", [UIColor colorWithRed:0.9 green:0.4 blue:0.5 alpha:1.0]],@[@"scissors", [UIColor colorWithRed:0.2 green:0.7 blue:0.8 alpha:1.0]],
+ @[@"teddy bear", [UIColor colorWithRed:0.6 green:0.3 blue:0.9 alpha:1.0]],@[@"hair drier", [UIColor colorWithRed:0.8 green:0.2 blue:0.3 alpha:1.0]],
+ @[@"toothbrush", [UIColor colorWithRed:0.4 green:0.7 blue:0.6 alpha:1.0]]
+ ];
+
+ NSString *documentsPath = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES) firstObject];
+ NSString *path = @"batch_req_%dx%d";
+
+ struct utsname systemInfo;
+ uname(&systemInfo);
+ NSString *deviceModel = [NSString stringWithCString:systemInfo.machine encoding:NSUTF8StringEncoding];
+
+ NSString *urlPath = @"https://raw.githubusercontent.com/roryclear/yolov8-tinygrad-ios/main/batch_req_%dx%d";
+ if ([deviceModel isEqualToString:@"iPhone8,4"]) { //IPHONE SE1
+ path = @"batch_req_se1_%dx%d";
+ urlPath = @"https://raw.githubusercontent.com/roryclear/yolov8-tinygrad-ios/main/batch_req_se1_%dx%d";
+ }
+ NSString *filePath = [documentsPath stringByAppendingPathComponent:[NSString stringWithFormat:path, self.yolo_res, self.yolo_res]];
+ if (![[NSFileManager defaultManager] fileExistsAtPath:filePath]) {
+ NSURL *url = [NSURL URLWithString:[NSString stringWithFormat:urlPath, self.yolo_res, self.yolo_res]];
+ NSData *data = [NSData dataWithContentsOfURL:url];
+ [data writeToFile:filePath atomically:YES];
+ }
+
+ NSData *ns_data = [NSData dataWithContentsOfFile:filePath];
+ self.data = CFDataCreate(NULL, [ns_data bytes], [ns_data length]);
+
+ const UInt8 *bytes = CFDataGetBytePtr(self.data);
+ NSInteger length = CFDataGetLength(self.data);
+ NSData *range_data;
+ self._h = [[NSMutableDictionary alloc] init];
+ NSInteger ptr = 0;
+ NSString *string_data;
+ NSMutableString *datahash = [NSMutableString stringWithCapacity:0x40];
+ while (ptr < length) {
+ NSData *slicedData = [NSData dataWithBytes:bytes + ptr + 0x20 length:0x28 - 0x20];
+ uint64_t datalen = 0;
+ [slicedData getBytes:&datalen length:sizeof(datalen)];
+ datalen = CFSwapInt64LittleToHost(datalen);
+ const UInt8 *datahash_bytes = bytes + ptr;
+ datahash = [NSMutableString stringWithCapacity:0x40];
+ for (int i = 0; i < 0x20; i++) {
+ [datahash appendFormat:@"%02x", datahash_bytes[i]];
+ }
+ range_data = [NSData dataWithBytes:bytes + (ptr + 0x28) length:datalen];
+ self._h[datahash] = range_data;
+ ptr += 0x28 + datalen;
+ }
+ CFRelease(self.data);
+ string_data = [[NSString alloc] initWithData:range_data encoding:NSUTF8StringEncoding];
+ self._q = [NSMutableArray array];
+ NSMutableArray *_q_exec = [NSMutableArray array];
+ NSArray *ops = @[@"BufferAlloc", @"CopyIn", @"ProgramAlloc",@"ProgramExec",@"CopyOut"];
+ NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern:[NSString stringWithFormat:@"(%@)\\(", [ops componentsJoinedByString:@"|"]] options:0 error:nil];
+ __block NSInteger lastIndex = 0;
+ [regex enumerateMatchesInString:string_data options:0 range:NSMakeRange(0, string_data.length) usingBlock:^(NSTextCheckingResult *match, NSMatchingFlags flags, BOOL *stop) {
+ [self._q addObject:[self extractValues:([[string_data substringWithRange:NSMakeRange(lastIndex, match.range.location - lastIndex)] stringByTrimmingCharactersInSet:[NSCharacterSet characterSetWithCharactersInString:@", "]])]];
+ lastIndex = match.range.location;
+ }];
+ [self._q addObject:[self extractValues:([[string_data substringFromIndex:lastIndex] stringByTrimmingCharactersInSet:[NSCharacterSet characterSetWithCharactersInString:@", "]])]];
+ for (NSMutableDictionary *values in self._q) {
+ if ([values[@"op"] isEqualToString:@"BufferAlloc"]) {
+ [self.buffers setObject:[self.device newBufferWithLength:[values[@"size"][0] intValue] options:MTLResourceStorageModeShared] forKey:values[@"buffer_num"][0]];
+ } else if ([values[@"op"] isEqualToString:@"CopyIn"]) {
+ id<MTLBuffer> buffer = self.buffers[values[@"buffer_num"][0]];
+ NSData *data = self._h[values[@"datahash"][0]];
+ memcpy(buffer.contents, data.bytes, data.length);
+ self.input_buffer = values[@"buffer_num"][0];
+ } else if ([values[@"op"] isEqualToString:@"ProgramAlloc"]) {
+ if ([self.pipeline_states objectForKey:@[values[@"name"][0],values[@"datahash"][0]]]) continue;
+ NSString *prg = [[NSString alloc] initWithData:self._h[values[@"datahash"][0]] encoding:NSUTF8StringEncoding];
+ NSError *error = nil;
+ id<MTLLibrary> library = [self.device newLibraryWithSource:prg
+ options:nil
+ error:&error];
+ MTLComputePipelineDescriptor *descriptor = [[MTLComputePipelineDescriptor alloc] init];
+ descriptor.computeFunction = [library newFunctionWithName:values[@"name"][0]];;
+ descriptor.supportIndirectCommandBuffers = YES;
+ MTLComputePipelineReflection *reflection = nil;
+ id<MTLComputePipelineState> pipeline_state = [self.device newComputePipelineStateWithDescriptor:descriptor
+ options:MTLPipelineOptionNone
+ reflection:&reflection
+ error:&error];
+ [self.pipeline_states setObject:pipeline_state forKey:@[values[@"name"][0],values[@"datahash"][0]]];
+ } else if ([values[@"op"] isEqualToString:@"ProgramExec"]) {
+ [_q_exec addObject:values];
+ } else if ([values[@"op"] isEqualToString:@"CopyOut"]) {
+ self.output_buffer = values[@"buffer_num"][0];
+ }
+ }
+ self._q = [_q_exec mutableCopy];
+ [self._h removeAllObjects];
+
+ return self;
+}
+
+- (NSArray *)processOutput:(const float *)output outputLength:(int)outputLength imgWidth:(float)imgWidth imgHeight:(float)imgHeight {
+ NSMutableArray *boxes = [NSMutableArray array];
+ int modelInputSize = self.yolo_res;
+ int numPredictions = pow(modelInputSize / 32, 2) * 21;
+
+ for (int index = 0; index < numPredictions; index++) {
+ int classId = 0;
+ float prob = 0.0;
+
+ for (int col = 0; col < self.yolo_classes.count; col++) {
+ float confidence = output[numPredictions * (col + 4) + index];
+ if (confidence > prob) {
+ prob = confidence;
+ classId = col;
+ }
+ }
+
+ if (prob < 0.25) continue;
+
+ float xc = output[index];
+ float yc = output[numPredictions + index];
+ float w = output[2 * numPredictions + index];
+ float h = output[3 * numPredictions + index];
+
+ float x1 = (xc - w / 2) / modelInputSize * imgWidth;
+ float y1 = (yc - h / 2) / modelInputSize * imgHeight;
+ float x2 = (xc + w / 2) / modelInputSize * imgWidth;
+ float y2 = (yc + h / 2) / modelInputSize * imgHeight;
+
+ [boxes addObject:@[@(x1), @(y1), @(x2), @(y2), @(classId), @(prob)]];
+ }
+
+ [boxes sortUsingComparator:^NSComparisonResult(NSArray *box1, NSArray *box2) {
+ NSNumber *prob1 = box1[5];
+ NSNumber *prob2 = box2[5];
+ return [prob2 compare:prob1];
+ }];
+
+ NSMutableArray *result = [NSMutableArray array];
+ while ([boxes count] > 0) {
+ NSArray *bestBox = boxes[0];
+ [result addObject:bestBox];
+ [boxes removeObjectAtIndex:0];
+
+ NSMutableArray *filteredBoxes = [NSMutableArray array];
+ for (NSArray *box in boxes) {
+ if ([self iouBetweenBox:bestBox andBox:box] < 0.7) {
+ [filteredBoxes addObject:box];
+ }
+ }
+ boxes = filteredBoxes;
+ }
+ return [result copy];
+}
+
+// Calculate intersection between two boxes
+- (CGFloat)intersectionBetweenBox:(NSArray *)box1 andBox:(NSArray *)box2 {
+ CGFloat x1 = MAX([box1[0] floatValue], [box2[0] floatValue]);
+ CGFloat y1 = MAX([box1[1] floatValue], [box2[1] floatValue]);
+ CGFloat x2 = MIN([box1[2] floatValue], [box2[2] floatValue]);
+ CGFloat y2 = MIN([box1[3] floatValue], [box2[3] floatValue]);
+ return MAX(0, x2 - x1) * MAX(0, y2 - y1);
+}
+
+// Calculate union between two boxes
+- (CGFloat)unionBetweenBox:(NSArray *)box1 andBox:(NSArray *)box2 {
+ CGFloat box1Area = ([box1[2] floatValue] - [box1[0] floatValue]) * ([box1[3] floatValue] - [box1[1] floatValue]);
+ CGFloat box2Area = ([box2[2] floatValue] - [box2[0] floatValue]) * ([box2[3] floatValue] - [box2[1] floatValue]);
+ return box1Area + box2Area - [self intersectionBetweenBox:box1 andBox:box2];
+}
+
+// Calculate Intersection over Union (IoU) between two boxes
+- (CGFloat)iouBetweenBox:(NSArray *)box1 andBox:(NSArray *)box2 {
+ return [self intersectionBetweenBox:box1 andBox:box2] / [self unionBetweenBox:box1 andBox:box2];
+}
+
+// Extract values from a string
+- (NSMutableDictionary<NSString *, id> *)extractValues:(NSString *)x {
+ NSMutableDictionary<NSString *, id> *values = [@{@"op": [x componentsSeparatedByString:@"("][0]} mutableCopy];
+ NSDictionary<NSString *, NSString *> *patterns = @{@"name": @"name='([^']+)'", @"datahash": @"datahash='([^']+)'", @"global_sizes": @"global_size=\\(([^)]+)\\)",
+ @"local_sizes": @"local_size=\\(([^)]+)\\)", @"bufs": @"bufs=\\(([^)]+)",
+ @"vals": @"vals=\\(([^)]+)", @"buffer_num": @"buffer_num=(\\d+)", @"size": @"size=(\\d+)"};
+ [patterns enumerateKeysAndObjectsUsingBlock:^(NSString *key, NSString *pattern, BOOL *stop) {
+ NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern:pattern options:0 error:nil];
+ NSTextCheckingResult *match = [regex firstMatchInString:x options:0 range:NSMakeRange(0, x.length)];
+ if (match) {
+ NSString *contents = [x substringWithRange:[match rangeAtIndex:1]];
+ NSMutableArray<NSString *> *extracted_values = [NSMutableArray array];
+ for (NSString *value in [contents componentsSeparatedByString:@","]) {
+ NSString *trimmed_value = [value stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]];
+ if (trimmed_value.length > 0) {
+ [extracted_values addObject:trimmed_value];
+ }
+ }
+ values[key] = [extracted_values copy];
+ }
+ }];
+ return values;
+}
+
+- (NSArray *)yolo_infer:(CGImageRef)cgImage withOrientation:(AVCaptureVideoOrientation)orientation {
+ CFDataRef rawData = CGDataProviderCopyData(CGImageGetDataProvider(cgImage));
+ if (!rawData) {
+ NSLog(@"Failed to get raw image data");
+ return nil;
+ }
+
+ const UInt8 *rawBytes = CFDataGetBytePtr(rawData);
+ size_t length = CFDataGetLength(rawData);
+ size_t width = CGImageGetWidth(cgImage);
+ size_t height = CGImageGetHeight(cgImage);
+ size_t rgbLength = (length / 4) * 3;
+
+ UInt8 *rgbData = (UInt8 *)malloc(width * width * 3);
+ if (!rgbData) {
+ CFRelease(rawData);
+ NSLog(@"Memory allocation failed for rgbData");
+ return nil;
+ }
+
+ if (orientation == AVCaptureVideoOrientationLandscapeRight) {
+ for (size_t i = 0, j = 0; i < length; i += 4, j += 3) {
+ rgbData[j] = rawBytes[i];
+ rgbData[j + 1] = rawBytes[i + 1];
+ rgbData[j + 2] = rawBytes[i + 2];
+ }
+ } else if (orientation == AVCaptureVideoOrientationLandscapeLeft) {
+ for (size_t i = 0, j = 0; i < length; i += 4, j += 3) {
+ rgbData[rgbLength - 1 - j - 2] = rawBytes[i];
+ rgbData[rgbLength - 1 - j - 1] = rawBytes[i + 1];
+ rgbData[rgbLength - 1 - j] = rawBytes[i + 2];
+ }
+ } else if (orientation == AVCaptureVideoOrientationPortrait) {
+ for (size_t i = 0; i < length; i += 4) {
+ int row = i / (width * 4);
+ int col = (i % (width * 4)) / 4;
+ rgbData[col * (width * 3) + ((height - 1 - row) * 3)] = rawBytes[i];
+ rgbData[col * (width * 3) + ((height - 1 - row) * 3) + 1] = rawBytes[i + 1];
+ rgbData[col * (width * 3) + ((height - 1 - row) * 3) + 2] = rawBytes[i + 2];
+ }
+ }
+
+ CFRelease(rawData);
+
+ id<MTLBuffer> buffer = self.buffers[self.input_buffer];
+ if (!buffer || !buffer.contents) {
+ free(rgbData);
+ NSLog(@"Metal buffer is not initialized or invalid");
+ return nil;
+ }
+
+ memset(buffer.contents, 0, buffer.length);
+ memcpy(buffer.contents, rgbData, MIN(buffer.length, width * width * 3));
+ free(rgbData);
+
+ for (NSMutableDictionary *values in self._q) {
+ id<MTLCommandBuffer> commandBuffer = [self.mtl_queue commandBuffer];
+ if (!commandBuffer) {
+ NSLog(@"Failed to create Metal command buffer");
+ continue;
+ }
+
+ id<MTLComputeCommandEncoder> encoder = [commandBuffer computeCommandEncoder];
+ if (!encoder) {
+ NSLog(@"Failed to create compute command encoder");
+ continue;
+ }
+
+ [encoder setComputePipelineState:self.pipeline_states[@[values[@"name"][0], values[@"datahash"][0]]]];
+ for (int i = 0; i < [(NSArray *)values[@"bufs"] count]; i++) {
+ [encoder setBuffer:self.buffers[values[@"bufs"][i]] offset:0 atIndex:i];
+ }
+ for (int i = 0; i < [(NSArray *)values[@"vals"] count]; i++) {
+ NSInteger value = [values[@"vals"][i] integerValue];
+ [encoder setBytes:&value length:sizeof(NSInteger) atIndex:i + [(NSArray *)values[@"bufs"] count]];
+ }
+
+ MTLSize globalSize = MTLSizeMake([values[@"global_sizes"][0] intValue], [values[@"global_sizes"][1] intValue], [values[@"global_sizes"][2] intValue]);
+ MTLSize localSize = MTLSizeMake([values[@"local_sizes"][0] intValue], [values[@"local_sizes"][1] intValue], [values[@"local_sizes"][2] intValue]);
+ [encoder dispatchThreadgroups:globalSize threadsPerThreadgroup:localSize];
+ [encoder endEncoding];
+ [commandBuffer commit];
+ [self.mtl_buffers_in_flight addObject:commandBuffer];
+ }
+
+ for (int i = 0; i < self.mtl_buffers_in_flight.count; i++) {
+ [self.mtl_buffers_in_flight[i] waitUntilCompleted];
+ }
+ [self.mtl_buffers_in_flight removeAllObjects];
+
+ buffer = self.buffers[self.output_buffer];
+ if (!buffer || !buffer.contents) {
+ NSLog(@"Output buffer is invalid");
+ return nil;
+ }
+
+ const void *bufferPointer = buffer.contents;
+ float *floatArray = malloc(buffer.length);
+ if (!floatArray) {
+ NSLog(@"Memory allocation failed for output array");
+ return nil;
+ }
+
+ memcpy(floatArray, bufferPointer, buffer.length);
+ NSArray *output = [self processOutput:floatArray outputLength:buffer.length / 4 imgWidth:self.yolo_res imgHeight:self.yolo_res];
+
+ NSMutableString *classNamesString = [NSMutableString string];
+ for (int i = 0; i < output.count; i++) {
+ [classNamesString appendString:self.yolo_classes[[output[i][4] intValue]][0]];
+ if (i < output.count - 1) [classNamesString appendString:@", "];
+ }
+ //NSLog(@"Class Names: %@", classNamesString);
+
+ NSDateFormatter *dateFormatter = [[NSDateFormatter alloc] init];
+ [dateFormatter setDateFormat:@"yyyy-MM-dd HH:mm:ss"];
+ NSString *currentTimestamp = [dateFormatter stringFromDate:[NSDate date]];
+
+ NSString *logEntry = [NSString stringWithFormat:@"%@ - Class Names: %@", currentTimestamp, classNamesString];
+ //NSLog(@"%@", logEntry);
+
+ NSString *filePath = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES)[0] stringByAppendingPathComponent:@"logs.txt"];
+ NSFileHandle *fileHandle = [NSFileHandle fileHandleForWritingAtPath:filePath] ?: [NSFileHandle fileHandleForWritingAtPath:([[NSFileManager defaultManager] createFileAtPath:filePath contents:nil attributes:nil], filePath)];
+ [fileHandle seekToEndOfFile];
+ [fileHandle writeData:[[logEntry stringByAppendingString:@"\n"] dataUsingEncoding:NSUTF8StringEncoding]];
+ [fileHandle closeFile];
+
+ free(floatArray);
+
+ return output;
+}
+
+@end
+