Add object detection, tracking, mobile notifications, and search to any security camera.
Fast remote (#57)
Rory Clear12a523c785ce
diff --git a/clearcam/Yolo.h b/clearcam/Yolo.hindex 561a6f1..05c3d8c 100644--- a/clearcam/Yolo.h+++ b/clearcam/Yolo.h@@ -19,7 +19,9 @@ NS_ASSUME_NONNULL_BEGIN@property (nonatomic, strong) NSString *input_buffer;@property (nonatomic, strong) NSString *output_buffer;@property (nonatomic, assign) UInt8 *rgbData;+@property (nonatomic, assign) UInt8 *last_rgbData;@property (nonatomic, strong) NSSet<NSNumber *> *yoloIndexSet;+@property (nonatomic, strong) NSString *sessionID;- (NSArray *)yolo_infer:(CGImageRef)cgImage withOrientation:(AVCaptureVideoOrientation)orientation;// Initializerdiff --git a/clearcam/Yolo.m b/clearcam/Yolo.mindex de099a7..95a1b87 100644--- a/clearcam/Yolo.m+++ b/clearcam/Yolo.m@@ -3,6 +3,7 @@#import <UIKit/UIKit.h>#import <sys/utsname.h>#import "SettingsManager.h"+#include <arm_neon.h>@implementation Yolo@@ -19,6 +20,8 @@ NSMutableArray *_q;NSString *input_buffer;NSString *output_buffer;UInt8 *rgbData;+UInt8 *last_rgbData; //for diff+NSString *sessionID;- (instancetype)init {self = [super init];@@ -29,6 +32,8 @@ UInt8 *rgbData;self.mtl_buffers_in_flight = [[NSMutableArray alloc] init];self.yolo_res = 640;self.rgbData = (UInt8 *)malloc(self.yolo_res * self.yolo_res * 3);+ self.last_rgbData = (UInt8 *)malloc(self.yolo_res * self.yolo_res * 3);+self.yolo_classes = @[@[NSLocalizedString(@"person", nil), [UIColor redColor]],@@ -262,7 +267,37 @@ UInt8 *rgbData;CFRelease(rawData);if ([[NSUserDefaults standardUserDefaults] boolForKey:@"useOwnInferenceServerEnabled"]) {- NSArray *yoloResults = [self sendYOLORequest];+ int total_bytes = self.yolo_res * self.yolo_res * 3;+ int diff_count = 0;+ int diff_limit = total_bytes / 10; // 10%+ uint8_t *diff_buffer = malloc(total_bytes);+ int diff_offset = 0;+ uint8_t *curr = self.rgbData;+ uint8_t *prev = self.last_rgbData;++ for (int i = 0; i < total_bytes; i++) {+ int delta = (int)curr[i] - (int)prev[i];+ if (delta > 10 || delta < -10) {+ prev[i] = curr[i];+ diff_count++;+ int32_t index = i;+ *(int32_t *)(diff_buffer + diff_offset) = index;+ diff_offset += 4;+ diff_buffer[diff_offset++] = curr[i];++ if (diff_count >= diff_limit) {+ memcpy(prev, curr, total_bytes);+ NSArray *yoloResults = [self sendYOLORequestWithDiffData:nil length:0];+ if (yoloResults.count > 0) {+ free(diff_buffer);+ return yoloResults;+ }+ break;+ }+ }+ }+ NSArray *yoloResults = [self sendYOLORequestWithDiffData:diff_buffer length:diff_offset];+ free(diff_buffer);if (yoloResults.count > 0) return yoloResults;}@@ -325,37 +360,48 @@ UInt8 *rgbData;return output;}-- (NSArray *)sendYOLORequest {- NSURL *url = [NSURL URLWithString: [[[NSUserDefaults standardUserDefaults] stringForKey:@"own_inference_server_address"] stringByAppendingPathComponent:@"yolo"]];+- (NSArray *)sendYOLORequestWithDiffData:(uint8_t *)diffData length:(NSInteger)length {+ BOOL isDiff = (diffData != nil && length > 0);+ NSString *endpoint = isDiff ? @"diff" : @"yolo";+ NSURL *url = [NSURL URLWithString:[[[NSUserDefaults standardUserDefaults] stringForKey:@"own_inference_server_address"] stringByAppendingPathComponent:endpoint]];NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];[request setHTTPMethod:@"POST"];- [request setHTTPBody:[NSData dataWithBytesNoCopy:self.rgbData length:self.yolo_res * self.yolo_res * 3 freeWhenDone:NO]];+ if (isDiff) {+ [request setHTTPBody:[NSData dataWithBytesNoCopy:diffData length:length freeWhenDone:NO]];+ } else {+ [request setHTTPBody:[NSData dataWithBytesNoCopy:self.rgbData length:self.yolo_res * self.yolo_res * 3 freeWhenDone:NO]];+ }+ if (self.sessionID) [request setValue:self.sessionID forHTTPHeaderField:@"x-session-id"];NSURLResponse *response = nil;NSError *error = nil;NSData *responseData = [NSURLConnection sendSynchronousRequest:request returningResponse:&response error:&error];if (error || !responseData) {- NSLog(@"YOLO request failed: %@", error);return @[];}+ if (!self.sessionID && [response isKindOfClass:[NSHTTPURLResponse class]]) {+ NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;+ NSString *newSessionID = httpResponse.allHeaderFields[@"x-session-id"];+ if (newSessionID) self.sessionID = newSessionID;+ }float *floatArray = (float *)responseData.bytes;NSUInteger floatCount = responseData.length / sizeof(float);NSMutableArray *output = [NSMutableArray new];- for (NSUInteger i = 0; i < floatCount; i += 6) { // 4 coords + class + conf+ float threshold = [[[NSUserDefaults standardUserDefaults] objectForKey:@"threshold"] ?: @(25) floatValue] / 100.0;++ for (NSUInteger i = 0; i < floatCount; i += 6) {float confidence = floatArray[i + 4];NSNumber *classId = @(floatArray[i + 5]);-- if ([self.yoloIndexSet containsObject:classId] && confidence > ([[[NSUserDefaults standardUserDefaults] objectForKey:@"threshold"] ?: @(25) floatValue] / 100.0)) {+ if ([self.yoloIndexSet containsObject:classId] && confidence > threshold) {[output addObject:@[- @(floatArray[i]), // x1- @(floatArray[i+1]), // y1- @(floatArray[i+2]), // x2- @(floatArray[i+3]), // y2- classId, // class- @(confidence) // confidence+ @(floatArray[i]), @(floatArray[i+1]),+ @(floatArray[i+2]), @(floatArray[i+3]),+ classId, @(confidence)]];}}return output;}+@end+diff --git a/yolov8.py b/yolov8.pyindex ed5ae67..bcbac8f 100644--- a/yolov8.py+++ b/yolov8.py@@ -331,11 +331,11 @@ from starlette.responses import Response@TinyJitdef do_inf(image):- image = image.reshape(1, 640, 640, 3)- image = image[..., ::-1].permute(0, 3, 1, 2)- image = image / 255.0- predictions = yolo_infer(image)- return predictions.flatten()+ image = image.reshape(1, 640, 640, 3)+ image = image[..., ::-1].permute(0, 3, 1, 2)+ image = image / 255.0+ predictions = yolo_infer(image)+ return predictions.flatten()# Model initialization (same as before)yolo_variant = sys.argv[1] if len(sys.argv) >= 2 else (print("No variant given, so choosing 'n' as the default. Yolov8 has different variants, you can choose from ['n', 's', 'm', 'l', 'x']") or 'n')@@ -344,40 +344,114 @@ yolo_infer = YOLOv8(w=width, r=ratio, d=depth, num_classes=80)state_dict = safe_load(get_weights_location(yolo_variant))load_state_dict(yolo_infer, state_dict)-class ASGIApp:- async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:- if scope["path"] != "/yolo" or scope["method"] != "POST":- await send({- "type": "http.response.start",- "status": 404,- "headers": [(b"content-type", b"text/plain")],- })- await send({"type": "http.response.body", "body": b"Not Found"})- return-- # Accumulate request body- body = b""- more_body = True- while more_body:- message = await receive()- body += message.get("body", b"")- more_body = message.get("more_body", False)-- # Inference- im = Tensor(body, dtype=dtypes.uint8)- pred = do_inf(im).numpy()- response_data = struct.pack('<1800f', *pred)+import uuid+from typing import Optional, Dict+import struct- # Send response- await send({- "type": "http.response.start",- "status": 200,- "headers": [- (b"content-type", b"application/octet-stream"),- (b"content-length", str(len(response_data)).encode()),- ],- })- await send({"type": "http.response.body", "body": response_data})+class ASGIApp:+ def __init__(self, yolo_res=640):+ self.yolo_res = yolo_res+ self.session_images: Dict[str, bytearray] = {}++ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ path = scope.get("path")+ method = scope.get("method")++ if method != "POST":+ await self.send_not_found(send)+ return++ # Read full request body+ body = b""+ more_body = True+ while more_body:+ message = await receive()+ body += message.get("body", b"")+ more_body = message.get("more_body", False)++ headers = dict(scope.get("headers", [])) # List of (key, value) tuples++ if path == "/yolo":+ session_id = headers.get(b"x-session-id")+ await self.handle_yolo(body, send, session_id)++ elif path == "/diff":+ session_id = headers.get(b"x-session-id")+ await self.handle_diff(body, send, session_id)++ else:+ await self.send_not_found(send)++ async def handle_yolo(self, body: bytes, send: Send, session_id: Optional[bytes]):+ if session_id is None:+ # Create new session+ session_id = str(uuid.uuid4()).encode()+ self.session_images[session_id] = bytearray(body)+ else:+ self.session_images[session_id] = bytearray(body)++ im = Tensor(body, dtype=dtypes.uint8)+ pred = do_inf(im).numpy()+ response_data = struct.pack('<1800f', *pred)++ headers = [+ (b"content-type", b"application/octet-stream"),+ (b"content-length", str(len(response_data)).encode()),+ (b"x-session-id", session_id)+ ]+ await self.send_with_headers(send, response_data, headers)++ async def handle_diff(self, body: bytes, send: Send, session_id: Optional[bytes]):+ if session_id is None or session_id not in self.session_images:+ await self.send_response(send, b"", status=400, content_type=b"text/plain", message=b"Invalid or missing session ID")+ return++ img = bytearray(self.session_images[session_id]) # Copy previous image++ i = 0+ while i + 5 <= len(body):+ index = struct.unpack_from("<i", body, i)[0]+ value = body[i + 4]+ if 0 <= index < len(img):+ img[index] = value+ i += 5++ im = Tensor(bytes(img), dtype=dtypes.uint8)+ pred = do_inf(im).numpy()+ response_data = struct.pack('<1800f', *pred)++ self.session_images[session_id] = img+ await self.send_response(send, response_data)++ async def send_response(self, send: Send, body: bytes, status: int = 200,+ content_type: bytes = b"application/octet-stream",+ message: Optional[bytes] = None):+ headers = [+ (b"content-type", content_type),+ (b"content-length", str(len(body)).encode())+ ]+ await send({+ "type": "http.response.start",+ "status": status,+ "headers": headers,+ })+ await send({"type": "http.response.body", "body": message or body})++ async def send_with_headers(self, send: Send, body: bytes, headers: list):+ await send({+ "type": "http.response.start",+ "status": 200,+ "headers": headers,+ })+ await send({"type": "http.response.body", "body": body})++ async def send_not_found(self, send: Send):+ await send({+ "type": "http.response.start",+ "status": 404,+ "headers": [(b"content-type", b"text/plain")],+ })+ await send({"type": "http.response.body", "body": b"Not Found"})if __name__ == "__main__":app = ASGIApp()@@ -429,4 +503,4 @@ if __name__ == '__main__':httpd = HTTPServer(server_address, YOLORequestHandler)print("Serving YOLO on http://0.0.0.0:6667")httpd.serve_forever()-'''\ No newline at end of file+'''