Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

Fast remote (#57)

Rory Clear12a523c785ce
diff --git a/clearcam/Yolo.h b/clearcam/Yolo.h
index 561a6f1..05c3d8c 100644
--- a/clearcam/Yolo.h
+++ b/clearcam/Yolo.h
@@ -19,7 +19,9 @@ NS_ASSUME_NONNULL_BEGIN
@property (nonatomic, strong) NSString *input_buffer;
@property (nonatomic, strong) NSString *output_buffer;
@property (nonatomic, assign) UInt8 *rgbData;
+@property (nonatomic, assign) UInt8 *last_rgbData;
@property (nonatomic, strong) NSSet<NSNumber *> *yoloIndexSet;
+@property (nonatomic, strong) NSString *sessionID;
- (NSArray *)yolo_infer:(CGImageRef)cgImage withOrientation:(AVCaptureVideoOrientation)orientation;
// Initializer
diff --git a/clearcam/Yolo.m b/clearcam/Yolo.m
index de099a7..95a1b87 100644
--- a/clearcam/Yolo.m
+++ b/clearcam/Yolo.m
@@ -3,6 +3,7 @@
#import <UIKit/UIKit.h>
#import <sys/utsname.h>
#import "SettingsManager.h"
+#include <arm_neon.h>
@implementation Yolo
@@ -19,6 +20,8 @@ NSMutableArray *_q;
NSString *input_buffer;
NSString *output_buffer;
UInt8 *rgbData;
+UInt8 *last_rgbData; //for diff
+NSString *sessionID;
- (instancetype)init {
self = [super init];
@@ -29,6 +32,8 @@ UInt8 *rgbData;
self.mtl_buffers_in_flight = [[NSMutableArray alloc] init];
self.yolo_res = 640;
self.rgbData = (UInt8 *)malloc(self.yolo_res * self.yolo_res * 3);
+ self.last_rgbData = (UInt8 *)malloc(self.yolo_res * self.yolo_res * 3);
+
self.yolo_classes = @[
@[NSLocalizedString(@"person", nil), [UIColor redColor]],
@@ -262,7 +267,37 @@ UInt8 *rgbData;
CFRelease(rawData);
if ([[NSUserDefaults standardUserDefaults] boolForKey:@"useOwnInferenceServerEnabled"]) {
- NSArray *yoloResults = [self sendYOLORequest];
+ int total_bytes = self.yolo_res * self.yolo_res * 3;
+ int diff_count = 0;
+ int diff_limit = total_bytes / 10; // 10%
+ uint8_t *diff_buffer = malloc(total_bytes);
+ int diff_offset = 0;
+ uint8_t *curr = self.rgbData;
+ uint8_t *prev = self.last_rgbData;
+
+ for (int i = 0; i < total_bytes; i++) {
+ int delta = (int)curr[i] - (int)prev[i];
+ if (delta > 10 || delta < -10) {
+ prev[i] = curr[i];
+ diff_count++;
+ int32_t index = i;
+ *(int32_t *)(diff_buffer + diff_offset) = index;
+ diff_offset += 4;
+ diff_buffer[diff_offset++] = curr[i];
+
+ if (diff_count >= diff_limit) {
+ memcpy(prev, curr, total_bytes);
+ NSArray *yoloResults = [self sendYOLORequestWithDiffData:nil length:0];
+ if (yoloResults.count > 0) {
+ free(diff_buffer);
+ return yoloResults;
+ }
+ break;
+ }
+ }
+ }
+ NSArray *yoloResults = [self sendYOLORequestWithDiffData:diff_buffer length:diff_offset];
+ free(diff_buffer);
if (yoloResults.count > 0) return yoloResults;
}
@@ -325,37 +360,48 @@ UInt8 *rgbData;
return output;
}
-- (NSArray *)sendYOLORequest {
- NSURL *url = [NSURL URLWithString: [[[NSUserDefaults standardUserDefaults] stringForKey:@"own_inference_server_address"] stringByAppendingPathComponent:@"yolo"]];
+- (NSArray *)sendYOLORequestWithDiffData:(uint8_t *)diffData length:(NSInteger)length {
+ BOOL isDiff = (diffData != nil && length > 0);
+ NSString *endpoint = isDiff ? @"diff" : @"yolo";
+ NSURL *url = [NSURL URLWithString:[[[NSUserDefaults standardUserDefaults] stringForKey:@"own_inference_server_address"] stringByAppendingPathComponent:endpoint]];
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
[request setHTTPMethod:@"POST"];
- [request setHTTPBody:[NSData dataWithBytesNoCopy:self.rgbData length:self.yolo_res * self.yolo_res * 3 freeWhenDone:NO]];
+ if (isDiff) {
+ [request setHTTPBody:[NSData dataWithBytesNoCopy:diffData length:length freeWhenDone:NO]];
+ } else {
+ [request setHTTPBody:[NSData dataWithBytesNoCopy:self.rgbData length:self.yolo_res * self.yolo_res * 3 freeWhenDone:NO]];
+ }
+ if (self.sessionID) [request setValue:self.sessionID forHTTPHeaderField:@"x-session-id"];
NSURLResponse *response = nil;
NSError *error = nil;
NSData *responseData = [NSURLConnection sendSynchronousRequest:request returningResponse:&response error:&error];
if (error || !responseData) {
- NSLog(@"YOLO request failed: %@", error);
return @[];
}
+ if (!self.sessionID && [response isKindOfClass:[NSHTTPURLResponse class]]) {
+ NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
+ NSString *newSessionID = httpResponse.allHeaderFields[@"x-session-id"];
+ if (newSessionID) self.sessionID = newSessionID;
+ }
float *floatArray = (float *)responseData.bytes;
NSUInteger floatCount = responseData.length / sizeof(float);
NSMutableArray *output = [NSMutableArray new];
- for (NSUInteger i = 0; i < floatCount; i += 6) { // 4 coords + class + conf
+ float threshold = [[[NSUserDefaults standardUserDefaults] objectForKey:@"threshold"] ?: @(25) floatValue] / 100.0;
+
+ for (NSUInteger i = 0; i < floatCount; i += 6) {
float confidence = floatArray[i + 4];
NSNumber *classId = @(floatArray[i + 5]);
-
- if ([self.yoloIndexSet containsObject:classId] && confidence > ([[[NSUserDefaults standardUserDefaults] objectForKey:@"threshold"] ?: @(25) floatValue] / 100.0)) {
+ if ([self.yoloIndexSet containsObject:classId] && confidence > threshold) {
[output addObject:@[
- @(floatArray[i]), // x1
- @(floatArray[i+1]), // y1
- @(floatArray[i+2]), // x2
- @(floatArray[i+3]), // y2
- classId, // class
- @(confidence) // confidence
+ @(floatArray[i]), @(floatArray[i+1]),
+ @(floatArray[i+2]), @(floatArray[i+3]),
+ classId, @(confidence)
]];
}
}
return output;
}
+
@end
+
diff --git a/yolov8.py b/yolov8.py
index ed5ae67..bcbac8f 100644
--- a/yolov8.py
+++ b/yolov8.py
@@ -331,11 +331,11 @@ from starlette.responses import Response
@TinyJit
def do_inf(image):
- image = image.reshape(1, 640, 640, 3)
- image = image[..., ::-1].permute(0, 3, 1, 2)
- image = image / 255.0
- predictions = yolo_infer(image)
- return predictions.flatten()
+ image = image.reshape(1, 640, 640, 3)
+ image = image[..., ::-1].permute(0, 3, 1, 2)
+ image = image / 255.0
+ predictions = yolo_infer(image)
+ return predictions.flatten()
# Model initialization (same as before)
yolo_variant = sys.argv[1] if len(sys.argv) >= 2 else (print("No variant given, so choosing 'n' as the default. Yolov8 has different variants, you can choose from ['n', 's', 'm', 'l', 'x']") or 'n')
@@ -344,40 +344,114 @@ yolo_infer = YOLOv8(w=width, r=ratio, d=depth, num_classes=80)
state_dict = safe_load(get_weights_location(yolo_variant))
load_state_dict(yolo_infer, state_dict)
-class ASGIApp:
- async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
- if scope["path"] != "/yolo" or scope["method"] != "POST":
- await send({
- "type": "http.response.start",
- "status": 404,
- "headers": [(b"content-type", b"text/plain")],
- })
- await send({"type": "http.response.body", "body": b"Not Found"})
- return
-
- # Accumulate request body
- body = b""
- more_body = True
- while more_body:
- message = await receive()
- body += message.get("body", b"")
- more_body = message.get("more_body", False)
-
- # Inference
- im = Tensor(body, dtype=dtypes.uint8)
- pred = do_inf(im).numpy()
- response_data = struct.pack('<1800f', *pred)
+import uuid
+from typing import Optional, Dict
+import struct
- # Send response
- await send({
- "type": "http.response.start",
- "status": 200,
- "headers": [
- (b"content-type", b"application/octet-stream"),
- (b"content-length", str(len(response_data)).encode()),
- ],
- })
- await send({"type": "http.response.body", "body": response_data})
+class ASGIApp:
+ def __init__(self, yolo_res=640):
+ self.yolo_res = yolo_res
+ self.session_images: Dict[str, bytearray] = {}
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ path = scope.get("path")
+ method = scope.get("method")
+
+ if method != "POST":
+ await self.send_not_found(send)
+ return
+
+ # Read full request body
+ body = b""
+ more_body = True
+ while more_body:
+ message = await receive()
+ body += message.get("body", b"")
+ more_body = message.get("more_body", False)
+
+ headers = dict(scope.get("headers", [])) # List of (key, value) tuples
+
+ if path == "/yolo":
+ session_id = headers.get(b"x-session-id")
+ await self.handle_yolo(body, send, session_id)
+
+ elif path == "/diff":
+ session_id = headers.get(b"x-session-id")
+ await self.handle_diff(body, send, session_id)
+
+ else:
+ await self.send_not_found(send)
+
+ async def handle_yolo(self, body: bytes, send: Send, session_id: Optional[bytes]):
+ if session_id is None:
+ # Create new session
+ session_id = str(uuid.uuid4()).encode()
+ self.session_images[session_id] = bytearray(body)
+ else:
+ self.session_images[session_id] = bytearray(body)
+
+ im = Tensor(body, dtype=dtypes.uint8)
+ pred = do_inf(im).numpy()
+ response_data = struct.pack('<1800f', *pred)
+
+ headers = [
+ (b"content-type", b"application/octet-stream"),
+ (b"content-length", str(len(response_data)).encode()),
+ (b"x-session-id", session_id)
+ ]
+ await self.send_with_headers(send, response_data, headers)
+
+ async def handle_diff(self, body: bytes, send: Send, session_id: Optional[bytes]):
+ if session_id is None or session_id not in self.session_images:
+ await self.send_response(send, b"", status=400, content_type=b"text/plain", message=b"Invalid or missing session ID")
+ return
+
+ img = bytearray(self.session_images[session_id]) # Copy previous image
+
+ i = 0
+ while i + 5 <= len(body):
+ index = struct.unpack_from("<i", body, i)[0]
+ value = body[i + 4]
+ if 0 <= index < len(img):
+ img[index] = value
+ i += 5
+
+ im = Tensor(bytes(img), dtype=dtypes.uint8)
+ pred = do_inf(im).numpy()
+ response_data = struct.pack('<1800f', *pred)
+
+ self.session_images[session_id] = img
+ await self.send_response(send, response_data)
+
+ async def send_response(self, send: Send, body: bytes, status: int = 200,
+ content_type: bytes = b"application/octet-stream",
+ message: Optional[bytes] = None):
+ headers = [
+ (b"content-type", content_type),
+ (b"content-length", str(len(body)).encode())
+ ]
+ await send({
+ "type": "http.response.start",
+ "status": status,
+ "headers": headers,
+ })
+ await send({"type": "http.response.body", "body": message or body})
+
+ async def send_with_headers(self, send: Send, body: bytes, headers: list):
+ await send({
+ "type": "http.response.start",
+ "status": 200,
+ "headers": headers,
+ })
+ await send({"type": "http.response.body", "body": body})
+
+ async def send_not_found(self, send: Send):
+ await send({
+ "type": "http.response.start",
+ "status": 404,
+ "headers": [(b"content-type", b"text/plain")],
+ })
+ await send({"type": "http.response.body", "body": b"Not Found"})
if __name__ == "__main__":
app = ASGIApp()
@@ -429,4 +503,4 @@ if __name__ == '__main__':
httpd = HTTPServer(server_address, YOLORequestHandler)
print("Serving YOLO on http://0.0.0.0:6667")
httpd.serve_forever()
-'''
\ No newline at end of file
+'''