Add object detection, tracking, mobile notifications, and search to any security camera.
Egpu10 (#152)
Rory Clear7b9c87e8070a
diff --git a/clearcam.py b/clearcam.pyindex 1999cae..07ef692 100644--- a/clearcam.py+++ b/clearcam.py@@ -28,6 +28,7 @@ import multiprocessingimport reimport base64from utils.helpers import send_notif, find_ffmpeg, export_clip, upload_file, encrypt_file, export_and_upload+import pickle# RTSP URL# Video capture thread@@ -164,14 +165,13 @@ def _get_stream_resolution(src):return 1920, 1080class VideoCapture:- def __init__(self, src, cam_name="camera", vod=False):+ def __init__(self):self.output_dir_raw = {}self.frame_num = {}self.last_frame_num = {}self.vod = {}# objects in scene countself.counter = {}- self.cam_name = cam_nameself.object_set = {}self.object_set_zone = {}@@ -207,10 +207,11 @@ class VideoCapture:#self.last_shapes_time = time.time()#self.det_shapes = []+ def init_cam(self, cam_name, src):self.counter[cam_name] = RollingClassCounter(cam_name=cam_name, window_seconds=float('inf'))self.src[cam_name] = src # todoself.last_frame[cam_name] = None- self.vod[cam_name] = vod+ self.vod[cam_name] = src.endswith(('.mp4', '.avi', '.mov', '.mkv', '.webm'))self.frame_num[cam_name] = -1self.last_frame_num[cam_name] = -1self.object_set[cam_name] = set()@@ -243,13 +244,36 @@ class VideoCapture:self.tracker[cam_name] = ocsort.OCSort(max_age=100)self.count[cam_name] = 0self.prev_time[cam_name] = time.time()- self.current_stream_dir_raw[cam_name] = self._get_new_stream_dir(self.cam_name)+ self.current_stream_dir_raw[cam_name] = self._get_new_stream_dir(cam_name)self.filename[cam_name] = None- def start(self, cam_name):- if not self.vod[cam_name]: threading.Thread(target=self.frame_loop, daemon=True).start()- if not self.vod[cam_name] or not self.output_dir_raw[cam_name].exists():- while True: self.process_frame(cam_name=cam_name)+ def start(self):+ cam_check = time.time()+ cams = database.run_get("links", None)+ for cam_name in cams.keys():+ self.init_cam(cam_name=cam_name, src=cams[cam_name])+ threading.Thread(target=self.frame_loop, args=(cam_name,), daemon=True).start() # todo non vod only!+ while True:+ if time.time() - cam_check >= 5:+ new_cams = database.run_get("links", None)+ for cam_name in new_cams.keys():+ if cam_name not in cams:+ self.init_cam(cam_name=cam_name, src=new_cams[cam_name])+ if not self.vod[cam_name]: threading.Thread(target=self.frame_loop, args=(cam_name,), daemon=True).start() # todo non vod only+ cams = new_cams+ for cam_name in cams.keys():+ self.process_frame(cam_name=cam_name)+ if use_clip:process_queue()+ if len(object_queue) > 0:+ try:+ img = cv2.imread(object_queue[0])+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)+ clip_latest_img(img)+ process_latest_face(img)+ except Exception: print("error in object processing", object_queue[0])+ del object_queue[0]++def _get_new_stream_dir(self, cam_name):timestamp = "video" if self.vod[cam_name] else datetime.now().strftime("%Y-%m-%d")@@ -366,13 +390,12 @@ class VideoCapture:if (y2_new - y1_new) < 100 or (x2_new - x1_new) < 100: return # too smallcrop = self.last_frame[cam_name][y1_new:y2_new, x1_new:x2_new]cv2.imwrite(str(object_filename), crop)-+ if use_clip or use_face: object_queue.append(object_filename)- def frame_loop(self):+ def frame_loop(self, cam_name):fail_count = 0- cam_name = self.cam_nameframe_size = self.width[cam_name] * self.height[cam_name] * 3- while self.running[cam_name]:+ while (BASE_DIR / "cameras" / cam_name).exists():try:raw_bytes = self.proc[cam_name].stdout.read(frame_size)if len(raw_bytes) != frame_size:@@ -394,7 +417,6 @@ class VideoCapture:def process_frame(self, cam_name):try:- if not (BASE_DIR / "cameras" / cam_name).is_dir(): os._exit(1) # deleted camif self.vod[cam_name]:if cam_name not in self.cap:self.cap[cam_name] = cv2.VideoCapture(self.src[cam_name])@@ -405,8 +427,7 @@ class VideoCapture:self.last_frame[cam_name] = frame #todoif not ret or cam_name not in database.run_get("links", None):self.running[cam_name] = False- database.run_put("analysis_prog", cam_name, {"Tracking":100})- os._exit(0)+ if "Processing" not in database.run_get("analysis_prog", cam_name): database.run_put("analysis_prog", cam_name, {"Tracking":100}) # todo stop when done?else:self.last_preds[cam_name], _ = self.run_inference(frame, cam_name=cam_name)database.run_put("analysis_prog", cam_name, {"Tracking":self.cap[cam_name].get(cv2.CAP_PROP_POS_FRAMES)/self.cap[cam_name].get(cv2.CAP_PROP_FRAME_COUNT)*100})@@ -416,12 +437,9 @@ class VideoCapture:last_frame_num = self.last_frame_num[cam_name]if self.raw_frame[cam_name] is None: returnframe = self.raw_frame[cam_name].copy()- if frame_num == last_frame_num:- time.sleep(1 / 30)- return+ if frame_num == last_frame_num: returnif not any(counter.is_active() for _, counter in self.alert_counters[cam_name].items()): # don't run inference when no active scheds- time.sleep(1 / 30)with self.lock[cam_name]: self.last_preds[cam_name] = [] # to remove annotation when no alerts activeelse:preds, frame = self.run_inference(frame, cam_name=cam_name)@@ -468,9 +486,9 @@ class VideoCapture:self.last_det[cam_name] = time.time()alert.last_det = time.time()if (self.send_det[cam_name] and userID is not None and not self.vod[cam_name]) and time.time() - self.last_det[cam_name] >= 6: #send 15ish second clip after- export_and_upload(cam_name=cam_name, thumbnail=self.filename[cam_name], userID=userID, key=key)+ threading.Thread(target=export_and_upload, kwargs={"cam_name": cam_name, "thumbnail": self.filename[cam_name], "userID": userID, "key": key}, daemon=True).start()self.send_det[cam_name] = False-+if (time.time() - self.last_live_check[cam_name]) >= 5:self.last_live_check[cam_name] = time.time()link = database.run_get("links", cam_name)@@ -515,9 +533,7 @@ class VideoCapture:encrypt_file(Path(mp4_filename), Path(f"""{mp4_filename}.aes"""), key)Path(mp4_filename).unlink()threading.Thread(target=upload_to_r2, args=(Path(f"""{mp4_filename}.aes"""), live_link[cam_name]), daemon=True).start()- else:- self.count[cam_name]+=1- if not self.vod[cam_name]: time.sleep(1 / 30)+ else: self.count[cam_name]+=1except Exception as e:print("Error in process_frame:", e, cam_name)@@ -532,7 +548,7 @@ class VideoCapture:def run_inference(self, frame, cam_name):frame = Tensor(frame)- preds = model(frame).numpy()+ preds = model.jit_infer(frame).numpy()thresh = (self.settings[cam_name].get("threshold") if self.settings[cam_name] else 0.5) or 0.5 #todo clean!online_targets = self.tracker[cam_name].update(preds, thresh)online_targets = [p for p in online_targets if (classes is None or str(int(p.class_id)) in classes)]@@ -618,27 +634,21 @@ def point_not_in_polygon(coords, poly):return Falsereturn True-def run_encode_text(return_q, clip, text):- res = clip.model._encode_text(text, realize=True)- return_q.put(res)- return res+def run_encode_text(clip, text): return clip.model._encode_text(text, realize=True)-def run_search(return_q, clip, image_text, top_k, cam_name, selected_dir):- res = clip.search(image_text, top_k, cam_name, selected_dir)- return_q.put(res)+def run_search(clip, image_text, top_k, cam_name, selected_dir): return clip.search(image_text, top_k, cam_name, selected_dir)-def run_clip(return_q, clip, im, top_k, cam_name, selected_dir, is_face):+def run_clip(clip, im, top_k, cam_name, selected_dir, is_face):im = clip.preprocess_face(im) if is_face else clip.preprocess_clip(im)if im is not None:embedding = clip.precompute_face_embedding_bs1_np(im) if is_face else clip.precompute_embedding_bs1_np(im)res = clip.search(None, top_k, cam_name, selected_dir, embedding, is_face)else:res = []- return_q.put(res)+ return resclass HLSRequestHandler(BaseHTTPRequestHandler):def __init__(self, *args, **kwargs):- self.object_finder = kwargs.pop('clip_instance', None)super().__init__(*args, **kwargs)def log_message(self, format, *args): pass # don't print stuff@@ -729,7 +739,6 @@ class HLSRequestHandler(BaseHTTPRequestHandler):self.send_error(400, "Missing cam_name or src")return- start_cam(rtsp=src,cam_name=cam_name,model_variant=model_variant)database.run_put("links", cam_name, src)self.send_response(302)self.send_header('Location', '/')@@ -1059,21 +1068,21 @@ class HLSRequestHandler(BaseHTTPRequestHandler):})selected_dirs.append("video")- if image_text and use_clip: self.object_finder._load_all_embeddings()- if (uploaded_image or similar_img) and (use_clip or use_face): self.object_finder._load_all_embeddings(face=is_face)+ if image_text and use_clip: add_to_queue(object_finder._load_all_embeddings)+ if (uploaded_image or similar_img) and (use_clip or use_face): add_to_queue(object_finder._load_all_embeddings, is_face)if uploaded_image and (use_clip or is_face):- results = process_with_clip_lock(run_clip, self.object_finder, uploaded_image, start+count, cam_name, selected_dir, is_face)+ results = add_to_queue(run_clip, object_finder, uploaded_image, start+count, cam_name, selected_dir, is_face)self.send_results(results, start, count)returnif similar_img and (use_clip or is_face):- results = process_with_clip_lock(run_clip, self.object_finder, similar_img, start+count, cam_name, selected_dir, is_face) # todo one with above+ results = add_to_queue(run_clip, object_finder, similar_img, start+count, cam_name, selected_dir, is_face) # todo one with aboveself.send_results(results, start, count)returnif image_text and use_clip:- results = process_with_clip_lock(run_search, self.object_finder, image_text, start+count, cam_name, selected_dir)+ results = add_to_queue(run_search, object_finder, image_text, start+count, cam_name, selected_dir)self.send_results(results, start, count)return@@ -1117,7 +1126,7 @@ def image_sort_key(item):try: return datetime.strptime(item["folder"], "%Y-%m-%d").timestamp() + item["timestamp"]except ValueError: return -1-def schedule_daily_restart(videocapture, restart_time):+def schedule_daily_restart(cam, restart_time):while True:now = datetime.now().time()target = time_obj(restart_time[0], restart_time[1])@@ -1127,10 +1136,12 @@ def schedule_daily_restart(videocapture, restart_time):delta = ((target.hour * 3600 + target.minute * 60) -(now.hour * 3600 + now.minute * 60 + now.second))time.sleep(delta)- videocapture.hls_proc[videocapture.cam_name].kill()- sys.stdout.flush()- python = sys.executable- os.execv(python, [python] + sys.argv)+ cams = database.run_get("links", None)+ for cam_name in cams.keys():+ cam.hls_proc[cam_name], cam.proc[cam_name] = cam._open_ffmpeg(cam_name)+ cam.current_stream_dir_raw[cam_name] = cam._get_new_stream_dir(cam_name)++def get_lan_ip():s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)@@ -1147,25 +1158,6 @@ def get_executable_args(): return ([sys.argv[0]], sys.argv[1:]) if getattr(sys,def event_img_info(image): return {"ts": int(float(image.split('_')[0])), "object_id":int(image.split('_')[1]), "class_id":int(image.split('_')[2])}-def start_cam(rtsp, cam_name, model_variant='t', yolo_res=960):- if not rtsp or not cam_name: return- def upsert_arg(args, key, value):- prefix = f"--{key}="- for i, arg in enumerate(args):- if arg.startswith(prefix):- args[i] = f"{prefix}{value}"- return args- return args + [f"{prefix}{value}"]-- executable, base_args = get_executable_args()- new_args = upsert_arg(base_args, "cam_name", cam_name)- new_args = upsert_arg(new_args, "rtsp", rtsp)- new_args = upsert_arg(new_args, "yolo_size", model_variant)- new_args = upsert_arg(new_args, "yolo_res", yolo_res)- new_args = upsert_arg(new_args, "use_clip", use_clip)- proc = subprocess.Popen(executable + new_args, close_fds=True)- active_subprocesses.append(proc)-live_link = dict()alerts_on = True@@ -1213,17 +1205,63 @@ def upload_to_r2(file_path: Path, signed_url: str, max_retries: int = 0) -> boolreturn Falseobject_finder_lock = threading.Lock()-def process_with_clip_lock(func, *args):- if not object_finder_lock.acquire(timeout=30): return None+import queue+task_queue = queue.Queue()+def add_to_queue(fn, *args):+ result_queue = queue.Queue(maxsize=1)+ task_queue.put((fn, args, result_queue))+ return result_queue.get()++def process_queue():try:- return_q = multiprocessing.Queue()- p = multiprocessing.Process(target=func, args=(return_q, *args))- p.start()- results = return_q.get(timeout=300)- p.join()- return results- finally:- object_finder_lock.release()+ fn, args, result_queue = task_queue.get_nowait()+ except queue.Empty: return+ result = fn(*args)+ result_queue.put(result)++def process_latest_face(img):+ if use_face and str(object_queue[0]).endswith("_0.jpg"):+ face_img = object_finder.img_to_face(img)++ if face_img is not None:+ cv2.imwrite(str(object_queue[0]).replace("/objects/", "/faces/"), face_img)+ date = object_queue[0].parent.name+ pkl_path = object_queue[0].parent.parent.parent / "faces" / date / "embeddings.pkl"+ face_emb = adaface_jit(object_finder.adaface, Tensor(face_img).contiguous()).numpy()+ data = pickle.load(open(pkl_path, "rb")) if pkl_path.exists() else {}+ if "embeddings" not in data: data["embeddings"] = {}+ data["embeddings"][str(object_queue[0])] = face_emb+ pkl_path.parent.mkdir(parents=True, exist_ok=True)+ pickle.dump(data, open(pkl_path, "wb"))++def clip_latest_img(img):+ if use_clip:+ img = object_finder.preprocess(img)+ data = pickle.load(open(object_queue[0].parent / 'embeddings.pkl', 'rb')) if os.path.exists(object_queue[0].parent / 'embeddings.pkl') else {}+ if "embeddings" not in data: data["embeddings"] = {}+ emb = precompute_embedding_jit_bs1(object_finder.model, Tensor(img).unsqueeze(0)).numpy()+ data["embeddings"][str(object_queue[0])] = emb+ with open(object_queue[0].parent / 'embeddings.pkl', "wb") as f: pickle.dump(data, f)++ if userID:+ cam_name = object_queue[0].parts[object_queue[0].parts.index("cameras")+1:object_queue[0].parts.index("objects")][0]+ alerts = database.run_get("alerts", cam_name) # todo, get cam_name from file path!+ for k, v in alerts.items():+ if time.time() - v.last_det < 60 or not v.is_active(): continue+ if v.desc is None: continue+ if not hasattr(v, "desc_emb") or v.desc_emb is None:+ v.desc_emb = run_encode_text(object_finder, v.desc)+ database.run_put("alerts", cam_name, v, id=k)++ similarity = (v.desc_emb @ emb.T).item()+ print("sim =",similarity,v.desc,object_queue[0])+ if similarity > v.threshold:+ send_notif(userID, f"Event Detected ({cam_name}: {v.desc})")+ alerts[k].last_det = time.time()+ database.run_put("alerts", cam_name, alerts[k], k)+ seen_time = event_img_info(str(object_queue[0]).split("/")[-1].split(".jpg")[0])["ts"]+ threading.Thread(target=export_and_upload, kwargs={"cam_name": cam_name, "thumbnail": object_queue[0], "userID": userID, "key": key, "start": seen_time, "length": 20, "wait": True}, daemon=True).start()+ breakcams = dict()active_subprocesses = []@@ -1239,25 +1277,16 @@ class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):database.run_put("max_storage", "all", 256)max_gb = database.run_get("max_storage", None)self.max_gb = max_gb["all"]- self.object_finder = ObjectFinder(prewarm=True, clip=use_clip, face=face) if (use_clip or face) else Noneself.object_finder_stop_event = threading.Event()self.object_finder_thread = Noneself._setup_cleanup_and_clip_thread()- def finish_request(self, request, client_address):- self.RequestHandlerClass(request, client_address, self, clip_instance=self.object_finder)-def _setup_cleanup_and_clip_thread(self):if self.cleanup_thread is None or not self.cleanup_thread.is_alive():self.cleanup_stop_event.clear()self.cleanup_thread = threading.Thread(target=self._cleanup_task, daemon=True, name="StorageCleanup")self.cleanup_thread.start()- if (use_clip or use_face) and (self.object_finder_thread is None or not self.object_finder_thread.is_alive()):- self.object_finder_stop_event.clear()- self.object_finder_thread = threading.Thread(target=self._objects_task, daemon=True, name="CLIPMaintenance")- self.object_finder_thread.start()-def _cleanup_task(self):while not self.cleanup_stop_event.is_set():try:@@ -1266,42 +1295,6 @@ class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):print(f"Cleanup error: {e}")self.cleanup_stop_event.wait(timeout=600)- def _objects_task(self):- while not self.object_finder_stop_event.is_set():- try:- object_folders = self.object_finder.find_object_folders("data/cameras")- for folder in object_folders:- name = folder.split("/")[2]- vod = is_vod(name)- if vod and name in database.run_get("analysis_prog", None) and database.run_get("analysis_prog", None)[name]["Tracking"] < 100: continue- embeddings, batch_paths = self.object_finder.precompute_embeddings(folder, vod=vod, database=database, cam_name=name, userID=userID, key=key)- alerts = database.run_get("alerts", name)- if userID:- for path, embedding in zip(batch_paths, embeddings):- for k, v in alerts.items():- if time.time() - v.last_det < 60 or not v.is_active(): continue- if v.desc is not None and hasattr(v, "desc_emb") and v.desc_emb is not None:- similarity = (v.desc_emb @ embedding.T).item()- print("sim =",similarity,v.desc,path)- if similarity > v.threshold:- send_notif(userID, f"Event Detected ({name}: {v.desc})")- alerts[k].last_det = time.time()- database.run_put("alerts", name, alerts[k], k)- seen_time = event_img_info(path.split("/")[-1].split(".jpg")[0])["ts"]- export_and_upload(cam_name=name, thumbnail=path, userID=userID, start=seen_time, length=20, key=key)- break-- if vod: database.run_delete("analysis_prog", folder.split("/")[2])- # todo, move to own loop- for k, alert in alerts.items():- if alert.desc is not None and alert.desc_emb is None:- alert.desc_emb = process_with_clip_lock(run_encode_text, self.object_finder, alert.desc)- database.run_put("alerts", name, alert, id=k)-- except Exception as e:- print(f"CLIP error: {e}")- self.object_finder_stop_event.wait(timeout=1)-def _check_and_cleanup_storage(self):total_size = sum(f.stat().st_size for f in (BASE_DIR / "cameras").glob('**/*') if f.is_file())size_gb = total_size / (1000 ** 3)@@ -1336,7 +1329,7 @@ class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):shutil.rmtree(oldest_recording)event_images_dir = largest_cam.with_name(largest_cam.name) / Path("event_images") / Path(oldest_recording.name)object_images_dir = largest_cam.with_name(largest_cam.name) / Path("objects") / Path(oldest_recording.name)- face_images_dir = largest_cam.with_name(largest_cam.name) / Path("objects") / Path(oldest_recording.name)+ face_images_dir = largest_cam.with_name(largest_cam.name) / Path("faces") / Path(oldest_recording.name)#dets_dir = largest_cam.with_name(largest_cam.name) / Path("dets") / Path(oldest_recording.name)if event_images_dir.exists(): shutil.rmtree(event_images_dir)if object_images_dir.exists(): shutil.rmtree(object_images_dir)@@ -1395,28 +1388,26 @@ if __name__ == "__main__":sys.exit(1)if use_clip or use_face:- from objects import ObjectFinder+ from objects import ObjectFinder, precompute_embedding_jit_bs1, adaface_jit+ object_queue = []cam_name = next((arg.split("=", 1)[1] for arg in sys.argv[1:] if arg.startswith("--cam_name=")), "my_camera")live_link = dict()- if url is None:- for cam_name in cams.keys():- start_cam(rtsp=cams[cam_name],cam_name=cam_name,model_variant=model_variant, yolo_res=yolo_res)-class_labels = fetch('https://raw.githubusercontent.com/pjreddie/darknet/master/data/coco.names').read_text().split("\n")color_dict = {label: tuple((((i+1) * 50) % 256, ((i+1) * 100) % 256, ((i+1) * 150) % 256)) for i, label in enumerate(class_labels)}cam = None- if url:- model = YOLOv9(models[int(model_variant)], res=int(yolo_res)) if int(model_variant) < 6 else RFDETR(models[int(model_variant)])- #model = RFDETR("small")- cam = VideoCapture(url,cam_name=cam_name, vod=is_file)- vod = url.endswith(('.mp4', '.avi', '.mov', '.mkv', '.webm'))++ model = YOLOv9(models[int(model_variant)], res=int(yolo_res)) if int(model_variant) < 6 else RFDETR(models[int(model_variant)])+ object_finder = ObjectFinder(clip=use_clip, face=use_face) if (use_clip or use_face) else None+ #model = RFDETR("small")+ cam = VideoCapture()try:try:server = ThreadedHTTPServer(('0.0.0.0', 8080), use_clip=use_clip, face=use_face, RequestHandlerClass=HLSRequestHandler)+ threading.Thread(target=server.serve_forever, daemon=True).start()print(f"Serving at http://{get_lan_ip()}:8080")except OSError as e:if e.errno == socket.errno.EADDRINUSE:@@ -1425,22 +1416,20 @@ if __name__ == "__main__":else:raise- if url:- restart_time = (0, 0)- threading.Thread(- target=schedule_daily_restart,- args=(cam, restart_time),- daemon=True- ).start()-- if server:- threading.Thread(target=server.serve_forever(), daemon=True).start()- if cam is not None:- cam.start(cam.cam_name)- else:- while True: time.sleep(3600)++ restart_time = (0, 0)+ threading.Thread(+ target=schedule_daily_restart,+ args=(cam, restart_time),+ daemon=True+ ).start()++ cam.start()+ while True: time.sleep(3600)except KeyboardInterrupt:if url:cam.release(cam.cam_name) # todo, needed?server.shutdown()++diff --git a/detection/rfdetr.py b/detection/rfdetr.pyindex cfb8d69..543cfa1 100644--- a/detection/rfdetr.py+++ b/detection/rfdetr.py@@ -602,8 +602,17 @@ class RFDETR():self.backbone.encoder.encoder.layer[i].mlp.fc2 = nn.Linear(1536, 384)state_dict = safe_load(fetch(f'https://huggingface.co/roryclear/rf-detr/resolve/main/{name}.safetensors'))load_state_dict(self, state_dict)+ self.jit_cache = {}++ # todo, this should be more generic and not in both here and yolo+ def jit_infer(self, frame):+ shape = tuple(frame.shape)+ if shape not in self.jit_cache:+ @TinyJit+ def fn(im, model): return model(im)+ self.jit_cache[shape] = fn+ return self.jit_cache[shape](frame, self)- @TinyJitdef __call__(self, frame):pre = self.preprocess(frame)predictions = self.predict(pre)diff --git a/detection/yolov9.py b/detection/yolov9.pyindex 0d298cf..838f8c1 100644--- a/detection/yolov9.py+++ b/detection/yolov9.py@@ -368,11 +368,19 @@ class YOLOv9():self.model[39] = ADown(ch0=256)self.model[40] = Concat(f=[-1, 29])self.model[41] = RepNCSPELAN4(1024, 256, 512, n=2)- self.model[42] = DDetect(a=256, b=512, c=512, d=256, f=[35, 38, 41])+ self.model[42] = DDetect(a=256, b=512, c=512, d=256, f=[35, 38, 41])state_dict = safe_load(fetch(f'https://huggingface.co/roryclear/yolov9/resolve/main/yolov9-{size}.safetensors'))load_state_dict(self, state_dict)+ self.jit_cache = {}++ def jit_infer(self, frame):+ shape = tuple(frame.shape)+ if shape not in self.jit_cache:+ @TinyJit+ def fn(im, model): return model(im)+ self.jit_cache[shape] = fn+ return self.jit_cache[shape](frame, self)- @TinyJitdef __call__(self, frame):pre = self.preprocess(frame)x = pre.unsqueeze(0)diff --git a/objects.py b/objects.pyindex 3fd98b5..43b6d93 100644--- a/objects.py+++ b/objects.py@@ -145,7 +145,7 @@ def encode_text(model, text):return x / (x * x).sum(axis=-1, keepdim=True).sqrt()class ObjectFinder:- def __init__(self, prewarm=False, base_path="data/cameras", clip=False, face=False):+ def __init__(self, base_path="data/cameras", clip=False, face=False):self.base_path = base_pathself.image_embeddings = {}self.face_embeddings = {}@@ -159,14 +159,6 @@ class ObjectFinder:if self.face:self.blazeface = BlazeFace()self.adaface = ADAFACE()-- # prewarm- if prewarm:- if self.face:- blazeface_jit(self.blazeface, Tensor.rand((640, 640, 3)).cast(dtype=dtypes.uchar))- adaface_jit(self.adaface, Tensor.rand((112, 112, 3)).cast(dtype=dtypes.uchar))- if self.clip:- precompute_embeddings_jit(self.model, Tensor.rand((1, 3, 224, 224), dtype=dtypes.float32))def find_object_folders(self, base_path="data/cameras"):object_folders = []@@ -181,70 +173,18 @@ class ObjectFinder:if os.path.isdir(date_path):object_folders.append(date_path)return object_folders- # db for progress- def precompute_embeddings(self, folder_path, batch_size=16, vod=False, database=None, cam_name=None, userID=None, key=None):- folder_embeddings, folder_paths = get_embeddings(folder_path, "embeddings.pkl")-- current_images = {- os.path.join(folder_path, f)- for f in os.listdir(folder_path)- if f.lower().endswith((".png", ".jpg", ".jpeg"))- }-- if self.face:- folder_embeddings_face, folder_paths_face = get_embeddings(folder_path.replace("objects", "faces"), "embeddings.pkl")- cached_images_face = set(folder_embeddings_face.keys())- new_images_face = current_images - cached_images_face- deleted_images_face = cached_images_face - current_images-- for img in deleted_images_face:- folder_embeddings_face.pop(img, None)- folder_embeddings_face.pop(img, None)- new_image_list_face = list(new_images_face)--- cached_images = set(folder_embeddings.keys())- new_images = current_images - cached_images- deleted_images = cached_images - current_images-- for img in deleted_images:- folder_embeddings.pop(img, None)- folder_paths.pop(img, None)-- if not new_images: return [], []- new_image_list = list(new_images)- if self.face:- face_paths, face_embeddings = self.process_faces(new_image_list_face)- for path, emb in zip(face_paths, face_embeddings):- folder_embeddings_face[path] = emb- folder_paths_face[path] = path- save_embeddings(folder_path.replace("objects", "faces"), "embeddings.pkl", folder_embeddings_face, folder_paths_face)- if not self.clip: return [], []- emb_ret = []- path_ret = []- for i in range(len(new_image_list)):- img_path = new_image_list[i]-- img = cv2.imread(img_path)- if img is None: continue- img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)- img = preprocess(img)-- emb = precompute_embedding_jit_bs1(self.model, Tensor([img])).numpy()- folder_embeddings[img_path] = emb- folder_paths[img_path] = img_path- emb_ret.append(emb)- path_ret.append(img_path)- if vod: database.run_put("analysis_prog", cam_name, {"Processing":((i + 1)/len(new_image_list))*100})- if (i+1) % 10 == 0 and i > 0: print(f"Processed {(i+1)} / {len(new_image_list)} new images...")- print(f"Processed {len(new_image_list)} new images...")- save_embeddings(folder_path, "embeddings.pkl", folder_embeddings, folder_paths)- return emb_ret, path_retdef precompute_embedding_bs1_np(self, img): return precompute_embedding_jit_bs1(self.model, Tensor(img)).numpy() # todo removedef precompute_face_embedding_bs1_np(self, img): return adaface_jit(self.adaface, Tensor(img)).numpy() # todo remove+ def preprocess(self, img):+ img = cv2.resize(img, (224, 224), interpolation=cv2.INTER_CUBIC)+ img = img.astype(np.float32) / 255.0+ img = (img - 0.5) / 0.5+ img = np.transpose(img, (2, 0, 1))+ return img+def preprocess_clip(self, img):if type(img) == bytes:img = cv2.imdecode(np.frombuffer(img, np.uint8), cv2.IMREAD_COLOR)@@ -252,7 +192,7 @@ class ObjectFinder:img = f"data/cameras{img}"img = cv2.imread(img)img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)- return [preprocess(img)]+ return [self.preprocess(img)]def preprocess_face(self, img):if type(img) == bytes: # todo dup of above@@ -352,34 +292,11 @@ class ObjectFinder:face_img = cv2.warpAffine(rotated, transform_mat, (112, 112))face_img = cv2.cvtColor(face_img, cv2.COLOR_RGB2BGR)- # Debug draw dots for the eyes at target positions- #cv2.circle(face_img, (38, 51), 2, (0, 255, 0), -1)- #cv2.circle(face_img, (73, 51), 2, (0, 255, 0), -1)return face_imgreturn None- def process_faces(self, paths):- ret_paths = []- ret_embeddings = []- for path in paths:- if path.endswith("_0.jpg"): # person- orig = cv2.imread(path)- orig = cv2.cvtColor(orig, cv2.COLOR_BGR2RGB)- face_img = self.img_to_face(orig)- if face_img is None:- if not self.clip: os.remove(path)- ret_embeddings.append(None) # todo hack for now- ret_paths.append(path)- continue- cv2.imwrite(path.replace("objects","faces"), face_img)- embeddings = adaface_jit(self.adaface, Tensor(face_img)).numpy()- ret_embeddings.append(embeddings)- ret_paths.append(path)-- return ret_paths, ret_embeddings-def search(self, query=None, top_k=10, cam_name=None, timestamp=None, text_embedding=None, is_face=False):embeddings = self.face_embeddings if is_face else self.image_embeddingsif not embeddings:@@ -416,32 +333,11 @@ class ObjectFinder:results.sort(key=lambda x: x[1], reverse=True)return results[:top_k]-- def _load_single_embeddings_file(self, cache_file):- try:- with open(cache_file, "rb") as f:- cache = pickle.load(f)-- folder_embeddings = cache.get("embeddings", {})- folder_paths = cache.get("paths", {})-- self.image_embeddings.update(folder_embeddings)- self.image_paths.update(folder_paths)-- loaded_count = len(folder_embeddings)- print(f"Loaded {loaded_count} embeddings from {cache_file}")- return loaded_count-- except Exception as e:- print(f"Error loading {cache_file}: {e}")- return 0-def _load_all_embeddings(self, face=False):total_loaded = 0valid_paths = set()cache_filename = "embeddings.pkl"target_embeddings = self.face_embeddings if face else self.image_embeddings- target_paths = self.face_paths if face else self.image_pathsfor camera_folder in os.listdir(self.base_path):camera_path = os.path.join(self.base_path, camera_folder)@@ -456,15 +352,11 @@ class ObjectFinder:with open(cache_file, "rb") as f:cache = pickle.load(f)folder_embeddings = cache.get("embeddings", {})- folder_paths = cache.get("paths", {})valid_paths.update(folder_embeddings.keys())target_embeddings.update(folder_embeddings)- target_paths.update(folder_paths)total_loaded += len(folder_embeddings)stale_keys = set(target_embeddings.keys()) - valid_paths- for k in stale_keys:- del target_embeddings[k]- target_paths.pop(k, None)+ for k in stale_keys: del target_embeddings[k]if face:self.face_embeddings = target_embeddings@@ -473,22 +365,6 @@ class ObjectFinder:print(f"\nTotal {'face' if face else 'image'} embeddings loaded: {total_loaded}")-def get_embeddings(path, filename):- cache_file = os.path.join(path, filename)- folder_embeddings = {}- folder_paths = {}- if os.path.exists(cache_file):- with open(cache_file, "rb") as f:- cache = pickle.load(f)- folder_embeddings = cache.get("embeddings", {})- folder_paths = cache.get("paths", {})- return folder_embeddings, folder_paths--def save_embeddings(folder_path, file_name, folder_embeddings, folder_paths):- cache_file = os.path.join(folder_path, file_name)- os.makedirs(os.path.dirname(cache_file), exist_ok=True)- with open(cache_file, "wb") as f: pickle.dump({"embeddings": folder_embeddings, "paths": folder_paths}, f)-@TinyJitdef precompute_embeddings_jit(model, x): return precompute_embedding(model, x)@@ -540,11 +416,4 @@ def precompute_embedding(model, x):image_embeds = x[:, 0, :]embeddings = image_embeds @ model.projembeddings = embeddings / (embeddings.pow(2).sum(axis=-1, keepdim=True).sqrt() + 1e-8)- return embeddings--def preprocess(img):- img = cv2.resize(img, (224, 224), interpolation=cv2.INTER_CUBIC)- img = img.astype(np.float32) / 255.0- img = (img - 0.5) / 0.5- img = np.transpose(img, (2, 0, 1))- return img\ No newline at end of file+ return embeddings\ No newline at end of filediff --git a/test/run_clip.py b/test/run_clip.pyindex 60faa7f..ad89624 100644--- a/test/run_clip.py+++ b/test/run_clip.py@@ -1,5 +1,5 @@from tinygrad import Tensor, TinyJit-from objects import ObjectFinder, precompute_embeddings_jit, precompute_embedding_jit_bs1, preprocess+from objects import ObjectFinder, precompute_embeddings_jit, precompute_embedding_jit_bs1import timeimport numpy as npimport sysdiff --git a/test/test_clip.py b/test/test_clip.pyindex f222420..b1feda2 100644--- a/test/test_clip.py+++ b/test/test_clip.py@@ -1,4 +1,4 @@-from objects import ObjectFinder, precompute_embeddings_jit, precompute_embedding_jit_bs1, preprocess+from objects import ObjectFinder, precompute_embeddings_jit, precompute_embedding_jit_bs1import numpy as npfrom tinygrad import Tensorimport time@@ -8,7 +8,7 @@ import sysdef test_clip_outputs():img1 = cv2.imread("test/clip_images/f40.jpg")- img1 = preprocess(img1)+ img1 = clip.preprocess(img1)emb1 = precompute_embedding_jit_bs1(clip.model, Tensor(img1).unsqueeze(0))emb1_text = clip.model._encode_text("ferrari f40")sim = (emb1_text @ emb1.T).numpy()[0]@@ -16,8 +16,8 @@ def test_clip_outputs():if __name__ == "__main__":# todo sepatate test for egpu?- clip = ObjectFinder(clip=True)- test_clip_outputs()+ #clip = ObjectFinder(clip=True)+ #test_clip_outputs()for i in range(8): subprocess.run([sys.executable, "test/run_clip.py", str(2**i)], check=True)diff --git a/utils/helpers.py b/utils/helpers.pyindex 0419990..e312f15 100644--- a/utils/helpers.py+++ b/utils/helpers.py@@ -192,7 +192,8 @@ def export_clip(stream_dir, output_path: Path, live=False, length=5, end=0, starfile_size = len(file_data)comp += 5-def export_and_upload(cam_name, thumbnail, userID, key, start=None, end=0, length=20):+def export_and_upload(cam_name, thumbnail, userID, key, start=None, end=0, length=20, wait=False):+ if wait: time.sleep(10) # todo, objects can be too far aheados.makedirs(BASE_DIR / "cameras" / cam_name / "event_clips", exist_ok=True)mp4_filename = BASE_DIR / "cameras" / f"{cam_name}/event_clips/{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.mp4"temp_output = BASE_DIR / "cameras" / f"{cam_name}/event_clips/{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}_temp.mp4"@@ -200,7 +201,7 @@ def export_and_upload(cam_name, thumbnail, userID, key, start=None, end=0, lengtsubprocess.run(['ffmpeg', '-i', mp4_filename, '-i', str(thumbnail), '-map', '0', '-map', '1', '-c', 'copy', '-disposition:v:1', 'attached_pic', '-y', temp_output])os.replace(temp_output, mp4_filename)encrypt_file(Path(mp4_filename), Path(f"""{mp4_filename}.aes"""), key)- threading.Thread(target=upload_file, args=(Path(f"""{mp4_filename}.aes"""), userID), daemon=True).start()+ upload_file(Path(f"{mp4_filename}.aes"), userID)os.unlink(mp4_filename)def find_ffmpeg():