Add object detection, tracking, mobile notifications, and search to any security camera.
Egpu4 (#147)
Rory Cleare37805cf6baa
diff --git a/clearcam.py b/clearcam.pyindex 3881a7c..ee93822 100644--- a/clearcam.py+++ b/clearcam.py@@ -167,28 +167,31 @@ class VideoCapture:self.vod = vodself.output_dir_raw = BASE_DIR / "cameras" / f'{cam_name}' / "streams"# objects in scene count- self.counter = RollingClassCounter(cam_name=cam_name, window_seconds=float('inf'))+ self.counter = {}self.cam_name = cam_nameself.object_set = set()self.object_set_zone = set()- self.src = src+ self.src = {}self.max_frame_rate = 10 # for vod onlyself.width, self.height = _get_stream_resolution(src)- self.proc = None- self.hls_proc = None+ self.proc = {}+ self.hls_proc = {}self.running = Trueself.raw_frame = Noneself.annotated_frame = Noneself.last_preds = []- self.last_frame = None+ self.last_frame = {}self.settings = None#self.last_shapes_time = time.time()#self.det_shapes = []- self.new_frame_event = threading.Event()++ self.counter[self.cam_name] = RollingClassCounter(cam_name=cam_name, window_seconds=float('inf'))+ self.src[self.cam_name] = src # todo+ self.last_frame[self.cam_name] = Noneself.alert_counters = database.run_get("alerts",self.cam_name)if not self.alert_counters:@@ -199,9 +202,8 @@ class VideoCapture:self.lock = threading.Lock()- if not self.vod or not self.output_dir_raw.exists():- self._open_ffmpeg()- threading.Thread(target=self.capture_loop, daemon=True).start()+ if not self.vod or not self.output_dir_raw.exists(): threading.Thread(target=self.capture_loop, daemon=True).start()+ if not self.vod: threading.Thread(target=self.inference_loop, daemon=True).start()def _get_new_stream_dir(self):timestamp = "video" if self.vod else datetime.now().strftime("%Y-%m-%d")@@ -220,18 +222,19 @@ class VideoCapture:except Exception:pass- def _open_ffmpeg(self):+ def _open_ffmpeg(self, cam_name):path = self._get_new_stream_dir()- self._safe_kill_process(self.proc)- self._safe_kill_process(self.hls_proc)+ if self.cam_name in self.proc: self._safe_kill_process(self.proc[self.cam_name])+ if self.cam_name in self.hls_proc: self._safe_kill_process(self.hls_proc[self.cam_name])+ src = self.src[cam_name]ffmpeg_path = find_ffmpeg()- is_rtsp = self.src.startswith("rtsp")+ is_rtsp = src.startswith("rtsp")if self.vod:command = [ffmpeg_path,- "-i", self.src,+ "-i", src,"-c:v", "copy","-an","-f", "hls",@@ -243,8 +246,7 @@ class VideoCapture:"-hls_segment_filename", str(path / "seg_%06d.m4s"),str(path / "stream.m3u8"),]- self.hls_proc = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)- self.proc = None+ return subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL), Noneelse: # Live streams# Original live stream pipeline@@ -253,7 +255,7 @@ class VideoCapture:*(["-rtsp_transport", "tcp"] if is_rtsp else []),"-fflags", "+genpts","-avoid_negative_ts", "make_zero",- "-i", self.src,+ "-i", src,"-c", "copy","-an","-f", "hls",@@ -264,7 +266,7 @@ class VideoCapture:"-hls_segment_filename", str(path / "stream_%06d.ts"),str(path / "stream.m3u8")]- self.hls_proc = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)+ hls_proc = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)self.start_time = time.time()time.sleep(15)@@ -290,9 +292,9 @@ class VideoCapture:"-threads", "1","-"]- self.proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)+ return hls_proc, subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)- def save_object(self, p, ts=0):+ def save_object(self, p, ts=0, cam_name=None):p = np.array([p.tlwh[0],p.tlwh[1],(p.tlwh[0]+p.tlwh[2]),(p.tlwh[1]+p.tlwh[3]),p.score,p.class_id,p.track_id])timestamp = "video" if self.vod else datetime.now().strftime("%Y-%m-%d")filepath = BASE_DIR / "cameras" / f"{self.cam_name}/objects/{timestamp}"@@ -310,13 +312,13 @@ class VideoCapture:x2_new = cx + hwy1_new = cy - hhy2_new = cy + hh- H, W = self.last_frame.shape[:2]+ H, W = self.last_frame[cam_name].shape[:2]x1_new = max(0, min(x1_new, W))x2_new = max(0, min(x2_new, W))y1_new = max(0, min(y1_new, H))y2_new = max(0, min(y2_new, H))if (y2_new - y1_new) < 100 or (x2_new - x1_new) < 100: return # too small- crop = self.last_frame[y1_new:y2_new, x1_new:x2_new]+ crop = self.last_frame[cam_name][y1_new:y2_new, x1_new:x2_new]cv2.imwrite(str(object_filename), crop)@@ -332,44 +334,46 @@ class VideoCapture:self.pred_occs = {}self.last_link_check = time.time()count = 0- self.det_path = BASE_DIR / "cameras" / self.cam_name / "dets" / datetime.now().strftime("%Y-%m-%d")- Path(self.det_path).mkdir(parents=True, exist_ok=True)- self.det_manifest = str(self.det_path / "det_manifest.txt")- if self.vod:- self.cap = cv2.VideoCapture(self.src)- self.src_fps = self.cap.get(cv2.CAP_PROP_FPS) or 30- self.frame_step = max(1, int(round(self.src_fps / self.max_frame_rate)))+ self.cap = {}+ self.src_fps = {}+ cam_name = self.cam_name+ self.pred_occs[cam_name] = {}+ self.hls_proc[cam_name], self.proc[cam_name] = self._open_ffmpeg(cam_name)+while self.running:- if time.time() - last_live_check > 5:+ if time.time() - last_live_check > 5: # todo, for all camslast_live_check = time.time()- link = database.run_get("links", self.cam_name)+ link = database.run_get("links", cam_name)if type(link) == list: link = link[0] # todo, flakey?- if link != self.src:- self.src = link- self._open_ffmpeg()+ if link != self.src[cam_name]:+ self.src[cam_name] = link+ self.hls_proc[cam_name], self.proc[cam_name] = self._open_ffmpeg(cam_name)try:- if not (BASE_DIR / "cameras" / self.cam_name).is_dir(): os._exit(1) # deleted cam+ if not (BASE_DIR / "cameras" / cam_name).is_dir(): os._exit(1) # deleted camif self.vod:- for _ in range(self.frame_step - 1):- self.cap.grab() # skip for max fps- ret, frame = self.cap.read()- self.last_frame = frame #todo- if not ret or self.cam_name not in database.run_get("links", None):+ if cam_name not in self.cap:+ self.cap[cam_name] = cv2.VideoCapture(self.src[cam_name])+ self.src_fps[cam_name] = self.cap[cam_name].get(cv2.CAP_PROP_FPS) or 30++ self.cap[cam_name].grab() # skip for max fps+ ret, frame = self.cap[cam_name].read()+ self.last_frame[self.cam_name] = frame #todo+ if not ret or cam_name not in database.run_get("links", None):self.running = Falsedatabase.run_put("analysis_prog", cam_name, {"Tracking":100})os._exit(0)else:- self.last_preds, _ = self.run_inference(frame)- database.run_put("analysis_prog", cam_name, {"Tracking":self.cap.get(cv2.CAP_PROP_POS_FRAMES)/self.cap.get(cv2.CAP_PROP_FRAME_COUNT)*100})+ self.last_preds, _ = self.run_inference(frame, cam_name=cam_name)+ database.run_put("analysis_prog", cam_name, {"Tracking":self.cap[cam_name].get(cv2.CAP_PROP_POS_FRAMES)/self.cap[cam_name].get(cv2.CAP_PROP_FRAME_COUNT)*100})else:- raw_bytes = self.proc.stdout.read(frame_size)+ raw_bytes = self.proc[cam_name].stdout.read(frame_size)if len(raw_bytes) != frame_size:fail_count += 1if fail_count > 5:- print(f"{self.cam_name} FFmpeg frame read failed (count={fail_count}), restarting stream...{self.src}")- self._open_ffmpeg()+ print(f"{cam_name} FFmpeg frame read failed (count={fail_count}), restarting stream...{self.src[cam_name]}")+ self.hls_proc[cam_name], self.proc[cam_name] = self._open_ffmpeg(cam_name)fail_count = 0time.sleep(0.5)continue@@ -396,9 +400,9 @@ class VideoCapture:timestamp = "video" if self.vod else datetime.now().strftime("%Y-%m-%d")filepath = BASE_DIR / "cameras" / f"{self.cam_name}/event_images/{timestamp}"filepath.mkdir(parents=True, exist_ok=True)- annotated_frame = draw_predictions(self.last_frame.copy(), filtered_preds, color_dict)+ annotated_frame = draw_predictions(self.last_frame[cam_name].copy(), filtered_preds, color_dict)# todo alerts can be sent with the wrong thumbnail if two happen quickly, use map- ts = int(self.cap.get(cv2.CAP_PROP_POS_FRAMES) / self.src_fps) - 5 if self.vod else int(time.time() - self.streamer.start_time - 5)+ ts = int(self.cap[self.cam_name].get(cv2.CAP_PROP_POS_FRAMES) / self.src_fps[self.cam_name]) - 5 if self.vod else int(time.time() - self.start_time - 5)filename = filepath / f"{ts}_notif.jpg" if alert.is_notif else filepath / f"{ts}.jpg"if not self.vod: cv2.imwrite(str(filename), annotated_frame, [cv2.IMWRITE_JPEG_QUALITY, 85]) # we've 10MB limit for video file, raw png is 3MB!if (plain := filepath / f"{ts}.jpg").exists() and (filepath / f"{ts}_notif.jpg").exists():@@ -420,9 +424,9 @@ class VideoCapture:counters = database.run_get("counters", self.cam_name)if counters not in [None, {}]:if counters.reset:- self.counter.reset_counts()- self.counter.reset = False- database.run_put("counters", cam_name, self.counter)+ self.counter[self.cam_name].reset_counts()+ self.counter[self.cam_name].reset = False+ database.run_put("counters", cam_name, self.counter[self.cam_name])alerts = database.run_get("alerts", self.cam_name)for id,a in alerts.items():@@ -446,7 +450,7 @@ class VideoCapture:if userID and not self.vod and self.cam_name in live_link and live_link[self.cam_name] and (time.time() - last_live_seg) >= 4:last_live_seg = time.time()mp4_filename = f"segment.mp4"- export_clip(self.streamer.current_stream_dir_raw, Path(mp4_filename), live=True)+ export_clip(self.current_stream_dir_raw, Path(mp4_filename), live=True)encrypt_file(Path(mp4_filename), Path(f"""{mp4_filename}.aes"""), key)Path(mp4_filename).unlink()threading.Thread(target=upload_to_r2, args=(Path(f"""{mp4_filename}.aes"""), live_link[self.cam_name]), daemon=True).start()@@ -454,42 +458,58 @@ class VideoCapture:count+=1with self.lock:self.raw_frame = frame.copy()- self.new_frame_event.set()-- if not self.vod:- time.sleep(1 / 30)- prev_time = time.time()- self.new_frame_event.wait()- self.new_frame_event.clear()- if not any(counter.is_active() for _, counter in self.alert_counters.items()): # don't run inference when no active scheds- time.sleep(1)- with self.lock: self.last_preds = [] # to remove annotation when no alerts active- continue- with self.lock:- frame = self.raw_frame.copy() if self.raw_frame is not None else None- if frame is not None:- preds, frame = self.run_inference(frame)- with self.lock:- self.last_preds = preds.copy()- self.last_frame = frame.numpy().copy()-- curr_time = time.time()- fps = 1 / (curr_time - prev_time)- prev_time = curr_time- print(f"\rFPS: {fps:.2f}", end="", flush=True)+ if not self.vod: time.sleep(1 / 30)except Exception as e:print("Error in capture_loop:", e, self.cam_name)- self._open_ffmpeg()+ self._open_ffmpeg(self.cam_name)time.sleep(1)+ def inference_loop(self):+ prev_time = time.time()+ while self.running:+ time.sleep(1/30) # todo+ if not any(counter.is_active() for _, counter in self.alert_counters.items()): # don't run inference when no active scheds+ time.sleep(1)+ with self.lock: self.last_preds = [] # to remove annotation when no alerts active+ continue+ with self.lock:+ frame = self.raw_frame.copy() if self.raw_frame is not None else None+ if frame is not None:+ preds, frame = self.run_inference(frame, cam_name=self.cam_name)+ with self.lock:+ self.last_preds = preds.copy()+ self.last_frame[self.cam_name] = frame.numpy().copy()++ '''+ if len(preds) > 0 and time.time() - self.last_shapes_time >= 1 / 24:+ h, w = frame.shape[:2]+ preds[:, [0, 2]] /= w+ preds[:, [1, 3]] /= h+ t = time.time() - self.start_time+ self.det_shapes.append({t: preds.tolist()})+ if len(self.det_shapes) == 1:+ with open(self.det_manifest, 'a') as f:+ f.write(f"{t:.3f}: {self.shape_seg}.json\n")+ if time.time() - self.last_shapes_time >= 4:+ print("4 secs")+ self.last_shapes_time = time.time()+ json.dump(self.det_shapes, open(self.det_path / f"{self.shape_seg}.json", "w"))+ self.shape_seg += 1+ self.det_shapes = []+ '''+ curr_time = time.time()+ fps = 1 / (curr_time - prev_time)+ prev_time = curr_time+ print(f"\rFPS: {fps:.2f}", end="", flush=True)+def reset_vod(self):- self.cap = cv2.VideoCapture(self.src) # reset video on settings change+ self.cap[self.cam_name] = cv2.VideoCapture(self.src[self.cam_name]) # reset video on settings changeshutil.rmtree(BASE_DIR / "cameras" / self.cam_name / "objects", ignore_errors=True)shutil.rmtree(BASE_DIR / "cameras" / self.cam_name / "faces", ignore_errors=True)shutil.rmtree(BASE_DIR / "cameras" / self.cam_name / "event_images", ignore_errors=True)- def run_inference(self, frame):+ def run_inference(self, frame, cam_name):frame = Tensor(frame)preds = model(frame).numpy()thresh = (self.settings.get("threshold") if self.settings else 0.5) or 0.5 #todo clean!@@ -501,11 +521,11 @@ class VideoCapture:for x in online_targets:if x.tracklet_len < 1: continue # dont alert for 1 frame, too many false positives.# add to objects, regarless of speed- if x.track_id not in self.pred_occs: self.pred_occs[x.track_id] = [time.time()]- if (len(self.pred_occs[x.track_id]) < 20 and (time.time() - self.pred_occs[x.track_id][-1]) > 1) or (time.time() - self.pred_occs[x.track_id][-1]) > 10:- self. pred_occs[x.track_id].append(time.time())- ts = round((self.cap.get(cv2.CAP_PROP_POS_FRAMES) / self.src_fps) - 5,1) if self.vod else round((time.time() - self.streamer.start_time - 5),1)- self.save_object(x, ts)+ if x.track_id not in self.pred_occs[cam_name]: self.pred_occs[cam_name][x.track_id] = [time.time()]+ if (len(self.pred_occs[cam_name][x.track_id]) < 20 and (time.time() - self.pred_occs[cam_name][x.track_id][-1]) > 1) or (time.time() - self.pred_occs[cam_name][x.track_id][-1]) > 10:+ self. pred_occs[cam_name][x.track_id].append(time.time())+ ts = round((self.cap[cam_name].get(cv2.CAP_PROP_POS_FRAMES) / self.src_fps[cam_name]) - 5,1) if self.vod else round((time.time() - self.start_time - 5),1)+ self.save_object(x, ts, cam_name=cam_name)if x.speed < 2.5: continue #min speed, don't detect still objects, they jitter too. # TODO what's the best min value?outside = False@@ -527,7 +547,7 @@ class VideoCapture:new_in_zone = int(x.track_id) not in self.object_set_zone and not outsideif new:self.object_set.add(int(x.track_id))- self.counter.add(int(x.class_id))+ self.counter[self.cam_name].add(int(x.class_id))if new_in_zone: self.object_set_zone.add(int(x.track_id))for _, alert in self.alert_counters.items():if not alert.get_counts()[1] and ((new and not alert.zone) or (new_in_zone and alert.zone)): alert.add(int(x.class_id))@@ -542,10 +562,8 @@ class VideoCapture:def release(self):self.running = False- if self.proc:- self.proc.kill()- if self.hls_proc:- self.hls_proc.kill()+ if self.cam_name in self.proc: self.proc[self.cam_name].kill()+ if self.cam_name in self.hls_proc: self.hls_proc[self.cam_name].kill()def is_bright_color(color):r, g, b = color@@ -564,51 +582,6 @@ def draw_predictions(frame, preds, color_dict):cv2.putText(frame, label, (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, font_color, 1, cv2.LINE_AA)return frame-class HLSStreamer:- def __init__(self, video_capture, output_dir="streams", segment_time=4, cam_name="camera", vod=False):- self.cam_name = cam_name- self.cam = video_capture- self.output_dir_raw = BASE_DIR / "cameras" / self.cam_name / output_dir- self.segment_time = segment_time- self.running = False- self.start_time = time.time()- self.feeding_frames = False- self._stop_event = threading.Event()- self.output_dir_raw.mkdir(parents=True, exist_ok=True)- self.vod = vod-- def _get_new_stream_dir(self):- timestamp = "video" if self.vod else datetime.now().strftime("%Y-%m-%d")- stream_dir_raw = self.output_dir_raw / timestamp- stream_dir_raw.mkdir(exist_ok=True)- return stream_dir_raw-- def start(self):- self.running = True- self.current_stream_dir_raw = self._get_new_stream_dir()- self.start_time = time.time()-- def _safe_restart(self):- try:- self.stop()- except Exception as e:- print(f"Error during stop: {e}")- time.sleep(2)- while True: # 3 second timeout- if self.cam.get_frame() is not None:- break- time.sleep(1)- try:- self.start()- print("HLS streamer restarted successfully")- except Exception as e:- print(f"Failed to restart HLS: {e}")--- def stop(self):- self.running = False--def point_not_in_polygon(coords, poly):n = len(poly)for j in range(len(coords)):@@ -1128,7 +1101,7 @@ def image_sort_key(item):try: return datetime.strptime(item["folder"], "%Y-%m-%d").timestamp() + item["timestamp"]except ValueError: return -1-def schedule_daily_restart(hls_streamer, videocapture, restart_time):+def schedule_daily_restart(videocapture, restart_time):while True:now = datetime.now().time()target = time_obj(restart_time[0], restart_time[1])@@ -1426,8 +1399,6 @@ if __name__ == "__main__":#model = RFDETR("small")cam = VideoCapture(url,cam_name=cam_name, vod=is_file)vod = url.endswith(('.mp4', '.avi', '.mov', '.mkv', '.webm'))- hls_streamer = HLSStreamer(cam,cam_name=cam_name, vod=vod)- cam.streamer = hls_streamertry:try:@@ -1441,11 +1412,10 @@ if __name__ == "__main__":raiseif url:- hls_streamer.start()restart_time = (0, 0)threading.Thread(target=schedule_daily_restart,- args=(hls_streamer, cam, restart_time),+ args=(cam, restart_time),daemon=True).start()@@ -1456,7 +1426,6 @@ if __name__ == "__main__":except KeyboardInterrupt:if url:- hls_streamer.stop()cam.release()server.shutdown()