Add object detection, tracking, mobile notifications, and search to any security camera.
Db (#107)
Rory Clear5f1b475a52e5
diff --git a/clearcam.py b/clearcam.pyindex 1c901ff..9029d72 100644--- a/clearcam.py+++ b/clearcam.py@@ -1,15 +1,13 @@from tinygrad.nn import Conv2d, BatchNorm2dfrom tinygrad.tensor import Tensorfrom tinygrad import TinyJit-from tinygrad.device import is_dtype_supported-from tinygrad import dtypes+from tinygrad.helpers import fetchimport numpy as npfrom itertools import chainfrom pathlib import Pathimport cv2from collections import defaultdict, dequeimport time, sys-from tinygrad.helpers import fetchfrom tinygrad.nn.state import safe_load, load_state_dictimport jsonimport http@@ -23,12 +21,10 @@ import urllibfrom urllib.parse import urlparse, parse_qsfrom pathlib import Pathimport struct-import pickle-from urllib.parse import unquote-from urllib.parse import quote-import platform-import ctypes+from urllib.parse import unquote, quoteimport zlib+from utils.db import db+import multiprocessingdef resize(img, new_size):img = img.permute(2,0,1)@@ -342,7 +338,6 @@ import threadingBASE = Path(__file__).parent / "data"CAMERA_BASE_DIR = BASE / "cameras"-CAMS_FILE = BASE / "cams.pkl"CAMERA_BASE_DIR.mkdir(parents=True, exist_ok=True)class RollingClassCounter:@@ -357,6 +352,8 @@ class RollingClassCounter:self.is_on = Trueself.is_notif = Trueself.zone = True+ self.reset = False+ self.new = Truedef add(self, class_id):if self.classes is not None and class_id not in self.classes: return@@ -373,6 +370,7 @@ class RollingClassCounter:def reset_counts(self):for class_id, _ in self.data.items():self.data[class_id] = deque() # todo, use in reset endpoint?+ self.reset = Truedef get_counts(self):window = self.window if self.window else (60 if self.is_notif else 1)@@ -475,28 +473,14 @@ class VideoCapture:self.settings = None- alerts_file = CAMERA_BASE_DIR / cam_name / "alerts.pkl"- alerts_file.parent.mkdir(parents=True, exist_ok=True)- settings_file = CAMERA_BASE_DIR / cam_name / "settings.pkl"- settings_file.parent.mkdir(parents=True, exist_ok=True)- try:- with open(alerts_file, "rb") as f:- self.alert_counters = pickle.load(f)- for _,a in self.alert_counters.items():- for c in a.classes: classes.add(str(c))- except Exception:- with open(alerts_file, 'wb') as f:- self.alert_counters = dict()- self.alert_counters[str(uuid.uuid4())] = RollingClassCounter(window_seconds=None, max=1, classes={0,1,2,3,5,7},cam_name=cam_name)- pickle.dump(self.alert_counters, f)- try:- with open(settings_file, "rb") as f:- self.settings = pickle.load(f)- except Exception:- print("zone file not found")+ self.alert_counters = database.run_get("alerts",self.cam_name)+ if not self.alert_counters:+ self.alert_counters = dict()+ id, alert_counter = str(uuid.uuid4()), RollingClassCounter(window_seconds=None, max=1, classes={0,1,2,3,5,7},cam_name=cam_name)+ self.alert_counters[id] = alert_counter+ database.run_put("alerts", self.cam_name, alert_counter, id=id)self.lock = threading.Lock()-self._open_ffmpeg()# Start threads@@ -610,7 +594,7 @@ class VideoCapture:count = 0while self.running:if not (CAMERA_BASE_DIR / self.cam_name).is_dir(): os._exit(1) # deleted cam- try:+ if 1==1:raw_bytes = self.proc.stdout.read(frame_size)if len(raw_bytes) != frame_size:fail_count += 1@@ -679,35 +663,30 @@ class VideoCapture:last_live_check = time.time()threading.Thread(target=check_upload_link, args=(self.cam_name,), daemon=True).start()if (time.time() - last_counter_update) >= 5: #update counter every 5 secs- counters_file = CAMERA_BASE_DIR / self.cam_name / "counters.pkl"- if os.path.exists(counters_file):- with open(counters_file, 'rb') as f:- counter = pickle.load(f)- if counter is None: self.counter = RollingClassCounter(cam_name=self.cam_name, window_seconds=float('inf'))- counter = self.counter- with open(counters_file, 'wb') as f:- pickle.dump(counter, f)+ last_counter_update = time.time()++ counters = database.run_get("counters", self.cam_name)+ if counters not in [None, {}]:+ if counters.reset:+ self.counter.reset_counts()+ self.counter.reset = False+ database.run_put("counters", cam_name, self.counter)- added_alerts_file = CAMERA_BASE_DIR / self.cam_name / "added_alerts.pkl"- if added_alerts_file.exists():- with open(added_alerts_file, 'rb') as f:- added_alerts = pickle.load(f)- for id,a in added_alerts:- if a is None:- del self.alert_counters[id]- continue- self.alert_counters[id] = a- for c in a.classes: classes.add(str(c))- added_alerts_file.unlink()+ alerts = database.run_get("alerts", self.cam_name)+ for id,a in alerts.items():+ if not a.new: continue+ a.new = False+ database.run_put("alerts", self.cam_name, a, id=id)+ if a is None:+ del self.alert_counters[id]+ continue+ self.alert_counters[id] = a+ for c in a.classes: classes.add(str(c))- edited_settings_file = CAMERA_BASE_DIR / self.cam_name / "edited_settings.pkl"- if edited_settings_file.exists():- with open(edited_settings_file, 'rb') as f:- zone = pickle.load(f)- self.settings = zone- edited_settings_file.unlink()-- if userID and live_link[self.cam_name] and (time.time() - last_live_seg) >= 4:+ self.settings = database.run_get("settings", self.cam_name)+ self.alert_counters = {i:a for i,a in self.alert_counters.items() if i in alerts}++ if userID and self.cam_name in live_link and live_link[self.cam_name] and (time.time() - last_live_seg) >= 4:last_live_seg = time.time()mp4_filename = f"segment.mp4"self.streamer.export_clip(Path(mp4_filename), live=True)@@ -720,10 +699,10 @@ class VideoCapture:self.raw_frame = frame.copy()if self.streamer.feeding_frames: self.annotated_frame = self.draw_predictions(frame.copy(), filtered_preds)time.sleep(1 / 30)- except Exception as e:- print("Error in capture_loop:", e)- self._open_ffmpeg()- time.sleep(1)+ #except Exception as e:+ # print("Error in capture_loop:", e)+ # self._open_ffmpeg()+ # time.sleep(1)def inference_loop(self):prev_time = time.time()@@ -1002,22 +981,6 @@ class HLSStreamer:except subprocess.TimeoutExpired:self.ffmpeg_proc.kill()--def append_to_pickle_list(pkl_path, item): # todo, still needed?- pkl_path = Path(pkl_path)- if pkl_path.exists():- try:- with open(pkl_path, 'rb') as f:- data = pickle.load(f)- except Exception as e:- data = []- else:- data = []- data.append(item)- with open(pkl_path, 'wb') as f:- pickle.dump(data, f)--def point_not_in_polygon(coords, poly):n = len(poly)for j in range(len(coords)):@@ -1038,8 +1001,6 @@ def point_not_in_polygon(coords, poly):return Falsereturn True-import multiprocessing-def run_search(return_q, searcher, image_text, top_k, cam_name, selected_dir):res = searcher.search(image_text, top_k, cam_name, selected_dir)return_q.put(res)@@ -1078,7 +1039,8 @@ class HLSRequestHandler(BaseHTTPRequestHandler):returnif parsed_path.path == "/list_cameras":- available_cams = [d.name for d in self.base_dir.iterdir() if d.is_dir() and not d.name.endswith("_det")]+ cams = database.run_get("links", None)+ available_cams = list(cams.keys())self.send_200(available_cams)return@@ -1091,10 +1053,7 @@ class HLSRequestHandler(BaseHTTPRequestHandler):returnstart_cam(rtsp=rtsp,cam_name=cam_name,yolo_variant=yolo_variant)- cams[cam_name] = rtsp-- with open(CAMS_FILE, 'wb') as f:- pickle.dump(cams, f)+ database.run_put("links", cam_name, rtsp)# Redirect back to homeself.send_response(302)@@ -1106,27 +1065,23 @@ class HLSRequestHandler(BaseHTTPRequestHandler):if not cam_name:self.send_error(400, "Missing cam or id")return- settings_file = CAMERA_BASE_DIR / cam_name / "settings.pkl"- edited_settings_file = CAMERA_BASE_DIR / cam_name / "edited_settings.pkl"- settings_file.parent.mkdir(parents=True, exist_ok=True)- if not settings_file.exists():- with open(settings_file, "wb") as f:- pickle.dump(None, f)- settings_file.parent.mkdir(parents=True, exist_ok=True)- with open(settings_file, "rb") as f:- zone = pickle.load(f)+ zone = database.run_get("settings", cam_name)if zone is None: zone = {}coords_json = query.get("coords", [None])[0]if coords_json is not None:coords = json.loads(coords_json)- if isinstance(coords, list) and len(coords) >= 3:- zone["coords"] = [[float(x), float(y)] for x, y in coords]+ if isinstance(coords, list):+ if len(coords) >= 3:+ zone["coords"] = [[float(x), float(y)] for x, y in coords]+ else:+ if "coords" in zone: del zone["coords"]zone["is_notif"] = (str(is_notif).lower() == "true") if (is_notif := query.get("is_notif", [None])[0]) is not None else zone.get("is_notif")zone["outside"] = (str(outside).lower() == "true") if (outside := query.get("outside", [None])[0]) is not None else zone.get("outside")query.get("threshold", [None])[0] is not None and zone.update({"threshold": float(query.get("threshold", [None])[0])}) #need the valif (val := query.get("show_dets", [None])[0]) is not None: zone["show_dets"] = str(int(time.time()))- with open(settings_file, 'wb') as f: pickle.dump(zone, f)- with open(edited_settings_file, 'wb') as f: pickle.dump(zone, f)++ database.run_put("settings", cam_name, zone) # todo, key for each+self.send_response(200)self.send_header("Content-Type", "application/json")self.end_headers()@@ -1137,9 +1092,8 @@ class HLSRequestHandler(BaseHTTPRequestHandler):if not cam_name:self.send_error(400, "Missing cam or id")return- alerts_file = CAMERA_BASE_DIR / cam_name / "alerts.pkl"- with open(alerts_file, "rb") as f:- raw_alerts = pickle.load(f)++ raw_alerts = database.run_get("alerts", cam_name)alert = Nonealert_id = query.get("id", [None])[0]is_on = query.get("is_on", [None])[0]@@ -1168,11 +1122,14 @@ class HLSRequestHandler(BaseHTTPRequestHandler):if is_notif is not None: raw_alerts[alert_id].is_notif = str(is_notif).lower() == "true"if zone is not None: raw_alerts[alert_id].zone = str(zone).lower() == "true"alert = raw_alerts[alert_id]+ alert.new = Trueelse:del raw_alerts[alert_id]- with open(alerts_file, 'wb') as f: pickle.dump(raw_alerts, f)- added_alerts_file = CAMERA_BASE_DIR / cam_name / "added_alerts.pkl"- append_to_pickle_list(pkl_path=added_alerts_file,item=[alert_id, alert])+ if alert is not None:+ database.run_put("alerts", cam_name, alert, alert_id)+ else:+ database.run_delete("alerts", cam_name, alert_id)+self.send_response(200)self.send_header("Content-Type", "application/json")@@ -1181,18 +1138,11 @@ class HLSRequestHandler(BaseHTTPRequestHandler):returnif parsed_path.path == "/get_settings":- zone = {}- if not cam_name:- self.send_error(400, "Missing cam parameter")- return- settings_file = CAMERA_BASE_DIR / cam_name / "settings.pkl"- if settings_file.exists():- try:- with open(settings_file, "rb") as f:- zone = pickle.load(f)- except Exception as e:- self.send_error(500, f"Failed to load zone: {e}")- return+ zone = database.run_get("settings",cam_name)+ if zone is not None:+ if cam_name in zone and "settings" in zone[cam_name]: zone = zone[cam_name]["settings"]+ else:+ zone = {}self.send_200(zone)return@@ -1202,28 +1152,20 @@ class HLSRequestHandler(BaseHTTPRequestHandler):self.send_error(400, "Missing cam parameter")return- alerts_file = CAMERA_BASE_DIR / cam_name / "alerts.pkl"+ raw_alerts = database.run_get("alerts",cam_name)alert_info = []- if alerts_file.exists():- try:- with open(alerts_file, "rb") as f:- raw_alerts = pickle.load(f)- for key,alert in raw_alerts.items():- sched = alert.sched if alert.sched else [[0,86399],True,True,True,True,True,True,True]- alert_info.append({- "window": alert.window,- "max": alert.max,- "classes": list(alert.classes),- "id": str(key),- "sched": sched,- "is_on": alert.is_on,- "is_notif": alert.is_notif,- "zone": alert.zone,- })- except Exception as e:- self.send_error(500, f"Failed to load alerts: {e}")- return-+ for key,alert in raw_alerts.items():+ sched = alert.sched if alert.sched else [[0,86399],True,True,True,True,True,True,True]+ alert_info.append({+ "window": alert.window,+ "max": alert.max,+ "classes": list(alert.classes),+ "id": str(key),+ "sched": sched,+ "is_on": alert.is_on,+ "is_notif": alert.is_notif,+ "zone": alert.zone,+ })self.send_200(alert_info)return@@ -1233,23 +1175,19 @@ class HLSRequestHandler(BaseHTTPRequestHandler):self.send_error(400, "Missing cam_name parameter")return- cam_name_raw = cam_name- cam_path_det = CAMERA_BASE_DIR / (cam_name + "_det")- cam_path_raw = CAMERA_BASE_DIR / cam_name- if cam_path_det.exists() and cam_path_det.is_dir():- try:- shutil.rmtree(cam_path_det)- shutil.rmtree(cam_path_raw)- cams.pop(cam_name, None)- cams.pop(cam_name_raw, None) # todo needed?- with open(CAMS_FILE, 'wb') as f:- pickle.dump(cams, f)- except Exception as e:- self.send_error(500, f"Error deleting camera: {e}")- return- else:- self.send_error(404, "Camera not found")- return+ try:+ shutil.rmtree(CAMERA_BASE_DIR / (cam_name + "_det"), ignore_errors=True)+ shutil.rmtree(CAMERA_BASE_DIR / cam_name, ignore_errors=True)+ # todo clean+ alerts = database.run_get("alerts", cam_name)+ for id, _ in alerts.items():+ database.run_delete("alerts", cam_name, id=id)+ database.run_delete("links", cam_name)+ database.run_delete("settings", cam_name)+ database.run_delete("counters", cam_name)+ except Exception as e:+ self.send_error(500, f"Error deleting camera: {e}")+ returnself.send_response(200)self.send_header("Content-Type", "application/json")@@ -1257,35 +1195,23 @@ class HLSRequestHandler(BaseHTTPRequestHandler):self.wfile.write(b'{"status":"deleted"}')return- if parsed_path.path == "/get_counts":+ if parsed_path.path == "/get_counts": # todo error fetching counts on firstif not cam_name:self.send_error(400, "Missing cam parameter")return- counters_file = CAMERA_BASE_DIR / cam_name / "counters.pkl"- try:- with open(counters_file, "rb") as f:- counter = pickle.load(f)- except Exception:- with open(counters_file, 'wb') as f:- counter = RollingClassCounter(cam_name=cam_name)- pickle.dump(counter, f)-- if not counter:- self.send_response(200)- self.send_header("Content-Type", "application/json")- self.end_headers()- self.wfile.write(b"{}")- return-- raw_counts = counter.get_counts()[0]- labeled_counts = {- class_labels[int(k)]: v- for k, v in raw_counts.items()++ counter = database.run_get("counters", cam_name)+ if counter:+ labeled_counts = {+ class_labels[int(k)]: len(v)+ for k, v in counter.data.items()if int(k) < len(class_labels)- }-- self.send_200(labeled_counts)- return+ }+ self.send_200(labeled_counts)+ return+ else:+ database.run_put("counters", cam_name, RollingClassCounter(cam_name=cam_name))+ self.send_200([])if parsed_path.path == "/shutdown":threading.Thread(target=self.server.shutdown).start()@@ -1301,22 +1227,17 @@ class HLSRequestHandler(BaseHTTPRequestHandler):sys.exit(0)if parsed_path.path == "/reset_counts":- if not cam_name:- self.send_error(400, "Missing cam parameter")- return- counters_file = CAMERA_BASE_DIR / cam_name / "counters.pkl"-- with open(counters_file, "rb") as f:- counter = pickle.load(f)- counter = None- with open(counters_file, 'wb') as f:- pickle.dump(counter, f)-- self.send_response(200)- self.send_header("Content-Type", "application/json")- self.end_headers()- self.wfile.write(b"{}")+ if not cam_name:+ self.send_error(400, "Missing cam parameter")return+ counter = database.run_get("counters",cam_name)+ if counter: counter.reset_counts()+ database.run_put("counters", cam_name, counter)+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.end_headers()+ self.wfile.write(b"{}")+ returnif parsed_path.path == '/' and "cam" not in query:@@ -1812,17 +1733,10 @@ class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):super().server_close()if __name__ == "__main__":- if platform.system() == 'Darwin': subprocess.Popen(['caffeinate', '-dimsu'])- elif platform.system() == 'Windows': ctypes.windll.kernel32.SetThreadExecutionState(0x80000002 | 0x00000001)- elif platform.system() == 'Linux': subprocess.Popen(['systemd-inhibit', '--why=Running script', '--mode=block', 'sleep', '999999'])-- if os.path.exists(CAMS_FILE):- with open(CAMS_FILE, 'rb') as f:- cams = pickle.load(f)- else:- with open(CAMS_FILE, 'wb') as f:- pickle.dump(cams, f)-+ multiprocessing.set_start_method("spawn", force=True)+ database = db()+ cams = database.run_get("links", None)+rtsp_url = next((arg.split("=", 1)[1] for arg in sys.argv[1:] if arg.startswith("--rtsp=")), None)classes = {"0","1","2","7"} # person, bike, car, truck, bird (14)@@ -1878,14 +1792,11 @@ if __name__ == "__main__":if rtsp_url:yolo_infer = YOLOv8(w=width, r=ratio, d=depth, num_classes=80)state_dict = safe_load(get_weights_location(yolo_variant))- for k, _ in state_dict.items(): # todo hack because https://github.com/tinygrad/tinygrad/commit/565a7a6218e5ab920360c21d23af6bcae8df82b5- if k.endswith("num_batches_tracked"): state_dict[k] = Tensor.empty()load_state_dict(yolo_infer, state_dict)cam = VideoCapture(rtsp_url,cam_name=cam_name)hls_streamer = HLSStreamer(cam,cam_name=cam_name)cam.streamer = hls_streamer- searcher = Nonetry:try:server = ThreadedHTTPServer(('0.0.0.0', 8080), HLSRequestHandler)diff --git a/mainview.html b/mainview.htmlindex 4f7e99d..3786674 100644--- a/mainview.html+++ b/mainview.html@@ -1745,10 +1745,10 @@return [x, y];});+ } else if (points.length === 0) {+ coords = [0];}-const outside = document.getElementById("outsideZoneCheckbox").checked;-const thresholdPercent = parseFloat(document.getElementById("zoneThreshold").value) || 50;const threshold = thresholdPercent / 100;@@ -1832,11 +1832,11 @@points.forEach(point => point.remove());zoneEditorState.firstPoint = null;zoneEditorState.isPolygonComplete = false;- if (zoneEditorState.polygonPath) {- zoneEditorState.polygonPath.setAttribute('d', '');- zoneEditorState.polygonPath.setAttribute('fill', 'none');- zoneEditorState.polygonPath.setAttribute('stroke', '#ff0000');+ const svg = overlay.querySelector('svg');+ if (svg) {+ svg.remove();}+ initPolygonOverlay();const previewEl = document.getElementById("zonePreview");previewEl.addEventListener('click', handlePreviewClick);}diff --git a/requirements.txt b/requirements.txtindex 8c37e79..4eec519 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,4 +1,4 @@-git+https://github.com/tinygrad/tinygrad.git@0874ba8cc89fb89a46217ff75e1ef2701b2b010a+git+https://github.com/tinygrad/tinygrad.git@42f6cf3a906dd17bc47c63894b7b5ca8f9c99a66numpy==2.0.0opencv-python-headless==4.10.0.84scipy==1.14.1\ No newline at end of filediff --git a/test/test.py b/test/test.pyindex 5d5bd86..73d2f58 100644--- a/test/test.py+++ b/test/test.py@@ -7,6 +7,7 @@ from tinygrad.nn.state import safe_load, load_state_dictfrom tinygrad import Tensorimport cv2from collections import defaultdict+from tinygrad.helpers import Contextdef test_linear_sum_assigment():for i in range(1,300):@@ -38,13 +39,32 @@ def test_yolo_infer():for k, _ in state_dict.items(): # todo hack because https://github.com/tinygrad/tinygrad/commit/565a7a6218e5ab920360c21d23af6bcae8df82b5if k.endswith("num_batches_tracked"): state_dict[k] = Tensor.empty()load_state_dict(yolo_infer, state_dict)- frame = cv2.VideoCapture("test/videos/MOT16-03.mp4").read()[1]- frame = Tensor(frame)- preds = do_inf(frame, yolo_infer).numpy()- counts = defaultdict(int)- for x in preds:- if x[-2] > 0: counts[int(x[-1])] += 1- assert {int(k): v for k, v in counts.items()} == {2: 1, 0: 35, 1: 2, 3: 1}++ cap = cv2.VideoCapture("test/videos/MOT16-03.mp4")++ frames = []+ with Context(BEAM=2):+ for i in range(10):+ ret, frame = cap.read()+ frame_tensor = Tensor(frame)+ preds = do_inf(frame_tensor, yolo_infer).numpy()+ counts = defaultdict(int)+ for x in preds:+ if x[-2] > 0: counts[int(x[-1])] += 1+ #print(f"{dict(counts)},")+ frames.append(dict(counts))+ assert frames == [{2: 1, 0: 35, 1: 2, 3: 1},+ {2: 1, 0: 35, 1: 2, 3: 1},+ {2: 1, 0: 36, 1: 2, 3: 1},+ {2: 1, 0: 36, 3: 1, 1: 2},+ {2: 1, 0: 38, 3: 1, 1: 2},+ {2: 1, 0: 39, 3: 1, 1: 1},+ {2: 1, 0: 38, 3: 2, 1: 1},+ {2: 1, 0: 35, 1: 1, 3: 2},+ {2: 1, 0: 36, 3: 1, 1: 2},+ {2: 1, 0: 35, 1: 2, 3: 1}]+ cap.release()+import picklefrom yolox.tracker.byte_tracker import BYTETracker@@ -79,7 +99,7 @@ def test_bytetracker():np.testing.assert_allclose(output[j].class_id, output2[j].class_id)assert total_time2 <= (total_time * 1.0), "slower"-test_bytetracker()+#test_bytetracker()test_yolo_infer()-test_linear_sum_assigment()-test_linear_sum_assigment_speed()\ No newline at end of file+#test_linear_sum_assigment()+#test_linear_sum_assigment_speed()\ No newline at end of filediff --git a/test/test_db.py b/test/test_db.pynew file mode 100644index 0000000..37e9a3d--- /dev/null+++ b/test/test_db.py@@ -0,0 +1,36 @@+from utils.db import db++if __name__ == "__main__":+ cache_db = db()+ x = cache_db.run_get("links", None)+ assert x == {}+ cache_db.run_put("links", "cam1", "https://link1")+ cache_db.run_put("links", "cam2", "https://link2")+ x = cache_db.run_get("links", "cam1")+ assert x == "https://link1"+ x = cache_db.run_get("links", None)+ assert x == {'cam1': 'https://link1', 'cam2': 'https://link2'}+ cache_db.run_delete("links", "cam2")+ x = cache_db.run_get("links", None)+ assert x == {'cam1': 'https://link1'}+ cache_db.run_put("alerts", "cam1", "a", replace=False)+ cache_db.run_put("alerts", "cam1", "b", replace=False)+ x = cache_db.run_get("alerts", "cam1")+ assert x == {'1': 'a', '2': 'b'}+ cache_db.run_put("alerts", "cam1", "b", replace=True)+ x = cache_db.run_get("alerts", "cam1")+ assert x == 'b'+ cache_db.run_delete("alerts", "cam1")+ x = cache_db.run_get("alerts", "cam1")+ assert x == {}+ cache_db.run_put("settings", "cam1", "x", id="zone")+ cache_db.run_put("settings", "cam1", "y", id="det")+ x = cache_db.run_get("settings", "cam1")+ assert x == {"zone": "x", "det": "y"}+ x = cache_db.run_get("settings", "cam1", "zone")+ assert x == "x"+ cache_db.run_delete("settings", "cam1", "zone")+ x = cache_db.run_get("settings", None)+ assert x == {'cam1': {'det': 'y'}}+ x = cache_db.run_get("settings", "cam1", None)+ assert x == {'det': 'y'}\ No newline at end of filediff --git a/utils/db.py b/utils/db.pynew file mode 100644index 0000000..e78ef48--- /dev/null+++ b/utils/db.py@@ -0,0 +1,177 @@+import os, sqlite3, contextlib, pickle+from typing import Any+from tinygrad.helpers import diskcache_get++_db_tables = set()+cache_dir: str = "data/"+CACHEDB = os.path.abspath(os.path.join(cache_dir, "cc_cache.db"))+_db_connection = None+VERSION = 1+def db_connection():+ global _db_connection+ if _db_connection is None:+ _db_connection = sqlite3.connect(CACHEDB, timeout=60, isolation_level=None, check_same_thread=False)+ with contextlib.suppress(sqlite3.OperationalError):+ _db_connection.execute("PRAGMA journal_mode=WAL").fetchone()+ _db_connection.execute("PRAGMA busy_timeout = 60000;")+ return _db_connection++def diskcache_put(table: str, key: dict|str|int, val: Any, id: int|str|None = None, prepickled=False, replace=True):+ if isinstance(key, (str, int)): key = {"key": key}+ conn = db_connection()+ cur = conn.cursor()+ table_name = f"{table}_{VERSION}"+ if table not in _db_tables:+ TYPES = {str: "text", bool: "integer", int: "integer", float: "numeric", bytes: "blob"}+ ltypes = ', '.join(f"{k} {TYPES[type(key[k])]}" for k in key.keys())+ cur.execute(f"""CREATE TABLE IF NOT EXISTS '{table_name}' (id TEXT, {ltypes}, val BLOB, PRIMARY KEY (id, {', '.join(key.keys())}))""")+ _db_tables.add(table)+ if replace:+ if id is None:+ cur.execute(f"""DELETE FROM '{table_name}' WHERE {' AND '.join([f'{x}=?' for x in key.keys()])}""", tuple(key.values()))+ id = "1"+ else:+ cur.execute(f"""DELETE FROM '{table_name}' WHERE id=? AND {' AND '.join([f'{x}=?' for x in key.keys()])} """,+ tuple([str(id)] + list(key.values())))+ else:+ if id is None:+ cur.execute(f""" SELECT COALESCE(MAX(CAST(id AS INTEGER)), 0) + 1 FROM '{table_name}' WHERE {' AND '.join([f'{x}=?' for x in key.keys()])}""",+ tuple(key.values()))+ result = cur.fetchone()+ id = str(result[0]) if result and result[0] else "1"+ else:+ id = str(id)+ columns = ["id"] + list(key.keys()) + ["val"]+ placeholders = ["?"] * len(columns)+ values = [id] + list(key.values()) + [val if prepickled else pickle.dumps(val)]+ cur.execute(f"""INSERT INTO '{table_name}' ({', '.join(columns)}) VALUES ({', '.join(placeholders)})""", tuple(values))+ conn.commit()+ cur.close()+ return val, id++def diskcache_delete(table: str, key: dict|str|int, id: int|str|None = None):+ if isinstance(key, (str, int)): key = {"key": key}+ conn = db_connection()+ cur = conn.cursor()+ table_name = f"{table}_{VERSION}"+ try:+ if id is None:+ cur.execute(f"""DELETE FROM '{table_name}' WHERE {' AND '.join([f'{x}=?' for x in key.keys()])}""", tuple(key.values()))+ else:+ cur.execute(f"""DELETE FROM '{table_name}' WHERE id=? AND {' AND '.join([f'{x}=?' for x in key.keys()])} """,+ tuple([str(id)] + list(key.values())))+ conn.commit()+ return True+ except sqlite3.OperationalError:+ return False+ finally:+ cur.close()++def diskcache_get(table: str, key: str|None, id: str|None = None) -> Any:+ cur = db_connection().cursor()+ try:+ if key is None:+ try: res = cur.execute(f"SELECT * FROM '{table}_{VERSION}'")+ except sqlite3.OperationalError: return {}+ out = {}+ for row in res.fetchall():+ row_id, user_key, pickled_val = row[0], row[1], row[-1]+ val = pickle.loads(pickled_val)+ if val is None: continue+ if user_key not in out: out[user_key] = {row_id: val} if row_id != "1" else val+ elif isinstance(out[user_key], dict): out[user_key][row_id] = val+ else: out[user_key] = {"1": out[user_key], row_id: val}+ for k in list(out.keys()):+ if isinstance(out[k], dict) and len(out[k]) == 1 and "1" in out[k]: out[k] = out[k]["1"]+ return out+ if id is not None:+ try:+ res = cur.execute(f"SELECT val FROM '{table}_{VERSION}' WHERE key=? AND id=?", (key, str(id)))+ row = res.fetchone()+ return pickle.loads(row[0]) if row else None+ except sqlite3.OperationalError:+ return {}+ else:+ try:+ res = cur.execute(f"SELECT id, val FROM '{table}_{VERSION}' WHERE key=?", (key,))+ rows = res.fetchall()+ except sqlite3.OperationalError:+ return {}+ if not rows:+ return {}+ if len(rows) == 1 and rows[0][0] == "1":+ return pickle.loads(rows[0][1])+ return {row_id: pickle.loads(val) for row_id, val in rows}+ finally: cur.close()+++import multiprocessing+import queue+import atexit++class db:+ def __init__(self):+ self._req_q = multiprocessing.Queue()+ self._resp_q = multiprocessing.Queue()+ self._stop = multiprocessing.Event()++ self._proc = multiprocessing.Process(+ target=self._worker_loop,+ args=(self._req_q, self._resp_q, self._stop),+ daemon=True+ )+ self._proc.start()++ atexit.register(self._cleanup)++ @staticmethod+ def _worker_loop(req_q, resp_q, stop_event):+ db_connection()+ while not stop_event.is_set():+ try:+ action, table, key, value = req_q.get(timeout=0.2)+ if action == "exit": break+ try:+ if action == "get":+ if value is not None: result = diskcache_get(table, key, value)+ else: result = diskcache_get(table, key)+ resp_q.put(("ok", result))+ elif action == "put":+ val, id_val, replace = value if value else (None, None, True)+ result = diskcache_put(table, key, val, id=id_val, replace=replace)+ resp_q.put(("ok", result))+ elif action == "delete":+ result = diskcache_delete(table, key, value)+ resp_q.put(("ok", result))++ except Exception as e: resp_q.put(("err", str(e)))+ except queue.Empty: continue++ def _cleanup(self):+ if hasattr(self, "_stop"):+ self._stop.set()+ try: self._req_q.put(("exit", None, None, None))+ except: pass++ if hasattr(self, "_proc") and self._proc.is_alive():+ self._proc.join(timeout=1)+ if self._proc.is_alive():+ self._proc.kill()++ def run_get(self, table, key=None, id=None, timeout=5):+ self._req_q.put(("get", table, key, id))+ status, result = self._resp_q.get(timeout=timeout)+ if status == "err": raise Exception(result)+ return result++ def run_put(self, table, key, val=None, id=None, replace=True, timeout=5):+ self._req_q.put(("put", table, key, (val, id, replace)))+ status, result = self._resp_q.get(timeout=timeout)+ if status == "err": raise Exception(result)+ return result++ def run_delete(self, table, key, id=None, timeout=5):+ self._req_q.put(("delete", table, key, id))+ status, result = self._resp_q.get(timeout=timeout)+ if status == "err": raise Exception(result)+ return result\ No newline at end of file