Add object detection, tracking, mobile notifications, and search to any security camera.
Search similar img (#114)
Rory Clear6751bc294996
diff --git a/clearcam.py b/clearcam.pyindex febca0e..a5dea16 100644--- a/clearcam.py+++ b/clearcam.py@@ -758,8 +758,14 @@ def run_search(return_q, searcher, image_text, top_k, cam_name, selected_dir):res = searcher.search(image_text, top_k, cam_name, selected_dir)return_q.put(res)+def run_clip(return_q, clip, searcher, im, top_k, cam_name, selected_dir):+ embedding = clip.precompute_embedding_bs1_np(im)+ res = searcher.search(None, top_k, cam_name, selected_dir, embedding)+ return_q.put(res)+class HLSRequestHandler(BaseHTTPRequestHandler):def __init__(self, *args, **kwargs):+ self.clip = kwargs.pop('clip_instance', None)self.base_dir = CAMERA_BASE_DIRself.show_dets = Noneself.searcher = None@@ -1022,6 +1028,7 @@ class HLSRequestHandler(BaseHTTPRequestHandler):selected_dir = parse_qs(parsed_path.query).get("folder", [None])[0]name_contains = parse_qs(parsed_path.query).get("name_contains", [None])[0]image_text = parse_qs(parsed_path.query).get("image_text", [None])[0]+ similar_img = parse_qs(parsed_path.query).get("similar_img", [None])[0]if cam_name:camera_dirs = [self.base_dir / cam_name]else:@@ -1038,11 +1045,47 @@ class HLSRequestHandler(BaseHTTPRequestHandler):if subdir.is_dir()})- if image_text and use_clip:+ if (image_text or similar_img) and use_clip:if not self.searcher:self.searcher = CLIPSearch()self.searcher._load_all_embeddings()+ if similar_img and use_clip:+ return_q = multiprocessing.Queue()+ p = multiprocessing.Process(target=run_clip, args=(return_q, self.clip, self.searcher, similar_img, 100, cam_name, selected_dir,))+ p.start()+ results = return_q.get(timeout=3600)+ p.join()+ # todo dup!+ image_data = []+ for path_str, score in results:+ if score < 0.21: break+ img_path = (self.base_dir.parent.parent / Path(path_str)).resolve()+ ts = int(img_path.stem.split('_')[0])+ parts = img_path.parts+ cam_index = parts.index("cameras") + 1+ cam = parts[cam_index]+ rel = img_path.relative_to(self.base_dir)+ image_url = f"/{rel}"+ image_data.append({+ "url": image_url,+ "timestamp": ts,+ "filename": img_path.name,+ "cam_name": cam,+ "folder": img_path.parts[-2]+ })+ response_data = {+ "images": image_data,+ "count": len(image_data),+ }+ self.send_response(200)+ self.send_header('Content-type', 'application/json')+ self.end_headers()+ self.wfile.write(json.dumps(response_data).encode('utf-8'))+ return+++ if image_text and use_clip:return_q = multiprocessing.Queue()p = multiprocessing.Process(target=run_search, args=(return_q, self.searcher, image_text, 100, cam_name, selected_dir,))p.start()@@ -1412,17 +1455,18 @@ cams = dict()active_subprocesses = []import socketclass ThreadedHTTPServer(ThreadingMixIn, HTTPServer):- def __init__(self, server_address, RequestHandlerClass):+ def __init__(self, server_address, use_clip, RequestHandlerClass):ThreadingMixIn.__init__(self)HTTPServer.__init__(self, server_address, RequestHandlerClass)self.cleanup_stop_event = threading.Event()self.cleanup_thread = Noneself.max_gb = 256- self.searcher = None+ self.clip = CachedCLIPSearch() if use_clip else Noneself.clip_stop_event = threading.Event()self.clip_thread = Noneself._setup_cleanup_and_clip_thread()+ def finish_request(self, request, client_address): self.RequestHandlerClass(request, client_address, self, clip_instance=self.clip)def _setup_cleanup_and_clip_thread(self):if self.cleanup_thread is None or not self.cleanup_thread.is_alive():@@ -1452,12 +1496,11 @@ class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):self.cleanup_stop_event.wait(timeout=600)def _clip_task(self):- if not self.searcher: self.searcher = CachedCLIPSearch()while not self.clip_stop_event.is_set():try:- object_folders = self.searcher.find_object_folders("data/cameras")+ object_folders = self.clip.find_object_folders("data/cameras")for folder in object_folders:- self.searcher.precompute_embeddings(folder)+ self.clip.precompute_embeddings(folder)except Exception as e:print(f"CLIP error: {e}")self.clip_stop_event.wait(timeout=60)@@ -1570,7 +1613,7 @@ if __name__ == "__main__":try:try:- server = ThreadedHTTPServer(('0.0.0.0', 8080), HLSRequestHandler)+ server = ThreadedHTTPServer(('0.0.0.0', 8080), use_clip, HLSRequestHandler)print(f"Serving at http://{get_lan_ip()}:8080")except OSError as e:if e.errno == socket.errno.EADDRINUSE:diff --git a/clip.py b/clip.pyindex 5215f26..cbfb471 100644--- a/clip.py+++ b/clip.py@@ -11,7 +11,7 @@ class Model: passclass CachedCLIPSearch:- def __init__(self, model_name="ViT-L-14", pretrained_name="laion2b_s32b_b82k"):+ def __init__(self, model_name="ViT-L-14", pretrained_name="laion2b_s32b_b82k", prewarm=True):self.image_embeddings = {}self.image_paths = {}@@ -71,8 +71,9 @@ class CachedCLIPSearch:weights = None# for BEAM- precompute_embeddings(self.model, Tensor.rand((1, 3, 224, 224), dtype=dtypes.float32))- precompute_embeddings(self.model, Tensor.rand((16, 3, 224, 224), dtype=dtypes.float32))+ if prewarm:+ precompute_embeddings(self.model, Tensor.rand((1, 3, 224, 224), dtype=dtypes.float32))+ precompute_embeddings(self.model, Tensor.rand((16, 3, 224, 224), dtype=dtypes.float32))def find_object_folders(self, base_path="data/cameras"):object_folders = []@@ -166,6 +167,14 @@ class CachedCLIPSearch:print(f"{datetime.now()}: Updated cache for {folder_path}. Total images stored: {len(folder_embeddings)}")+ def precompute_embedding_bs1_np(self, img):+ img = f"data/cameras{img}" # todo base path?+ img = cv2.imread(img)+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)+ img = [preprocess(img)]+ ret = precompute_embedding_bs1(self.model, Tensor(img)).numpy()+ return ret+@TinyJitdef precompute_embeddings(model, x): return precompute_embedding(model, x)diff --git a/clip_search.py b/clip_search.pyindex 4b506eb..389686b 100644--- a/clip_search.py+++ b/clip_search.py@@ -131,12 +131,13 @@ class CLIPSearch:text_emb = encode_text(self.model, tokens)return text_emb- def search(self, query, top_k=10, cam_name=None, timestamp=None):+ def search(self, query=None, top_k=10, cam_name=None, timestamp=None, text_embedding=None):if not self.image_embeddings:print("No embeddings available.")return []- text_embedding = self._encode_text(query)- text_embedding = text_embedding.numpy()+ if text_embedding is None:+ text_embedding = self._encode_text(query)+ text_embedding = text_embedding.numpy()all_similarities = []for path, img_embedding in self.image_embeddings.items():normalized_path = path.replace("\\", "/")diff --git a/mainview.html b/mainview.htmlindex 0cfae88..2963565 100644--- a/mainview.html+++ b/mainview.html@@ -2828,22 +2828,36 @@const searchButton = document.getElementById('searchButton');const searchText = document.getElementById('searchText');const searchSpinner = document.getElementById('searchSpinner');+ const magnifyButtons = document.querySelectorAll('.event-image-actions button[title="Search Similar"]');currentImageSearchText = searchInput.value.trim();searchText.textContent = 'Searching';searchSpinner.style.display = 'block';searchButton.disabled = true;+ magnifyButtons.forEach(btn => btn.disabled = true);loadEventImages().finally(() => {searchText.textContent = 'Search';searchSpinner.style.display = 'none';searchButton.disabled = false;+ magnifyButtons.forEach(btn => btn.disabled = false);});}function clearSearch() {+ const searchButton = document.getElementById('searchButton');+ const searchText = document.getElementById('searchText');+ const searchSpinner = document.getElementById('searchSpinner');+ const magnifyButtons = document.querySelectorAll('.event-image-actions button[title="Search Similar"]');++ searchText.textContent = 'Search';+ searchSpinner.style.display = 'none';+ searchButton.disabled = false;+ magnifyButtons.forEach(btn => btn.disabled = false);+document.getElementById('imageSearchInput').value = '';currentImageSearchText = '';+ currentSimilarImage = null;loadEventImages();}@@ -2865,8 +2879,11 @@if (alertsOnlyFilter) {url += `&name_contains=_notif`;}- if (currentImageSearchText) {- url += `&image_text=${encodeURIComponent(currentImageSearchText)}`;+ if (currentSimilarImage) {+ url += `&similar_img=/${encodeURIComponent(currentSimilarImage)}`;+ currentSimilarImage = null;+ } else if (currentImageSearchText) {+ url += `&image_text=${encodeURIComponent(currentImageSearchText)}`;}const res = await fetch(url);@@ -2902,7 +2919,7 @@<small>${img.folder} ${timeStr}</small></div><div class="event-image-actions">- <button title="View" onclick="viewImage('${img.url}')">+ <button title="Search Similar" onclick="searchWithImage('${img.url}')"><span class="icon"><svg><use href="#icon-magnify"></use></svg></span>@@ -2929,9 +2946,45 @@console.error("Failed to load event images:", err);const container = document.getElementById("eventImagesContainer");container.innerHTML = '<p style="text-align:center; color:red; width:100%;">Error loading events.</p>';+ } finally {+ const searchButton = document.getElementById('searchButton');+ const searchText = document.getElementById('searchText');+ const searchSpinner = document.getElementById('searchSpinner');+ const magnifyButtons = document.querySelectorAll('.event-image-actions button[title="Search Similar"]');++ searchText.textContent = 'Search';+ searchSpinner.style.display = 'none';+ searchButton.disabled = false;+ magnifyButtons.forEach(btn => btn.disabled = false);}}+ let currentSimilarImage = null;+ function searchWithImage(imageUrl) {+ const searchButton = document.getElementById('searchButton');+ const searchText = document.getElementById('searchText');+ const searchSpinner = document.getElementById('searchSpinner');+ const magnifyButtons = document.querySelectorAll('.event-image-actions button[title="Search Similar"]');++ const match = imageUrl.match(/\/([^/]+\/objects\/[^/]+\/[^/]+\.jpg)/);+ if (!match) return;+ document.querySelector('.events-section').scrollIntoView({ behavior: 'smooth' });+ searchText.textContent = 'Searching';+ searchSpinner.style.display = 'block';+ searchButton.disabled = true;+ magnifyButtons.forEach(btn => btn.disabled = true);++ currentSimilarImage = match[1];+ document.getElementById('imageSearchInput').value = '';++ loadEventImages().finally(() => {+ searchText.textContent = 'Search';+ searchSpinner.style.display = 'none';+ searchButton.disabled = false;+ magnifyButtons.forEach(btn => btn.disabled = false);+ });+ }+function secondsToTimeString(seconds) {const hours = Math.floor(seconds / 3600);const minutes = Math.floor((seconds % 3600) / 60);