Add object detection, tracking, mobile notifications, and search to any security camera.
Tiny clip (#103)
Rory Clear4ebf2184fc96
diff --git a/clearcam.py b/clearcam.pyindex 189d510..ea249dc 100644--- a/clearcam.py+++ b/clearcam.py@@ -1042,6 +1042,12 @@ def point_not_in_polygon(coords, poly):return Falsereturn True+import multiprocessing++def run_search(return_q, searcher, image_text, top_k, cam_name, selected_dir):+ res = searcher.search(image_text, top_k, cam_name, selected_dir)+ return_q.put(res)+class HLSRequestHandler(BaseHTTPRequestHandler):def __init__(self, *args, **kwargs):self.base_dir = CAMERA_BASE_DIR@@ -1338,7 +1344,12 @@ class HLSRequestHandler(BaseHTTPRequestHandler):if not self.searcher:self.searcher = CLIPSearch()self.searcher._load_all_embeddings()- results = self.searcher.search(image_text, top_k=100, cam_name=cam_name, timestamp=selected_dir)++ return_q = multiprocessing.Queue()+ p = multiprocessing.Process(target=run_search, args=(return_q, self.searcher, image_text, 100, cam_name, selected_dir,))+ p.start()+ results = return_q.get(timeout=3600)+ p.join()image_data = []for path_str, score in results:if score < 0.21: break@@ -1830,7 +1841,7 @@ if __name__ == "__main__":use_clip = use_clip in ["y", "Y"]if use_clip:- from clip_search import CLIPSearch+ from tiny_clip_search import CLIPSearchfrom clip import CachedCLIPSearchif rtsp_url is None and userID is None:diff --git a/test/test_clip.py b/test/test_clip.pyindex c49e828..86b6566 100644--- a/test/test_clip.py+++ b/test/test_clip.py@@ -1,24 +1,74 @@from clip import CachedCLIPSearchfrom clip_search import CLIPSearch+from tiny_clip_search import CLIPSearch as tiny_ClipSearchimport numpy as np+import open_clip+import torch+import jsondef setup_clip_test():scanner = CachedCLIPSearch()scanner.precompute_embeddings("test/clip_images")def test_clip_search():- searcher = CLIPSearch()+ searcher = tiny_ClipSearch()searcher._load_single_embeddings_file("test/clip_images/embeddings.pkl")res = searcher.search("ferrari f40")- np.testing.assert_allclose(res[0][1], 0.33788394927978516)- np.testing.assert_allclose(res[1][1], 0.07776700705289841)+ np.testing.assert_allclose(res[0][1], 0.33788394927978516, rtol=1e-03)+ np.testing.assert_allclose(res[1][1], 0.07776700705289841, rtol=1e-02) # careful nowassert res[0][0] == "test/clip_images/f40.jpg"assert res[1][0] == "test/clip_images/micra.jpg"res = searcher.search("nissan micra")- np.testing.assert_allclose(res[0][1], 0.3227463960647583)- np.testing.assert_allclose(res[1][1], 0.04420311748981476)+ np.testing.assert_allclose(res[0][1], 0.3227463960647583, rtol=1e-03)+ np.testing.assert_allclose(res[1][1], 0.04420311748981476, rtol=1e-02)assert res[1][0] == "test/clip_images/f40.jpg"assert res[0][0] == "test/clip_images/micra.jpg"+def test_clip_search_jit():+ searcher = tiny_ClipSearch()+ searcher._load_single_embeddings_file("test/clip_images/embeddings.pkl")+ for _ in range(5): res = searcher.search("ferrari f40")+ np.testing.assert_allclose(res[0][1], 0.33788394927978516, rtol=1e-03)+ np.testing.assert_allclose(res[1][1], 0.07776700705289841, rtol=1e-02) # careful now+ assert res[0][0] == "test/clip_images/f40.jpg"+ assert res[1][0] == "test/clip_images/micra.jpg"+ for _ in range(5): res = searcher.search("nissan micra")+ np.testing.assert_allclose(res[0][1], 0.3227463960647583, rtol=1e-03)+ np.testing.assert_allclose(res[1][1], 0.04420311748981476, rtol=1e-02)+ assert res[1][0] == "test/clip_images/f40.jpg"+ assert res[0][0] == "test/clip_images/micra.jpg"++def test_tokenizer(s):+ a = open_clip.tokenize([s]).to(torch.device("cpu")).detach().numpy()+ b = my_tokenizer(s)+ np.testing.assert_allclose(b, a)++def my_tokenizer(s):+ with open("tokenizer.json", "r") as f: tok = json.load(f)+ vocab = tok["model"]["vocab"]+ #ret = [49406, 49407]+ ret = [49406] # start++ if len(s) > 0: s += "</w>"+ s = s.replace(" ", " ")+ s = s.replace(" ", "</w>")++ longest = max(len(k) for k in vocab)+ i = 0+ while i < len(s):+ j = longest+ while j >= 0 and s[i:i+j] not in vocab: j -= 1+ token = vocab[s[i:i+j]]+ ret.append(token)+ i+=j++ ret.append(49407) # end+ if len(ret) < 77: ret += [0] * (77 - len(ret))+ return np.asarray([ret])++#for x in ["", "a", "car", "a car", "a bugatti veyron", "deep learning engineer","sxokwasikwqoiwdjwqdioqjdi"]: test_tokenizer(x) # todo failing, not bpe yet+#for x in ["mp4-12c"]: test_tokenizer(x) # failing#setup_clip_test()-test_clip_search()\ No newline at end of file+test_clip_search()+test_clip_search_jit()+diff --git a/tiny_clip_search.py b/tiny_clip_search.pynew file mode 100644index 0000000..4011361--- /dev/null+++ b/tiny_clip_search.py@@ -0,0 +1,208 @@+import os+import pickle+import torch+import open_clip+from typing import Optional+from torch.nn import functional as F+import torch.nn as nn+from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit++class CLIPSearch:+ def __init__(self, base_path="data/cameras"):+ self.base_path = base_path+ self.image_embeddings = {}+ self.image_paths = {}+ self.device = torch.device("cpu")+ self.model, _, self.preprocess = open_clip.create_model_and_transforms(+ 'ViT-L-14',+ pretrained='laion2b_s32b_b82k'+ )+ self.model = self.model.to(self.device).eval()++ # convert here+ # .copy() or trouble+ self.model.tiny_text_projection = tiny_Tensor(self.model.text_projection.detach().numpy().copy())+ self.model.tiny_token_embedding = tiny_nn.Embedding(self.model.token_embedding.num_embeddings, self.model.token_embedding.embedding_dim)+ self.model.tiny_token_embedding.weight = tiny_Tensor(self.model.token_embedding.weight.data.detach().numpy().copy())++ self.model.tiny_ln_1 = []+ self.model.tiny_ln_2 = []+ self.model.out_proj_bias_tiny = []+ self.model.mlp_c_fc = []+ self.model.mlp_c_proj = []+ for resblock in self.model.transformer.resblocks:+ layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_1.normalized_shape, eps=resblock.ln_1.eps, elementwise_affine=True)+ layernorm.weight = tiny_Tensor(resblock.ln_1.weight.detach().numpy().copy())+ layernorm.bias = tiny_Tensor(resblock.ln_1.bias.detach().numpy().copy())+ self.model.tiny_ln_1.append(layernorm)++ layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_2.normalized_shape, eps=resblock.ln_2.eps, elementwise_affine=True)+ layernorm.weight = tiny_Tensor(resblock.ln_2.weight.detach().numpy().copy())+ layernorm.bias = tiny_Tensor(resblock.ln_2.bias.detach().numpy().copy())+ self.model.tiny_ln_2.append(layernorm)++ opb = tiny_Tensor(resblock.attn.out_proj.bias.detach().numpy().copy())+ self.model.out_proj_bias_tiny.append(opb)++ # todo+ mlpcfc = tiny_nn.Linear(resblock.mlp.c_fc.weight.shape[0], resblock.mlp.c_fc.weight.shape[1])+ mlpcfc.weight = tiny_Tensor(resblock.mlp.c_fc.weight.detach().numpy().copy())+ mlpcfc.bias = tiny_Tensor(resblock.mlp.c_fc.bias.detach().numpy().copy())+ self.model.mlp_c_fc.append(mlpcfc)++ mlpcp = tiny_nn.Linear(resblock.mlp.c_proj.weight.shape[0], resblock.mlp.c_proj.weight.shape[1])+ mlpcp.weight = tiny_Tensor(resblock.mlp.c_proj.weight.detach().numpy().copy())+ mlpcp.bias = tiny_Tensor(resblock.mlp.c_proj.bias.detach().numpy().copy())+ self.model.mlp_c_proj.append(mlpcp)++ self.model.tiny_ln_final = tiny_nn.LayerNorm(normalized_shape=self.model.ln_final.normalized_shape, eps=self.model.ln_final.eps, elementwise_affine=True)+ self.model.tiny_ln_final .weight = tiny_Tensor(self.model.ln_final.weight.detach().numpy().copy())+ self.model.tiny_ln_final .bias = tiny_Tensor(self.model.ln_final.bias.detach().numpy().copy())++ def _load_single_embeddings_file(self, cache_file):+ try:+ with open(cache_file, "rb") as f:+ cache = pickle.load(f)++ folder_embeddings = cache.get("embeddings", {})+ folder_paths = cache.get("paths", {})++ self.image_embeddings.update(folder_embeddings)+ self.image_paths.update(folder_paths)++ loaded_count = len(folder_embeddings)+ print(f"Loaded {loaded_count} embeddings from {cache_file}")+ return loaded_count++ except Exception as e:+ print(f"Error loading {cache_file}: {e}")+ return 0++ def _load_all_embeddings(self):+ total_loaded = 0++ if not os.path.exists(self.base_path):+ print(f"Base path {self.base_path} does not exist!")+ return++ for camera_folder in os.listdir(self.base_path):+ camera_path = os.path.join(self.base_path, camera_folder)+ if not os.path.isdir(camera_path):+ continue++ objects_path = os.path.join(camera_path, "objects")+ if not os.path.isdir(objects_path):+ continue++ for date_folder in os.listdir(objects_path):+ date_path = os.path.join(objects_path, date_folder)+ if not os.path.isdir(date_path):+ continue++ cache_file = os.path.join(date_path, "embeddings.pkl")++ if os.path.exists(cache_file):+ total_loaded += self._load_single_embeddings_file(cache_file)++ print(f"\nTotal images loaded: {total_loaded}")++ def _encode_text(self, query):+ tokens = open_clip.tokenize([query]).to(self.device)+ tokens = tiny_Tensor(tokens.detach().numpy())+ text_emb = encode_text(self.model, tokens)+ return text_emb++ def search(self, query, top_k=10, cam_name=None, timestamp=None):+ if not self.image_embeddings:+ print("No embeddings available.")+ return []+ text_embedding = self._encode_text(query)+ text_embedding = torch.Tensor(text_embedding.numpy())+ all_similarities = []+ for path, img_embedding in self.image_embeddings.items():+ normalized_path = path.replace("\\", "/")+ if cam_name and f"/cameras/{cam_name}/" not in normalized_path:+ continue+ if timestamp and f"/objects/{timestamp}/" not in normalized_path:+ continue+ similarity = (img_embedding @ text_embedding.T).item()+ filename = os.path.basename(path)+ if filename.lower().endswith((".jpg", ".jpeg", ".png")):+ object_id = filename.split("_")[1].split(".")[0] if "_" in filename else None+ all_similarities.append((path, similarity, object_id))++ if any(item[2] for item in all_similarities):+ best_per_id = {}+ for path, score, object_id in all_similarities:+ if object_id is not None:+ if object_id not in best_per_id or score > best_per_id[object_id][1]:+ best_per_id[object_id] = (path, score)++ items_without_id = [(path, score) for path, score, object_id in all_similarities if object_id is None]+ results = list(best_per_id.values()) + items_without_id+ else:+ results = [(path, score) for path, score, _ in all_similarities]+ results.sort(key=lambda x: x[1], reverse=True)+ return results[:top_k]++@TinyJit+def encode_text(model, text):+ x = text+ x = model.tiny_token_embedding(x)+ if not hasattr(model, 'tiny_positional_embedding'): model.tiny_positional_embedding = tiny_Tensor(model.positional_embedding.detach().numpy())+ x = x + model.tiny_positional_embedding++ for i, resblock in enumerate(model.transformer.resblocks):+ residual = x+ x = model.tiny_ln_1[i](x)++ B, L, D = x.shape+ H = resblock.attn.num_heads+ d_head = D // H++ in_proj_weight_tiny = tiny_Tensor(resblock.attn.in_proj_weight.detach().numpy()) # todo store these+ in_proj_bias_tiny = tiny_Tensor(resblock.attn.in_proj_bias.detach().numpy())+ out_proj_weight_tiny = tiny_Tensor(resblock.attn.out_proj.weight.detach().numpy())+ attn_mask = tiny_Tensor(model.attn_mask.detach().numpy())+++ qkv = x.matmul(in_proj_weight_tiny.T) + in_proj_bias_tiny+ q, k, v = qkv.split(D, dim=-1)+ q = q.view(B, L, H, d_head).transpose(1, 2)+ k = k.view(B, L, H, d_head).transpose(1, 2)+ v = v.view(B, L, H, d_head).transpose(1, 2)++ scale = 1.0 / (d_head ** 0.5)+ attn_scores = q.matmul(k.transpose(-2, -1)) * scale # (B,H,L,L)+ bool_mask = attn_mask < 0+ attn_scores = attn_scores.masked_fill(bool_mask, float("-inf"))+ attn_probs = tiny_Tensor.softmax(attn_scores)+ context = attn_probs.matmul(v)+ context = context.transpose(1, 2).contiguous().view(B, L, D)+ x = context.matmul(out_proj_weight_tiny.T) + model.out_proj_bias_tiny[i]++ x += residual+ residual = x+ x = model.tiny_ln_2[i](x)+ x = model.mlp_c_fc[i](x)+ x = x.gelu()+ x = model.mlp_c_proj[i](x)+ x += residual++ x = model.tiny_ln_final(x) # [batch_size, n_ctx, transformer.width]+ argmax = text.argmax()+ x = x[0][argmax]+ x = x @ model.tiny_text_projection+ return x / (x * x).sum(axis=-1, keepdim=True).sqrt()++'''+if __name__ == "__main__":+ searcher = CLIPSearch()+ query = input("\nEnter search query (or 'quit' to exit): ").strip()+ results = searcher.search(query, top_k=10)++ print(f"\nTop results for '{query}' (best per object):")+ for i, (path, score) in enumerate(results, 1):+ print(f"{i}. Score: {score:.3f} - \"{path}\"")+'''+