Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

Tiny clip (#103)

Rory Clear4ebf2184fc96
diff --git a/clearcam.py b/clearcam.py
index 189d510..ea249dc 100644
--- a/clearcam.py
+++ b/clearcam.py
@@ -1042,6 +1042,12 @@ def point_not_in_polygon(coords, poly):
return False
return True
+import multiprocessing
+
+def run_search(return_q, searcher, image_text, top_k, cam_name, selected_dir):
+ res = searcher.search(image_text, top_k, cam_name, selected_dir)
+ return_q.put(res)
+
class HLSRequestHandler(BaseHTTPRequestHandler):
def __init__(self, *args, **kwargs):
self.base_dir = CAMERA_BASE_DIR
@@ -1338,7 +1344,12 @@ class HLSRequestHandler(BaseHTTPRequestHandler):
if not self.searcher:
self.searcher = CLIPSearch()
self.searcher._load_all_embeddings()
- results = self.searcher.search(image_text, top_k=100, cam_name=cam_name, timestamp=selected_dir)
+
+ return_q = multiprocessing.Queue()
+ p = multiprocessing.Process(target=run_search, args=(return_q, self.searcher, image_text, 100, cam_name, selected_dir,))
+ p.start()
+ results = return_q.get(timeout=3600)
+ p.join()
image_data = []
for path_str, score in results:
if score < 0.21: break
@@ -1830,7 +1841,7 @@ if __name__ == "__main__":
use_clip = use_clip in ["y", "Y"]
if use_clip:
- from clip_search import CLIPSearch
+ from tiny_clip_search import CLIPSearch
from clip import CachedCLIPSearch
if rtsp_url is None and userID is None:
diff --git a/test/test_clip.py b/test/test_clip.py
index c49e828..86b6566 100644
--- a/test/test_clip.py
+++ b/test/test_clip.py
@@ -1,24 +1,74 @@
from clip import CachedCLIPSearch
from clip_search import CLIPSearch
+from tiny_clip_search import CLIPSearch as tiny_ClipSearch
import numpy as np
+import open_clip
+import torch
+import json
def setup_clip_test():
scanner = CachedCLIPSearch()
scanner.precompute_embeddings("test/clip_images")
def test_clip_search():
- searcher = CLIPSearch()
+ searcher = tiny_ClipSearch()
searcher._load_single_embeddings_file("test/clip_images/embeddings.pkl")
res = searcher.search("ferrari f40")
- np.testing.assert_allclose(res[0][1], 0.33788394927978516)
- np.testing.assert_allclose(res[1][1], 0.07776700705289841)
+ np.testing.assert_allclose(res[0][1], 0.33788394927978516, rtol=1e-03)
+ np.testing.assert_allclose(res[1][1], 0.07776700705289841, rtol=1e-02) # careful now
assert res[0][0] == "test/clip_images/f40.jpg"
assert res[1][0] == "test/clip_images/micra.jpg"
res = searcher.search("nissan micra")
- np.testing.assert_allclose(res[0][1], 0.3227463960647583)
- np.testing.assert_allclose(res[1][1], 0.04420311748981476)
+ np.testing.assert_allclose(res[0][1], 0.3227463960647583, rtol=1e-03)
+ np.testing.assert_allclose(res[1][1], 0.04420311748981476, rtol=1e-02)
assert res[1][0] == "test/clip_images/f40.jpg"
assert res[0][0] == "test/clip_images/micra.jpg"
+def test_clip_search_jit():
+ searcher = tiny_ClipSearch()
+ searcher._load_single_embeddings_file("test/clip_images/embeddings.pkl")
+ for _ in range(5): res = searcher.search("ferrari f40")
+ np.testing.assert_allclose(res[0][1], 0.33788394927978516, rtol=1e-03)
+ np.testing.assert_allclose(res[1][1], 0.07776700705289841, rtol=1e-02) # careful now
+ assert res[0][0] == "test/clip_images/f40.jpg"
+ assert res[1][0] == "test/clip_images/micra.jpg"
+ for _ in range(5): res = searcher.search("nissan micra")
+ np.testing.assert_allclose(res[0][1], 0.3227463960647583, rtol=1e-03)
+ np.testing.assert_allclose(res[1][1], 0.04420311748981476, rtol=1e-02)
+ assert res[1][0] == "test/clip_images/f40.jpg"
+ assert res[0][0] == "test/clip_images/micra.jpg"
+
+def test_tokenizer(s):
+ a = open_clip.tokenize([s]).to(torch.device("cpu")).detach().numpy()
+ b = my_tokenizer(s)
+ np.testing.assert_allclose(b, a)
+
+def my_tokenizer(s):
+ with open("tokenizer.json", "r") as f: tok = json.load(f)
+ vocab = tok["model"]["vocab"]
+ #ret = [49406, 49407]
+ ret = [49406] # start
+
+ if len(s) > 0: s += "</w>"
+ s = s.replace(" ", " ")
+ s = s.replace(" ", "</w>")
+
+ longest = max(len(k) for k in vocab)
+ i = 0
+ while i < len(s):
+ j = longest
+ while j >= 0 and s[i:i+j] not in vocab: j -= 1
+ token = vocab[s[i:i+j]]
+ ret.append(token)
+ i+=j
+
+ ret.append(49407) # end
+ if len(ret) < 77: ret += [0] * (77 - len(ret))
+ return np.asarray([ret])
+
+#for x in ["", "a", "car", "a car", "a bugatti veyron", "deep learning engineer","sxokwasikwqoiwdjwqdioqjdi"]: test_tokenizer(x) # todo failing, not bpe yet
+#for x in ["mp4-12c"]: test_tokenizer(x) # failing
#setup_clip_test()
-test_clip_search()
\ No newline at end of file
+test_clip_search()
+test_clip_search_jit()
+
diff --git a/tiny_clip_search.py b/tiny_clip_search.py
new file mode 100644
index 0000000..4011361
--- /dev/null
+++ b/tiny_clip_search.py
@@ -0,0 +1,208 @@
+import os
+import pickle
+import torch
+import open_clip
+from typing import Optional
+from torch.nn import functional as F
+import torch.nn as nn
+from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit
+
+class CLIPSearch:
+ def __init__(self, base_path="data/cameras"):
+ self.base_path = base_path
+ self.image_embeddings = {}
+ self.image_paths = {}
+ self.device = torch.device("cpu")
+ self.model, _, self.preprocess = open_clip.create_model_and_transforms(
+ 'ViT-L-14',
+ pretrained='laion2b_s32b_b82k'
+ )
+ self.model = self.model.to(self.device).eval()
+
+ # convert here
+ # .copy() or trouble
+ self.model.tiny_text_projection = tiny_Tensor(self.model.text_projection.detach().numpy().copy())
+ self.model.tiny_token_embedding = tiny_nn.Embedding(self.model.token_embedding.num_embeddings, self.model.token_embedding.embedding_dim)
+ self.model.tiny_token_embedding.weight = tiny_Tensor(self.model.token_embedding.weight.data.detach().numpy().copy())
+
+ self.model.tiny_ln_1 = []
+ self.model.tiny_ln_2 = []
+ self.model.out_proj_bias_tiny = []
+ self.model.mlp_c_fc = []
+ self.model.mlp_c_proj = []
+ for resblock in self.model.transformer.resblocks:
+ layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_1.normalized_shape, eps=resblock.ln_1.eps, elementwise_affine=True)
+ layernorm.weight = tiny_Tensor(resblock.ln_1.weight.detach().numpy().copy())
+ layernorm.bias = tiny_Tensor(resblock.ln_1.bias.detach().numpy().copy())
+ self.model.tiny_ln_1.append(layernorm)
+
+ layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_2.normalized_shape, eps=resblock.ln_2.eps, elementwise_affine=True)
+ layernorm.weight = tiny_Tensor(resblock.ln_2.weight.detach().numpy().copy())
+ layernorm.bias = tiny_Tensor(resblock.ln_2.bias.detach().numpy().copy())
+ self.model.tiny_ln_2.append(layernorm)
+
+ opb = tiny_Tensor(resblock.attn.out_proj.bias.detach().numpy().copy())
+ self.model.out_proj_bias_tiny.append(opb)
+
+ # todo
+ mlpcfc = tiny_nn.Linear(resblock.mlp.c_fc.weight.shape[0], resblock.mlp.c_fc.weight.shape[1])
+ mlpcfc.weight = tiny_Tensor(resblock.mlp.c_fc.weight.detach().numpy().copy())
+ mlpcfc.bias = tiny_Tensor(resblock.mlp.c_fc.bias.detach().numpy().copy())
+ self.model.mlp_c_fc.append(mlpcfc)
+
+ mlpcp = tiny_nn.Linear(resblock.mlp.c_proj.weight.shape[0], resblock.mlp.c_proj.weight.shape[1])
+ mlpcp.weight = tiny_Tensor(resblock.mlp.c_proj.weight.detach().numpy().copy())
+ mlpcp.bias = tiny_Tensor(resblock.mlp.c_proj.bias.detach().numpy().copy())
+ self.model.mlp_c_proj.append(mlpcp)
+
+ self.model.tiny_ln_final = tiny_nn.LayerNorm(normalized_shape=self.model.ln_final.normalized_shape, eps=self.model.ln_final.eps, elementwise_affine=True)
+ self.model.tiny_ln_final .weight = tiny_Tensor(self.model.ln_final.weight.detach().numpy().copy())
+ self.model.tiny_ln_final .bias = tiny_Tensor(self.model.ln_final.bias.detach().numpy().copy())
+
+ def _load_single_embeddings_file(self, cache_file):
+ try:
+ with open(cache_file, "rb") as f:
+ cache = pickle.load(f)
+
+ folder_embeddings = cache.get("embeddings", {})
+ folder_paths = cache.get("paths", {})
+
+ self.image_embeddings.update(folder_embeddings)
+ self.image_paths.update(folder_paths)
+
+ loaded_count = len(folder_embeddings)
+ print(f"Loaded {loaded_count} embeddings from {cache_file}")
+ return loaded_count
+
+ except Exception as e:
+ print(f"Error loading {cache_file}: {e}")
+ return 0
+
+ def _load_all_embeddings(self):
+ total_loaded = 0
+
+ if not os.path.exists(self.base_path):
+ print(f"Base path {self.base_path} does not exist!")
+ return
+
+ for camera_folder in os.listdir(self.base_path):
+ camera_path = os.path.join(self.base_path, camera_folder)
+ if not os.path.isdir(camera_path):
+ continue
+
+ objects_path = os.path.join(camera_path, "objects")
+ if not os.path.isdir(objects_path):
+ continue
+
+ for date_folder in os.listdir(objects_path):
+ date_path = os.path.join(objects_path, date_folder)
+ if not os.path.isdir(date_path):
+ continue
+
+ cache_file = os.path.join(date_path, "embeddings.pkl")
+
+ if os.path.exists(cache_file):
+ total_loaded += self._load_single_embeddings_file(cache_file)
+
+ print(f"\nTotal images loaded: {total_loaded}")
+
+ def _encode_text(self, query):
+ tokens = open_clip.tokenize([query]).to(self.device)
+ tokens = tiny_Tensor(tokens.detach().numpy())
+ text_emb = encode_text(self.model, tokens)
+ return text_emb
+
+ def search(self, query, top_k=10, cam_name=None, timestamp=None):
+ if not self.image_embeddings:
+ print("No embeddings available.")
+ return []
+ text_embedding = self._encode_text(query)
+ text_embedding = torch.Tensor(text_embedding.numpy())
+ all_similarities = []
+ for path, img_embedding in self.image_embeddings.items():
+ normalized_path = path.replace("\\", "/")
+ if cam_name and f"/cameras/{cam_name}/" not in normalized_path:
+ continue
+ if timestamp and f"/objects/{timestamp}/" not in normalized_path:
+ continue
+ similarity = (img_embedding @ text_embedding.T).item()
+ filename = os.path.basename(path)
+ if filename.lower().endswith((".jpg", ".jpeg", ".png")):
+ object_id = filename.split("_")[1].split(".")[0] if "_" in filename else None
+ all_similarities.append((path, similarity, object_id))
+
+ if any(item[2] for item in all_similarities):
+ best_per_id = {}
+ for path, score, object_id in all_similarities:
+ if object_id is not None:
+ if object_id not in best_per_id or score > best_per_id[object_id][1]:
+ best_per_id[object_id] = (path, score)
+
+ items_without_id = [(path, score) for path, score, object_id in all_similarities if object_id is None]
+ results = list(best_per_id.values()) + items_without_id
+ else:
+ results = [(path, score) for path, score, _ in all_similarities]
+ results.sort(key=lambda x: x[1], reverse=True)
+ return results[:top_k]
+
+@TinyJit
+def encode_text(model, text):
+ x = text
+ x = model.tiny_token_embedding(x)
+ if not hasattr(model, 'tiny_positional_embedding'): model.tiny_positional_embedding = tiny_Tensor(model.positional_embedding.detach().numpy())
+ x = x + model.tiny_positional_embedding
+
+ for i, resblock in enumerate(model.transformer.resblocks):
+ residual = x
+ x = model.tiny_ln_1[i](x)
+
+ B, L, D = x.shape
+ H = resblock.attn.num_heads
+ d_head = D // H
+
+ in_proj_weight_tiny = tiny_Tensor(resblock.attn.in_proj_weight.detach().numpy()) # todo store these
+ in_proj_bias_tiny = tiny_Tensor(resblock.attn.in_proj_bias.detach().numpy())
+ out_proj_weight_tiny = tiny_Tensor(resblock.attn.out_proj.weight.detach().numpy())
+ attn_mask = tiny_Tensor(model.attn_mask.detach().numpy())
+
+
+ qkv = x.matmul(in_proj_weight_tiny.T) + in_proj_bias_tiny
+ q, k, v = qkv.split(D, dim=-1)
+ q = q.view(B, L, H, d_head).transpose(1, 2)
+ k = k.view(B, L, H, d_head).transpose(1, 2)
+ v = v.view(B, L, H, d_head).transpose(1, 2)
+
+ scale = 1.0 / (d_head ** 0.5)
+ attn_scores = q.matmul(k.transpose(-2, -1)) * scale # (B,H,L,L)
+ bool_mask = attn_mask < 0
+ attn_scores = attn_scores.masked_fill(bool_mask, float("-inf"))
+ attn_probs = tiny_Tensor.softmax(attn_scores)
+ context = attn_probs.matmul(v)
+ context = context.transpose(1, 2).contiguous().view(B, L, D)
+ x = context.matmul(out_proj_weight_tiny.T) + model.out_proj_bias_tiny[i]
+
+ x += residual
+ residual = x
+ x = model.tiny_ln_2[i](x)
+ x = model.mlp_c_fc[i](x)
+ x = x.gelu()
+ x = model.mlp_c_proj[i](x)
+ x += residual
+
+ x = model.tiny_ln_final(x) # [batch_size, n_ctx, transformer.width]
+ argmax = text.argmax()
+ x = x[0][argmax]
+ x = x @ model.tiny_text_projection
+ return x / (x * x).sum(axis=-1, keepdim=True).sqrt()
+
+'''
+if __name__ == "__main__":
+ searcher = CLIPSearch()
+ query = input("\nEnter search query (or 'quit' to exit): ").strip()
+ results = searcher.search(query, top_k=10)
+
+ print(f"\nTop results for '{query}' (best per object):")
+ for i, (path, score) in enumerate(results, 1):
+ print(f"{i}. Score: {score:.3f} - \"{path}\"")
+'''
+