Add object detection, tracking, mobile notifications, and search to any security camera.
import os
import pickle
from tinygrad import nn, Tensor, TinyJit
from tinygrad.helpers import fetch
from tinygrad.dtype import dtypes
import numpy as np
import cv2
from utils.helpers import jit_infer
from models.blazeface import BlazeFace
from models.adaface import ADAFACE
from tinygrad.nn.state import safe_load, load_state_dict
from utils.clip_tokenizer import SimpleTokenizer
import math
from clearcam import event_img_info
from utils.clip_tokenizer import SimpleTokenizer
class Blank: pass
class OpenCLIP:
def __init__(self, base_path="data/cameras"):
self.base_path = base_path
self.image_embeddings = {}
self.face_embeddings = {}
self.image_paths = {}
self.face_paths = {}
self.tokenizer = SimpleTokenizer()
self.text_projection = Tensor.empty(768, 768)
self.positional_embedding_text = Tensor.empty(77, 768)
self.token_embedding = nn.Embedding(49408, 768)
self.ln_final = nn.LayerNorm(768, eps=1e-5, elementwise_affine=True)
self.attn_mask = Tensor.ones(77, 77).tril().cast(dtypes.bool).where(0.0, -math.inf).cast(dtypes.float)
self.resblocks = []
for i in range(12):
resblock = Blank()
resblock.ln_1 = nn.LayerNorm(768, 1e-5, elementwise_affine=True)
resblock.ln_2 = nn.LayerNorm(768, 1e-5, elementwise_affine=True)
resblock.attn_out_proj_weight = Tensor.empty(768, 768)
resblock.attn_out_proj_bias = Tensor.empty(768)
resblock.mlp_c_fc = nn.Linear(768, 3072)
resblock.mlp_c_proj = nn.Linear(3072, 768)
resblock.in_proj_weight = Tensor.empty(2304, 768)
resblock.in_proj_bias = Tensor.empty(2304)
self.resblocks.append(resblock)
self.visual_conv1 = nn.Conv2d(3, 1024, (14, 14), (14, 14), (0, 0), (1, 1), 1, bias=False)
self.class_embedding = Tensor.empty(1024)
self.positional_embedding = Tensor.empty(257, 1024)
self.ln_pre = nn.LayerNorm(1024)
self.ln_post = nn.LayerNorm(1024)
self.proj = Tensor.empty(1024, 768)
self.resblocks_img = []
for i in range(24):
resblock = Blank()
resblock.ln_1 = nn.LayerNorm(1024, 1e-05, elementwise_affine=True)
resblock.ln_2 = nn.LayerNorm(1024, 1e-05, elementwise_affine=True)
resblock.in_proj_weight = Tensor.empty(3072, 1024)
resblock.in_proj_bias = Tensor.empty(3072)
resblock.out_proj_weight = Tensor.empty(1024, 1024)
resblock.out_proj_bias = Tensor.empty(1024)
resblock.mlp_c_fc = nn.Linear(1024, 4096)
resblock.mlp_c_proj = nn.Linear(4096, 1024)
self.resblocks_img.append(resblock)
state_dict = safe_load(fetch("https://huggingface.co/roryclear/CLIP-ViT-L-14-laion2B-s32B-b82K/resolve/main/CLIP-ViT-L-14-laion2B-s32B-b82K.safetensors"))
load_state_dict(self, state_dict)
def precompute_embedding(self, x):
x = self.visual_conv1(x)
x = x.reshape(x.shape[0], x.shape[1], -1)
x = x.permute(0, 2, 1)
cls_emb = self.class_embedding
cls_emb = cls_emb.unsqueeze(0).expand(x.shape[0], -1, -1)
x = Tensor.cat(cls_emb, x, dim=1)
x = x + self.positional_embedding
x = self.ln_pre(x)
# https://github.com/pytorch/pytorch/blob/v2.9.1/torch/nn/modules/activation.py#L1252
for i in range(len(self.resblocks_img)):
x_ln1 = self.resblocks_img[i].ln_1(x)
B, L, D = x_ln1.shape
H = 16 #block.attn.num_heads
d_head = D // H
qkv = x_ln1 @ self.resblocks_img[i].in_proj_weight.T + self.resblocks_img[i].in_proj_bias
q, k, v = qkv.split(D, dim=-1)
def shape(x): return x.view(B, L, H, d_head).transpose(1, 2)
q = shape(q)
k = shape(k)
v = shape(v)
scale = 1.0 / (d_head ** 0.5)
attn_scores = q.matmul(k.transpose(-2, -1)) * scale
attn_probs = Tensor.softmax(attn_scores)
context = attn_probs.matmul(v)
context = context.transpose(1, 2).contiguous().view(B, L, D)
attn_out = context @ self.resblocks_img[i].out_proj_weight.T + self.resblocks_img[i].out_proj_bias
attn_scaled = attn_out
x = x + attn_scaled
x_ln2 = self.resblocks_img[i].ln_2(x)
ff = self.resblocks_img[i].mlp_c_fc(x_ln2)
ff = ff.gelu()
ff = self.resblocks_img[i].mlp_c_proj(ff)
x = x + ff
x = self.ln_post(x)
image_embeds = x[:, 0, :]
embeddings = image_embeds @ self.proj
embeddings = embeddings / (embeddings.pow(2).sum(axis=-1, keepdim=True).sqrt() + 1e-8)
return embeddings
def _encode_text(self, query, realize=False):
tokens = [49406]
tokens += self.tokenizer.encode(query)
tokens.append(49407)
if len(tokens) < 77: tokens += [0] * (77 - len(tokens))
tokens = Tensor([tokens])
text_emb = encode_text(self, tokens)
if realize: return text_emb.numpy()
return text_emb
@TinyJit
def encode_text(model, text):
x = text
x = model.token_embedding(x)
x = x + model.positional_embedding_text
for i in range(len(model.resblocks)):
residual = x
x = model.resblocks[i].ln_1(x)
B, L, D = x.shape
H = 12 #resblock.attn.num_heads
d_head = D // H
qkv = x.matmul(model.resblocks[i].in_proj_weight.T) + model.resblocks[i].in_proj_bias
q, k, v = qkv.split(D, dim=-1)
q = q.view(B, L, H, d_head).transpose(1, 2)
k = k.view(B, L, H, d_head).transpose(1, 2)
v = v.view(B, L, H, d_head).transpose(1, 2)
scale = 1.0 / (d_head ** 0.5)
attn_scores = q.matmul(k.transpose(-2, -1)) * scale # (B,H,L,L)
bool_mask = model.attn_mask < 0
attn_scores = attn_scores.masked_fill(bool_mask, float("-inf"))
attn_probs = Tensor.softmax(attn_scores)
context = attn_probs.matmul(v)
context = context.transpose(1, 2).contiguous().view(B, L, D)
x = context.matmul(model.resblocks[i].attn_out_proj_weight.T) + model.resblocks[i].attn_out_proj_bias
x += residual
residual = x
x = model.resblocks[i].ln_2(x)
x = model.resblocks[i].mlp_c_fc(x)
x = x.gelu()
x = model.resblocks[i].mlp_c_proj(x)
x += residual
x = model.ln_final(x) # [batch_size, n_ctx, transformer.width]
argmax = text.argmax()
x = x[0][argmax]
x = x @ model.text_projection
return x / (x * x).sum(axis=-1, keepdim=True).sqrt()
class ObjectFinder:
def __init__(self, base_path="data/cameras"):
self.base_path = base_path
self.image_embeddings = {}
self.face_embeddings = {}
self.image_paths = {}
self.face_paths = {}
self.clip = False # todo, should be able to init at start
self.face = False # todo
self.jit_cache = {}
def init_clip(self):
if self.clip: return
self.clip = True
self.model = OpenCLIP()
print("prewarming CLIP....")
for _ in range(2): _ = self.model._encode_text("text here", realize=True)
for _ in range(2): _ = jit_infer(self.model.precompute_embedding, Tensor.rand(1, 3, 224, 224), jit_cache=self.jit_cache).numpy()
print("DONE")
def turn_off_clip(self):
self.clip = False
self.model = None
def init_face(self):
if self.face: return
self.face = True
self.blazeface = BlazeFace()
self.adaface = ADAFACE()
def turn_off_face(self):
self.face = False
self.blazeface = None
self.adaface = None
def find_object_folders(self, base_path="data/cameras"):
object_folders = []
if not os.path.exists(base_path): return object_folders
for camera_folder in os.listdir(base_path):
camera_path = os.path.join(base_path, camera_folder)
if os.path.isdir(camera_path):
objects_path = os.path.join(camera_path, "objects")
if os.path.isdir(objects_path):
for date_folder in os.listdir(objects_path):
date_path = os.path.join(objects_path, date_folder)
if os.path.isdir(date_path):
object_folders.append(date_path)
return object_folders
def preprocess(self, img):
img = cv2.resize(img, (224, 224), interpolation=cv2.INTER_CUBIC)
img = img.astype(np.float32) / 255.0
img = (img - 0.5) / 0.5
img = np.transpose(img, (2, 0, 1))
return img
def preprocess_clip(self, img):
if type(img) == bytes:
img = cv2.imdecode(np.frombuffer(img, np.uint8), cv2.IMREAD_COLOR)
else:
img = f"data/cameras{img}"
img = cv2.imread(img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
return [self.preprocess(img)]
def preprocess_face(self, img):
if type(img) == bytes: # todo dup of above
img = cv2.imdecode(np.frombuffer(img, np.uint8), cv2.IMREAD_COLOR)
else:
img = f"data/cameras{img}"
img = cv2.imread(img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if img.shape != (112, 112, 3): img = self.img_to_face(img)
return img
def img_to_face(self, orig):
orig_full = orig.copy()
h, w = orig.shape[:2]
scale = 640 / max(h, w)
resized = cv2.resize(orig, (int(w*scale), int(h*scale)))
delta_w, delta_h = 640 - resized.shape[1], 640 - resized.shape[0]
top, bottom = delta_h // 2, delta_h - (delta_h // 2)
left, right = delta_w // 2, delta_w - (delta_w // 2)
orig = cv2.copyMakeBorder(resized, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0,0,0])
detections = self.blazeface(Tensor(orig)).numpy()
detections = detections[detections[:, 0] != 0]
# one face per person for now
if detections.shape[0] > 0:
y1, x1, y2, x2 = detections[0][:4]
left_eye = np.array([detections[0][4], detections[0][5]])
right_eye = np.array([detections[0][6], detections[0][7]])
x1 -= left; x2 -= left
y1 -= top; y2 -= top
left_eye -= np.array([left, top])
right_eye -= np.array([left, top])
x1 /= scale; x2 /= scale
y1 /= scale; y2 /= scale
left_eye /= scale
right_eye /= scale
if (x2 - x1) < 50: return None
TARGET_LEFT_EYE = np.array([38, 51])
TARGET_RIGHT_EYE = np.array([73, 51])
eye_center = (left_eye + right_eye) / 2
target_eye_distance = np.linalg.norm(TARGET_RIGHT_EYE - TARGET_LEFT_EYE)
angle_rad = np.arctan2(right_eye[1] - left_eye[1], right_eye[0] - left_eye[0])
angle = np.degrees(angle_rad)
face_width = x2 - x1
face_height = y2 - y1
crop_size = max(face_width, face_height) * 2.0
x1_crop = int(eye_center[0] - crop_size / 2)
y1_crop = int(eye_center[1] - crop_size / 2)
x2_crop = int(eye_center[0] + crop_size / 2)
y2_crop = int(eye_center[1] + crop_size / 2)
orig = orig_full
H, W = orig_full.shape[:2]
x1_crop = max(0, x1_crop)
y1_crop = max(0, y1_crop)
x2_crop = min(W, x2_crop)
y2_crop = min(H, y2_crop)
if x2_crop <= x1_crop or y2_crop <= y1_crop: return None
cropped = orig[y1_crop:y2_crop, x1_crop:x2_crop]
crop_h, crop_w = cropped.shape[:2]
if crop_h == 0 or crop_w == 0: return None
left_eye_crop = left_eye - np.array([x1_crop, y1_crop])
right_eye_crop = right_eye - np.array([x1_crop, y1_crop])
rot_mat = cv2.getRotationMatrix2D((crop_w/2, crop_h/2), angle, 1.0)
cos_a = np.abs(rot_mat[0, 0])
sin_a = np.abs(rot_mat[0, 1])
new_w = int(crop_h * sin_a + crop_w * cos_a)
new_h = int(crop_h * cos_a + crop_w * sin_a)
rot_mat[0, 2] += (new_w / 2) - crop_w / 2
rot_mat[1, 2] += (new_h / 2) - crop_h / 2
rotated = cv2.warpAffine(cropped, rot_mat, (new_w, new_h))
left_eye_rot = rot_mat[:, :2] @ left_eye_crop + rot_mat[:, 2]
right_eye_rot = rot_mat[:, :2] @ right_eye_crop + rot_mat[:, 2]
rot_eye_distance = np.linalg.norm(right_eye_rot - left_eye_rot)
final_scale = target_eye_distance / rot_eye_distance
tx = TARGET_LEFT_EYE[0] - left_eye_rot[0] * final_scale
ty = TARGET_LEFT_EYE[1] - left_eye_rot[1] * final_scale
transform_mat = np.array([[final_scale, 0, tx], [0, final_scale, ty]], dtype=np.float32)
face_img = cv2.warpAffine(rotated, transform_mat, (112, 112))
face_img = cv2.cvtColor(face_img, cv2.COLOR_RGB2BGR)
return face_img
return None
def search(self, query=None, top_k=10, cam_name=None, timestamp=None, text_embedding=None, is_face=False):
embeddings = self.face_embeddings if is_face else self.image_embeddings
if not embeddings:
print("No embeddings available.")
return []
if text_embedding is None:
text_embedding = self.model._encode_text(query)
text_embedding = text_embedding.numpy()
all_similarities = []
for path, img_embedding in embeddings.items():
if img_embedding is None: continue
normalized_path = path.replace("\\", "/")
if cam_name and f"/cameras/{cam_name}/" not in normalized_path:
continue
if timestamp and f"/objects/{timestamp}/" not in normalized_path and "/objects/video/" not in normalized_path:
continue
similarity = (img_embedding @ text_embedding.T).item()
filename = os.path.basename(path)
if filename.lower().endswith((".jpg")):
object_id = event_img_info(filename.split(".jpg")[0])["object_id"] if "_" in filename else None
all_similarities.append((path, similarity, object_id))
if any(item[2] for item in all_similarities):
best_per_id = {}
for path, score, object_id in all_similarities:
if object_id is not None:
if object_id not in best_per_id or score > best_per_id[object_id][1]:
best_per_id[object_id] = (path, score)
items_without_id = [(path, score) for path, score, object_id in all_similarities if object_id is None]
results = list(best_per_id.values()) + items_without_id
else:
results = [(path, score) for path, score, _ in all_similarities]
results.sort(key=lambda x: x[1], reverse=True)
return results[:top_k]
def _load_all_embeddings(self, face=False):
total_loaded = 0
valid_paths = set()
cache_filename = "embeddings.pkl"
target_embeddings = self.face_embeddings if face else self.image_embeddings
for camera_folder in os.listdir(self.base_path):
camera_path = os.path.join(self.base_path, camera_folder)
if not os.path.isdir(camera_path): continue
objects_path = os.path.join(camera_path, "faces" if face else "objects")
if not os.path.isdir(objects_path): continue
for date_folder in os.listdir(objects_path):
date_path = os.path.join(objects_path, date_folder)
if not os.path.isdir(date_path): continue
cache_file = os.path.join(date_path, cache_filename)
if not os.path.exists(cache_file): continue
with open(cache_file, "rb") as f:
cache = pickle.load(f)
folder_embeddings = cache.get("embeddings", {})
valid_paths.update(folder_embeddings.keys())
target_embeddings.update(folder_embeddings)
total_loaded += len(folder_embeddings)
stale_keys = set(target_embeddings.keys()) - valid_paths
for k in stale_keys: del target_embeddings[k]
if face:
self.face_embeddings = target_embeddings
else:
self.image_embeddings = target_embeddings
print(f"\nTotal {'face' if face else 'image'} embeddings loaded: {total_loaded}")