Add object detection, tracking, mobile notifications, and search to any security camera.
Tiny clip3 (#105)
Rory Clearac1dd96a378d
diff --git a/test/clip_images/embeddings.pkl b/test/clip_images/embeddings.pklindex b5fd91c..0bbbc6a 100644Binary files a/test/clip_images/embeddings.pkl and b/test/clip_images/embeddings.pkl differdiff --git a/tiny_clip.py b/tiny_clip.pyindex 1217c25..8bbb2a1 100644--- a/tiny_clip.py+++ b/tiny_clip.py@@ -1,103 +1,81 @@-import torchfrom PIL import Image-import open_clipimport osimport pickle-import timefrom datetime import datetime-from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit+from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit, Device+from tinygrad.helpers import fetchfrom torchvision.transforms import functional as Ffrom tinygrad.dtype import dtypes-# DO NOT USE WITH BEAM+import numpy as np+import gc++class TinyModel: pass++class CachedCLIPSearch:def __init__(self, model_name="ViT-L-14", pretrained_name="laion2b_s32b_b82k"):- print(f"Loading OpenCLIP model: {model_name} ({pretrained_name})")-- if os.path.exists("laion2b_s32b_b82k.pkl"):- # Load from existing pickle file- with open("laion2b_s32b_b82k.pkl", 'rb') as f:- self.model = pickle.load(f)- print("Loaded data from pickle file")- else:- self.model, _, _ = open_clip.create_model_and_transforms(- model_name,- pretrained=pretrained_name- )-- self.model = self.model.to("cpu").eval()-- self.tokenizer = open_clip.get_tokenizer(model_name)-- self.image_embeddings = {}- self.image_paths = {}-- # convert- self.model.visual.tiny_vc = tiny_nn.Conv2d(in_channels=self.model.visual.conv1.in_channels, out_channels=self.model.visual.conv1.out_channels, kernel_size=self.model.visual.conv1.kernel_size, stride=self.model.visual.conv1.stride,\- padding=self.model.visual.conv1.padding, dilation=self.model.visual.conv1.dilation, groups=self.model.visual.conv1.groups, bias=False)- self.model.visual.tiny_vc.weight = tiny_Tensor(self.model.visual.conv1.weight.detach().numpy().copy())-- self.model.visual.tiny_class_embedding = tiny_Tensor(self.model.visual.class_embedding.detach().numpy().copy())- self.model.visual.tiny_positional_embedding = tiny_Tensor(self.model.visual.positional_embedding.detach().numpy().copy())-- self.model.visual.tiny_ln_pre = tiny_nn.LayerNorm(self.model.visual.ln_pre.weight.shape[0])- self.model.visual.tiny_ln_pre.weight = tiny_Tensor(self.model.visual.ln_pre.weight.detach().numpy().copy())- self.model.visual.tiny_ln_pre.bias = tiny_Tensor(self.model.visual.ln_pre.bias.detach().numpy().copy())-- self.model.visual.tiny_ln_post = tiny_nn.LayerNorm(self.model.visual.ln_post.weight.shape[0])- self.model.visual.tiny_ln_post.weight = tiny_Tensor(self.model.visual.ln_post.weight.detach().numpy().copy())- self.model.visual.tiny_ln_post.bias = tiny_Tensor(self.model.visual.ln_post.bias.detach().numpy().copy())-- self.model.tiny_ln_1 = []- self.model.tiny_ln_2 = []- self.model.tiny_in_proj_weight = []- self.model.tiny_in_proj_bias = []-- self.model.tiny_out_proj_weight = []- self.model.tiny_out_proj_bias = []-- self.model.tiny_c_fc = []- self.model.tiny_c_proj = []-- self.model.visual.tiny_proj = tiny_Tensor(self.model.visual.proj.detach().numpy().copy())-- for resblock in self.model.visual.transformer.resblocks:- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_1.normalized_shape, eps=resblock.ln_1.eps, elementwise_affine=True)- layernorm.weight = tiny_Tensor(resblock.ln_1.weight.detach().numpy().copy())- layernorm.bias = tiny_Tensor(resblock.ln_1.bias.detach().numpy().copy())- self.model.tiny_ln_1.append(layernorm)-- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_2.normalized_shape, eps=resblock.ln_2.eps, elementwise_affine=True)- layernorm.weight = tiny_Tensor(resblock.ln_2.weight.detach().numpy().copy())- layernorm.bias = tiny_Tensor(resblock.ln_2.bias.detach().numpy().copy())- self.model.tiny_ln_2.append(layernorm)-- w = tiny_Tensor(resblock.attn.in_proj_weight.detach().numpy().copy())- self.model.tiny_in_proj_weight.append(w)-- b = tiny_Tensor(resblock.attn.in_proj_bias.detach().numpy().copy())- self.model.tiny_in_proj_bias.append(b)-- w = tiny_Tensor(resblock.attn.out_proj.weight.detach().numpy().copy())- self.model.tiny_out_proj_weight.append(w)-- b = tiny_Tensor(resblock.attn.out_proj.bias.detach().numpy().copy())- self.model.tiny_out_proj_bias.append(b)-- linear = tiny_nn.Linear(resblock.mlp.c_fc.weight.shape[0], resblock.mlp.c_fc.weight.shape[1])- linear.weight = tiny_Tensor(resblock.mlp.c_fc.weight.detach().numpy().copy())- linear.bias = tiny_Tensor(resblock.mlp.c_fc.bias.detach().numpy().copy())- self.model.tiny_c_fc.append(linear)-- linear = tiny_nn.Linear(resblock.mlp.c_proj.weight.shape[0], resblock.mlp.c_proj.weight.shape[1])- linear.weight = tiny_Tensor(resblock.mlp.c_proj.weight.detach().numpy().copy())- linear.bias = tiny_Tensor(resblock.mlp.c_proj.bias.detach().numpy().copy())- self.model.tiny_c_proj.append(linear)-- with open('laion2b_s32b_b82k.pkl', 'wb') as f: pickle.dump(self.model, f)+ self.image_embeddings = {}+ self.image_paths = {}++ self.tiny_model = TinyModel()+ device = Device.DEFAULT+ # convert+ weights = tiny_nn.state.safe_load(fetch("http://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K/resolve/main/open_clip_pytorch_model.safetensors"))++ self.tiny_model.visual_conv1 = tiny_nn.Conv2d(3, 1024, (14, 14), (14, 14), (0, 0), (1, 1), 1, bias=False)+ self.tiny_model.visual_conv1.weight = weights["visual.conv1.weight"].to(device)++ self.tiny_model.class_embedding = weights["visual.class_embedding"].to(device)+ self.tiny_model.positional_embedding = weights["visual.positional_embedding"].to(device)+++ self.tiny_model.ln_pre = tiny_nn.LayerNorm(1024)+ self.tiny_model.ln_pre.weight = weights["visual.ln_pre.weight"].to(device)+ self.tiny_model.ln_pre.bias = weights["visual.ln_pre.bias"].to(device)+++ self.tiny_model.ln_post = tiny_nn.LayerNorm(1024)+ self.tiny_model.ln_post.weight = weights["visual.ln_post.weight"].to(device)+ self.tiny_model.ln_post.bias = weights["visual.ln_post.bias"].to(device)++ self.tiny_model.proj = weights["visual.proj"].to(device)++ self.tiny_model.resblocks = []++ for i in range(24):+ tiny_resblock = TinyModel()++ tiny_resblock.ln_1 = tiny_nn.LayerNorm(1024, 1e-05, elementwise_affine=True)+ tiny_resblock.ln_1.weight = weights[f"visual.transformer.resblocks.{i}.ln_1.weight"].to(device)+ tiny_resblock.ln_1.bias = weights[f"visual.transformer.resblocks.{i}.ln_1.bias"].to(device)++ tiny_resblock.ln_2 = tiny_nn.LayerNorm(1024, 1e-05, elementwise_affine=True)+ tiny_resblock.ln_2.weight = weights[f"visual.transformer.resblocks.{i}.ln_2.weight"].to(device)+ tiny_resblock.ln_2.bias = weights[f"visual.transformer.resblocks.{i}.ln_2.bias"].to(device)++ tiny_resblock.in_proj_weight = weights[f"visual.transformer.resblocks.{i}.attn.in_proj_weight"].to(device)+ tiny_resblock.in_proj_bias = weights[f"visual.transformer.resblocks.{i}.attn.in_proj_bias"].to(device)++ tiny_resblock.out_proj_weight = weights[f"visual.transformer.resblocks.{i}.attn.out_proj.weight"].to(device)+ tiny_resblock.out_proj_bias = weights[f"visual.transformer.resblocks.{i}.attn.out_proj.bias"].to(device)+++ tiny_resblock.mlp_c_fc = tiny_nn.Linear(4096, 1024)+ tiny_resblock.mlp_c_fc.weight = weights[f"visual.transformer.resblocks.{i}.mlp.c_fc.weight"].to(device)+ tiny_resblock.mlp_c_fc.bias = weights[f"visual.transformer.resblocks.{i}.mlp.c_fc.bias"].to(device)++ tiny_resblock.mlp_c_proj = tiny_nn.Linear(1024, 4096)+ tiny_resblock.mlp_c_proj.weight = weights[f"visual.transformer.resblocks.{i}.mlp.c_proj.weight"].to(device)+ tiny_resblock.mlp_c_proj.bias = weights[f"visual.transformer.resblocks.{i}.mlp.c_proj.bias"].to(device)++ self.tiny_model.resblocks.append(tiny_resblock)++ weights = None+ gc.collect()# for BEAM- tiny_precompute_embeddings(self.model, tiny_Tensor.rand((1, 3, 224, 224), dtype=dtypes.float32))- tiny_precompute_embeddings(self.model, tiny_Tensor.rand((16, 3, 224, 224), dtype=dtypes.float32))+ tiny_precompute_embeddings(self.tiny_model, tiny_Tensor.rand((1, 3, 224, 224), dtype=dtypes.float32))+ tiny_precompute_embeddings(self.tiny_model, tiny_Tensor.rand((16, 3, 224, 224), dtype=dtypes.float32))def find_object_folders(self, base_path="data/cameras"):object_folders = []@@ -170,19 +148,18 @@ class CachedCLIPSearch:print(f"Error loading {img_path}: {e}")continue- batch_tensors = torch.stack([inline_preprocess(img) for img in batch_images])+ batch_tensors = np.stack([inline_preprocess(img) for img in batch_images])if len(batch_images) == batch_size:- x = tiny_Tensor(batch_tensors.detach().numpy())- embeddings = tiny_precompute_embeddings(self.model, x)- embeddings = torch.Tensor(embeddings.numpy())+ x = tiny_Tensor(batch_tensors)+ embeddings = tiny_precompute_embeddings(self.tiny_model, x)+ embeddings = embeddings.numpy()else:embeddings = []for i in range(len(batch_images)): # one by one if not batch_size- single_x = tiny_Tensor(batch_tensors[i:i+1].detach().numpy())- single_embedding = tiny_precompute_embedding(self.model, single_x)- single_embedding_torch = torch.Tensor(single_embedding.numpy())+ single_x = tiny_Tensor(batch_tensors[i:i+1])+ single_embedding = tiny_precompute_embedding(self.tiny_model, single_x)+ single_embedding_torch = single_embedding.numpy()embeddings.append(single_embedding_torch)- embeddings = torch.stack(embeddings)for path, embedding in zip(batch_paths, embeddings):folder_embeddings[path] = embedding@@ -200,25 +177,24 @@ class CachedCLIPSearch:print(f"{datetime.now()}: Updated cache for {folder_path}. Total images stored: {len(folder_embeddings)}")@TinyJit-def tiny_precompute_embeddings(model, x): return tiny_precompute_embedding(model, x)+def tiny_precompute_embeddings(tiny_model, x): return tiny_precompute_embedding(tiny_model, x)-def tiny_precompute_embedding(model, x):- visual = model.visual- x = visual.tiny_vc(x)+def tiny_precompute_embedding(tiny_model, x):+ x = tiny_model.visual_conv1(x)x = x.reshape(x.shape[0], x.shape[1], -1)x = x.permute(0, 2, 1)- cls_emb = visual.tiny_class_embedding+ cls_emb = tiny_model.class_embeddingcls_emb = cls_emb.unsqueeze(0).expand(x.shape[0], -1, -1)x = tiny_Tensor.cat(cls_emb, x, dim=1)- x = x + visual.tiny_positional_embedding- x = visual.tiny_ln_pre(x)+ x = x + tiny_model.positional_embedding+ x = tiny_model.ln_pre(x)# https://github.com/pytorch/pytorch/blob/v2.9.1/torch/nn/modules/activation.py#L1252- for i, block in enumerate(visual.transformer.resblocks):- x_ln1 = model.tiny_ln_1[i](x)+ for i in range(len(tiny_model.resblocks)):+ x_ln1 = tiny_model.resblocks[i].ln_1(x)B, L, D = x_ln1.shape- H = block.attn.num_heads+ H = 16 #block.attn.num_headsd_head = D // H- qkv = x_ln1 @ model.tiny_in_proj_weight[i].T + model.tiny_in_proj_bias[i]+ qkv = x_ln1 @ tiny_model.resblocks[i].in_proj_weight.T + tiny_model.resblocks[i].in_proj_biasq, k, v = qkv.split(D, dim=-1)def shape(x): return x.view(B, L, H, d_head).transpose(1, 2)q = shape(q)@@ -229,18 +205,18 @@ def tiny_precompute_embedding(model, x):attn_probs = tiny_Tensor.softmax(attn_scores)context = attn_probs.matmul(v)context = context.transpose(1, 2).contiguous().view(B, L, D)- attn_out = context @ model.tiny_out_proj_weight[i].T + model.tiny_out_proj_bias[i]- attn_scaled = block.ls_1(attn_out)+ attn_out = context @ tiny_model.resblocks[i].out_proj_weight.T + tiny_model.resblocks[i].out_proj_bias+ attn_scaled = attn_outx = x + attn_scaled- x_ln2 = model.tiny_ln_2[i](x)- ff = model.tiny_c_fc[i](x_ln2)+ x_ln2 = tiny_model.resblocks[i].ln_2(x)+ ff = tiny_model.resblocks[i].mlp_c_fc(x_ln2)ff = ff.gelu()- ff = model.tiny_c_proj[i](ff)+ ff = tiny_model.resblocks[i].mlp_c_proj(ff)x = x + ff- x = visual.tiny_ln_post(x)+ x = tiny_model.ln_post(x)image_embeds = x[:, 0, :]- embeddings = image_embeds @ visual.tiny_proj+ embeddings = image_embeds @ tiny_model.projembeddings = embeddings / (embeddings.pow(2).sum(axis=-1, keepdim=True).sqrt() + 1e-8)return embeddings@@ -270,4 +246,4 @@ if __name__ == "__main__":print(f"{datetime.now()}: Sleeping for 15 mins...")time.sleep(900)-'''\ No newline at end of file+'''diff --git a/tiny_clip_search.py b/tiny_clip_search.pyindex 4011361..dcc2e0e 100644--- a/tiny_clip_search.py+++ b/tiny_clip_search.py@@ -1,63 +1,80 @@import osimport pickle-import torch-import open_clip-from typing import Optional-from torch.nn import functional as F-import torch.nn as nn-from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit+from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit, Device+from tinygrad.helpers import fetch+from utils.clip_tokenizer import SimpleTokenizer+import numpy as np+import gc++class TinyModel:+ passclass CLIPSearch:def __init__(self, base_path="data/cameras"):self.base_path = base_pathself.image_embeddings = {}self.image_paths = {}- self.device = torch.device("cpu")- self.model, _, self.preprocess = open_clip.create_model_and_transforms(- 'ViT-L-14',- pretrained='laion2b_s32b_b82k'- )- self.model = self.model.to(self.device).eval()-- # convert here- # .copy() or trouble- self.model.tiny_text_projection = tiny_Tensor(self.model.text_projection.detach().numpy().copy())- self.model.tiny_token_embedding = tiny_nn.Embedding(self.model.token_embedding.num_embeddings, self.model.token_embedding.embedding_dim)- self.model.tiny_token_embedding.weight = tiny_Tensor(self.model.token_embedding.weight.data.detach().numpy().copy())+ self.tokenizer = SimpleTokenizer()++ device = Device.DEFAULT+++ self.model = TinyModel()++ weights = tiny_nn.state.safe_load(fetch("http://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K/resolve/main/open_clip_pytorch_model.safetensors"))++ self.model.text_projection = weights["text_projection"].to(device)+ self.model.positional_embedding = weights["positional_embedding"].to(device)++ self.model.token_embedding = tiny_nn.Embedding(49408, 768) # todo unhardcode+ self.model.token_embedding.weight = weights["token_embedding.weight"].to(device)++ self.model.ln_final = tiny_nn.LayerNorm(768, eps=1e-5, elementwise_affine=True)+ self.model.ln_final.weight = weights["ln_final.weight"].to(device)+ self.model.ln_final.bias = weights["ln_final.bias"].to(device)++ attn_mask = np.where(np.tri(77, dtype=bool), 0.0, -np.inf).astype(np.float32)+ self.model.attn_mask = tiny_Tensor(attn_mask)++ self.model.resblocks = []- self.model.tiny_ln_1 = []- self.model.tiny_ln_2 = []- self.model.out_proj_bias_tiny = []- self.model.mlp_c_fc = []- self.model.mlp_c_proj = []- for resblock in self.model.transformer.resblocks:- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_1.normalized_shape, eps=resblock.ln_1.eps, elementwise_affine=True)- layernorm.weight = tiny_Tensor(resblock.ln_1.weight.detach().numpy().copy())- layernorm.bias = tiny_Tensor(resblock.ln_1.bias.detach().numpy().copy())- self.model.tiny_ln_1.append(layernorm)-- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_2.normalized_shape, eps=resblock.ln_2.eps, elementwise_affine=True)- layernorm.weight = tiny_Tensor(resblock.ln_2.weight.detach().numpy().copy())- layernorm.bias = tiny_Tensor(resblock.ln_2.bias.detach().numpy().copy())- self.model.tiny_ln_2.append(layernorm)-- opb = tiny_Tensor(resblock.attn.out_proj.bias.detach().numpy().copy())- self.model.out_proj_bias_tiny.append(opb)-- # todo- mlpcfc = tiny_nn.Linear(resblock.mlp.c_fc.weight.shape[0], resblock.mlp.c_fc.weight.shape[1])- mlpcfc.weight = tiny_Tensor(resblock.mlp.c_fc.weight.detach().numpy().copy())- mlpcfc.bias = tiny_Tensor(resblock.mlp.c_fc.bias.detach().numpy().copy())- self.model.mlp_c_fc.append(mlpcfc)-- mlpcp = tiny_nn.Linear(resblock.mlp.c_proj.weight.shape[0], resblock.mlp.c_proj.weight.shape[1])- mlpcp.weight = tiny_Tensor(resblock.mlp.c_proj.weight.detach().numpy().copy())- mlpcp.bias = tiny_Tensor(resblock.mlp.c_proj.bias.detach().numpy().copy())- self.model.mlp_c_proj.append(mlpcp)-- self.model.tiny_ln_final = tiny_nn.LayerNorm(normalized_shape=self.model.ln_final.normalized_shape, eps=self.model.ln_final.eps, elementwise_affine=True)- self.model.tiny_ln_final .weight = tiny_Tensor(self.model.ln_final.weight.detach().numpy().copy())- self.model.tiny_ln_final .bias = tiny_Tensor(self.model.ln_final.bias.detach().numpy().copy())+ for i in range(12):+ resblock = TinyModel()+ layernorm = tiny_nn.LayerNorm(768, 1e-5, elementwise_affine=True)+ layernorm.weight = weights[f"transformer.resblocks.{i}.ln_1.weight"].to(device)+ layernorm.bias = weights[f"transformer.resblocks.{i}.ln_1.bias"].to(device)+ resblock.ln_1 = layernorm++ layernorm = tiny_nn.LayerNorm(768, 1e-5, elementwise_affine=True)+ layernorm.weight = weights[f"transformer.resblocks.{i}.ln_2.weight"].to(device)+ layernorm.bias = weights[f"transformer.resblocks.{i}.ln_2.bias"].to(device)+ resblock.ln_2 = layernorm++ weight = weights[f"transformer.resblocks.{i}.attn.out_proj.weight"].to(device)+ resblock.attn_out_proj_weight = weight++ bias = weights[f"transformer.resblocks.{i}.attn.out_proj.bias"].to(device)+ resblock.attn_out_proj_bias = bias++ mlpcfc = tiny_nn.Linear(3072, 768)+ mlpcfc.weight = weights[f"transformer.resblocks.{i}.mlp.c_fc.weight"].to(device)+ mlpcfc.bias = weights[f"transformer.resblocks.{i}.mlp.c_fc.bias"].to(device)+ resblock.mlp_c_fc = mlpcfc++ mlpcp = tiny_nn.Linear(768, 3072)+ mlpcp.weight = weights[f"transformer.resblocks.{i}.mlp.c_proj.weight"].to(device)+ mlpcp.bias = weights[f"transformer.resblocks.{i}.mlp.c_proj.bias"].to(device)+ resblock.mlp_c_proj = mlpcp++ weight = weights[f"transformer.resblocks.{i}.attn.in_proj_weight"].to(device)+ resblock.in_proj_weight = weight++ bias = weights[f"transformer.resblocks.{i}.attn.in_proj_bias"].to(device)+ resblock.in_proj_bias = bias++ self.model.resblocks.append(resblock)++ weights = Nonedef _load_single_embeddings_file(self, cache_file):try:@@ -107,8 +124,11 @@ class CLIPSearch:print(f"\nTotal images loaded: {total_loaded}")def _encode_text(self, query):- tokens = open_clip.tokenize([query]).to(self.device)- tokens = tiny_Tensor(tokens.detach().numpy())+ tokens = [49406]+ tokens += self.tokenizer.encode(query)+ tokens.append(49407)+ if len(tokens) < 77: tokens += [0] * (77 - len(tokens))+ tokens = tiny_Tensor([tokens])text_emb = encode_text(self.model, tokens)return text_emb@@ -117,7 +137,7 @@ class CLIPSearch:print("No embeddings available.")return []text_embedding = self._encode_text(query)- text_embedding = torch.Tensor(text_embedding.numpy())+ text_embedding = text_embedding.numpy()all_similarities = []for path, img_embedding in self.image_embeddings.items():normalized_path = path.replace("\\", "/")@@ -148,25 +168,18 @@ class CLIPSearch:@TinyJitdef encode_text(model, text):x = text- x = model.tiny_token_embedding(x)- if not hasattr(model, 'tiny_positional_embedding'): model.tiny_positional_embedding = tiny_Tensor(model.positional_embedding.detach().numpy())- x = x + model.tiny_positional_embedding+ x = model.token_embedding(x)+ x = x + model.positional_embedding- for i, resblock in enumerate(model.transformer.resblocks):+ for i in range(len(model.resblocks)):residual = x- x = model.tiny_ln_1[i](x)+ x = model.resblocks[i].ln_1(x)B, L, D = x.shape- H = resblock.attn.num_heads+ H = 12 #resblock.attn.num_headsd_head = D // H- in_proj_weight_tiny = tiny_Tensor(resblock.attn.in_proj_weight.detach().numpy()) # todo store these- in_proj_bias_tiny = tiny_Tensor(resblock.attn.in_proj_bias.detach().numpy())- out_proj_weight_tiny = tiny_Tensor(resblock.attn.out_proj.weight.detach().numpy())- attn_mask = tiny_Tensor(model.attn_mask.detach().numpy())--- qkv = x.matmul(in_proj_weight_tiny.T) + in_proj_bias_tiny+ qkv = x.matmul(model.resblocks[i].in_proj_weight.T) + model.resblocks[i].in_proj_biasq, k, v = qkv.split(D, dim=-1)q = q.view(B, L, H, d_head).transpose(1, 2)k = k.view(B, L, H, d_head).transpose(1, 2)@@ -174,25 +187,25 @@ def encode_text(model, text):scale = 1.0 / (d_head ** 0.5)attn_scores = q.matmul(k.transpose(-2, -1)) * scale # (B,H,L,L)- bool_mask = attn_mask < 0+ bool_mask = model.attn_mask < 0attn_scores = attn_scores.masked_fill(bool_mask, float("-inf"))attn_probs = tiny_Tensor.softmax(attn_scores)context = attn_probs.matmul(v)context = context.transpose(1, 2).contiguous().view(B, L, D)- x = context.matmul(out_proj_weight_tiny.T) + model.out_proj_bias_tiny[i]+ x = context.matmul(model.resblocks[i].attn_out_proj_weight.T) + model.resblocks[i].attn_out_proj_biasx += residualresidual = x- x = model.tiny_ln_2[i](x)- x = model.mlp_c_fc[i](x)+ x = model.resblocks[i].ln_2(x)+ x = model.resblocks[i].mlp_c_fc(x)x = x.gelu()- x = model.mlp_c_proj[i](x)+ x = model.resblocks[i].mlp_c_proj(x)x += residual- x = model.tiny_ln_final(x) # [batch_size, n_ctx, transformer.width]+ x = model.ln_final(x) # [batch_size, n_ctx, transformer.width]argmax = text.argmax()x = x[0][argmax]- x = x @ model.tiny_text_projection+ x = x @ model.text_projectionreturn x / (x * x).sum(axis=-1, keepdim=True).sqrt()'''@@ -205,4 +218,3 @@ if __name__ == "__main__":for i, (path, score) in enumerate(results, 1):print(f"{i}. Score: {score:.3f} - \"{path}\"")'''-diff --git a/utils/__pycache__/clip_tokenizer.cpython-311.pyc b/utils/__pycache__/clip_tokenizer.cpython-311.pycnew file mode 100644index 0000000..85203b6Binary files /dev/null and b/utils/__pycache__/clip_tokenizer.cpython-311.pyc differdiff --git a/utils/bpe_simple_vocab_16e6.txt.gz b/utils/bpe_simple_vocab_16e6.txt.gznew file mode 100644index 0000000..7b5088aBinary files /dev/null and b/utils/bpe_simple_vocab_16e6.txt.gz differdiff --git a/utils/clip_tokenizer.py b/utils/clip_tokenizer.pynew file mode 100644index 0000000..c83133d--- /dev/null+++ b/utils/clip_tokenizer.py@@ -0,0 +1,621 @@+""" CLIP tokenizer++Copied from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI.+"""+#https://github.com/mlfoundations/open_clip/blob/d3cdb734a2710feeb4c6307df037afa5f786a3e1/src/open_clip/tokenizer.py+import gzip+import html+import os+import random+import string+from functools import lru_cache, partial+from typing import Callable, List, Optional, Union, Dict+import warnings++import ftfy+import numpy as np+import regex as re+import torch++# https://stackoverflow.com/q/62691279+os.environ["TOKENIZERS_PARALLELISM"] = "false"+_nltk_init = False++DEFAULT_CONTEXT_LENGTH = 77 # default context length for OpenAI CLIP+++@lru_cache()+def default_bpe():+ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")++@lru_cache()+def bytes_to_unicode():+ """+ Returns list of utf-8 byte and a corresponding list of unicode strings.+ The reversible bpe codes work on unicode strings.+ This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.+ When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.+ This is a significant percentage of your normal, say, 32K bpe vocab.+ To avoid that, we want lookup tables between utf-8 bytes and unicode strings.+ And avoids mapping to whitespace/control characters the bpe code barfs on.+ """+ bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))+ cs = bs[:]+ n = 0+ for b in range(2**8):+ if b not in bs:+ bs.append(b)+ cs.append(2**8+n)+ n += 1+ cs = [chr(n) for n in cs]+ return dict(zip(bs, cs))+++def get_pairs(word):+ """Return set of symbol pairs in a word.+ Word is represented as tuple of symbols (symbols being variable-length strings).+ """+ pairs = set()+ prev_char = word[0]+ for char in word[1:]:+ pairs.add((prev_char, char))+ prev_char = char+ return pairs+++def basic_clean(text):+ text = ftfy.fix_text(text)+ text = html.unescape(html.unescape(text))+ return text.strip()+++def whitespace_clean(text):+ text = " ".join(text.split())+ text = text.strip()+ return text+++def _clean_canonicalize(x):+ # basic, remove whitespace, remove punctuation, lower case+ return canonicalize_text(basic_clean(x))+++def _clean_lower(x):+ # basic, remove whitespace, lower case+ return whitespace_clean(basic_clean(x)).lower()+++def _clean_whitespace(x):+ # basic, remove whitespace+ return whitespace_clean(basic_clean(x))+++def get_clean_fn(type: str):+ if type == 'canonicalize':+ return _clean_canonicalize+ elif type == 'lower':+ return _clean_lower+ elif type == 'whitespace':+ return _clean_whitespace+ else:+ assert False, f"Invalid clean function ({type})."+++def canonicalize_text(+ text,+ *,+ keep_punctuation_exact_string=None,+ trans_punctuation: dict = str.maketrans("", "", string.punctuation),+):+ """Returns canonicalized `text` (lowercase and punctuation removed).++ From: https://github.com/google-research/big_vision/blob/53f18caf27a9419231bbf08d3388b07671616d3d/big_vision/evaluators/proj/image_text/prompt_engineering.py#L94++ Args:+ text: string to be canonicalized.+ keep_punctuation_exact_string: If provided, then this exact string kept.+ For example providing '{}' will keep any occurrences of '{}' (but will+ still remove '{' and '}' that appear separately).+ """+ text = text.replace("_", " ")+ if keep_punctuation_exact_string:+ text = keep_punctuation_exact_string.join(+ part.translate(trans_punctuation)+ for part in text.split(keep_punctuation_exact_string)+ )+ else:+ text = text.translate(trans_punctuation)+ text = text.lower()+ text = " ".join(text.split())+ return text.strip()+++class SimpleTokenizer(object):+ def __init__(+ self,+ bpe_path: str = default_bpe(),+ additional_special_tokens: Optional[List[str]] = None,+ context_length: Optional[int] = DEFAULT_CONTEXT_LENGTH,+ clean: str = 'lower',+ reduction_mask: str = ''+ ):+ self.byte_encoder = bytes_to_unicode()+ self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}+ merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')+ merges = merges[1:49152-256-2+1]+ merges = [tuple(merge.split()) for merge in merges]+ vocab = list(bytes_to_unicode().values())+ vocab = vocab + [v+'</w>' for v in vocab]+ for merge in merges:+ vocab.append(''.join(merge))+ special_tokens = ['<start_of_text>', '<end_of_text>']+ if additional_special_tokens:+ special_tokens += additional_special_tokens+ vocab.extend(special_tokens)+ self.encoder = dict(zip(vocab, range(len(vocab))))+ self.decoder = {v: k for k, v in self.encoder.items()}+ self.bpe_ranks = dict(zip(merges, range(len(merges))))+ self.cache = {t:t for t in special_tokens}+ special = "|".join(special_tokens)+ self.pat = re.compile(+ special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",+ re.IGNORECASE,+ )+ self.vocab_size = len(self.encoder)+ self.all_special_ids = [self.encoder[t] for t in special_tokens]+ self.sot_token_id = self.all_special_ids[0]+ self.eot_token_id = self.all_special_ids[1]+ self.context_length = context_length+ self.clean_fn = get_clean_fn(clean)+ self.reduction_fn = get_reduction_mask_fn(reduction_mask) if reduction_mask else None++ def bpe(self, token):+ if token in self.cache:+ return self.cache[token]+ word = tuple(token[:-1]) + ( token[-1] + '</w>',)+ pairs = get_pairs(word)++ if not pairs:+ return token+'</w>'++ while True:+ bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))+ if bigram not in self.bpe_ranks:+ break+ first, second = bigram+ new_word = []+ i = 0+ while i < len(word):+ try:+ j = word.index(first, i)+ new_word.extend(word[i:j])+ i = j+ except Exception:+ new_word.extend(word[i:])+ break++ if word[i] == first and i < len(word)-1 and word[i+1] == second:+ new_word.append(first+second)+ i += 2+ else:+ new_word.append(word[i])+ i += 1+ new_word = tuple(new_word)+ word = new_word+ if len(word) == 1:+ break+ else:+ pairs = get_pairs(word)+ word = ' '.join(word)+ self.cache[token] = word+ return word++ def encode(self, text):+ bpe_tokens = []+ text = self.clean_fn(text)+ for token in re.findall(self.pat, text):+ token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))+ bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))+ return bpe_tokens++ def decode(self, tokens):+ text = ''.join([self.decoder[token] for token in tokens])+ text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')+ return text++ def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.LongTensor:+ """ Returns the tokenized representation of given input string(s)++ Parameters+ ----------+ texts : Union[str, List[str]]+ An input string or a list of input strings to tokenize+ context_length : int+ The context length to use; all CLIP models use 77 as the context length++ Returns+ -------+ A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]+ """+ if isinstance(texts, str):+ texts = [texts]++ context_length = context_length or self.context_length+ assert context_length, 'Please set a valid context length'++ if self.reduction_fn is not None:+ # use reduction strategy for tokenize if set, otherwise default to truncation below+ return self.reduction_fn(+ texts,+ context_length=context_length,+ sot_token_id=self.sot_token_id,+ eot_token_id=self.eot_token_id,+ encode_fn=self.encode,+ )++ all_tokens = [[self.sot_token_id] + self.encode(text) + [self.eot_token_id] for text in texts]+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)++ for i, tokens in enumerate(all_tokens):+ if len(tokens) > context_length:+ tokens = tokens[:context_length] # Truncate+ tokens[-1] = self.eot_token_id+ result[i, :len(tokens)] = torch.tensor(tokens)++ return result+++_tokenizer = SimpleTokenizer()+++def decode(output_ids: torch.Tensor):+ output_ids = output_ids.cpu().numpy()+ return _tokenizer.decode(output_ids)+++def tokenize(texts: Union[str, List[str]], context_length: int = DEFAULT_CONTEXT_LENGTH) -> torch.LongTensor:+ return _tokenizer(texts, context_length=context_length)+++def random_mask_tokenize(+ texts: Union[str, List[str]],+ context_length: int,+ sot_token_id: int,+ eot_token_id: int,+ encode_fn: Callable,+ shuffle: bool = False,+):+ all_tokens = [encode_fn(text) for text in texts]+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)++ for i, tokens in enumerate(all_tokens):+ tokens = torch.tensor(tokens)+ num_tokens = len(tokens)+ if num_tokens > context_length - 2: # 2 for sot and eot token+ num_keep = context_length - 2+ indices = torch.randperm(len(tokens))+ indices = indices[:num_keep]+ if not shuffle:+ indices = indices.msort()+ tokens = tokens[indices]+ num_tokens = num_keep+ result[i, 0] = sot_token_id+ result[i, 1:num_tokens + 1] = tokens+ result[i, num_tokens + 1] = eot_token_id++ return result+++def simple_mask_tokenize(+ texts: Union[str, List[str]],+ context_length: int,+ sot_token_id: int,+ eot_token_id: int,+ encode_fn: Callable,+):+ all_tokens = [encode_fn(text) for text in texts]+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)++ for i, tokens in enumerate(all_tokens):+ num_tokens = len(tokens)+ if num_tokens > context_length - 2: # 2 for sot and eot token+ num_keep = context_length - 2+ start_index = random.randint(0, num_tokens - num_keep) # high is incl+ tokens = tokens[start_index: start_index + num_keep]+ tokens = [sot_token_id] + tokens + [eot_token_id]+ result[i, :len(tokens)] = torch.tensor(tokens)++ return result+++def syntax_mask_tokenize(+ texts: Union[str, List[str]],+ context_length: int,+ sot_token_id: int,+ eot_token_id: int,+ encode_fn: Callable,+) -> torch.LongTensor:+ """ Returns the tokenized representation of given input string(s).+ Apply syntax masking before tokenize.+ """+ import nltk+ global _nltk_init+ if not _nltk_init:+ # run them for the first time+ nltk.download('punkt')+ nltk.download('averaged_perceptron_tagger')+ _nltk_init = True++ def get_order(x):+ if x.startswith('NN'):+ return 1+ elif x.startswith('JJ'):+ return 2+ elif x.startswith('VB'):+ return 3+ else:+ return 4++ # syntax masking+ new_texts = []+ for text in texts:+ list_tokens = nltk.tokenize.word_tokenize(text)+ pos_tags = nltk.pos_tag(list_tokens)+ # sample the words by get_order method+ order_list = [get_order(tag) for _, tag in pos_tags]+ sorted_ids = np.argsort(np.array(order_list))+ sampled_ids = sorted(sorted_ids[:context_length - 2]) # need 2 slots for sot and eot tokens+ sampled_tokens = np.take(np.array(list_tokens), sampled_ids, axis=0) # sample the tokens++ new_text = ''+ for token in sampled_tokens:+ new_text = new_text + str(token) + ' '+ new_text = new_text.strip()+ new_texts.append(new_text)+ texts = new_texts++ all_tokens = [[sot_token_id] + encode_fn(text) + [eot_token_id] for text in texts]+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)++ for i, tokens in enumerate(all_tokens):+ # still need first truncate because some words produces two tokens+ if len(tokens) > context_length:+ tokens = tokens[:context_length] # Truncate+ tokens[-1] = eot_token_id+ result[i, :len(tokens)] = torch.tensor(tokens)++ return result+++def get_reduction_mask_fn(type: str):+ """ Choose strategy for dropping (masking) tokens to achieve target context length"""+ assert type in ('simple', 'random', 'shuffle', 'syntax')+ if type == 'simple':+ return simple_mask_tokenize # randomly select block [start:end]+ elif type == 'random':+ return random_mask_tokenize # randomly drop tokens (keep order)+ elif type == 'shuffle':+ return partial(random_mask_tokenize, shuffle=True) # randomly drop tokens (shuffle order)+ elif type == 'syntax':+ return syntax_mask_tokenize # randomly drop prioritized by syntax+ else:+ assert False, F'Unknown type {type}.'+++class HFTokenizer:+ """HuggingFace tokenizer wrapper with support for custom tokenization modes"""++ def __init__(+ self,+ tokenizer_name: str,+ context_length: Optional[int] = DEFAULT_CONTEXT_LENGTH,+ clean: str = 'whitespace',+ strip_sep_token: bool = False,+ language: Optional[str] = None,+ cache_dir: Optional[str] = None,+ tokenizer_mode: Optional[str] = None, # None, 'clips'+ **kwargs+ ):+ self.tokenizer_mode = tokenizer_mode or ''+ self.context_length = context_length+ self.clean_fn = get_clean_fn(clean)+ self.strip_sep_token = strip_sep_token++ # NOTE: Left as example of loading custom tokenizer from file for experimentation+ # if self.tokenizer_mode == 'bert_clips':+ # self.special_tokens = {+ # "bos_token": 1,+ # "eos_token": 2,+ # "cls_token": 101,+ # "pad_token": 0+ # }+ #+ # # For BERT CLIPS mode with vocab file+ # from tokenizers import BertWordPieceTokenizer+ # if tokenizer_name.startswith('hf-hub:'):+ # from huggingface_hub import hf_hub_download+ # # Format: hf-hub:repo_id/filename+ # repo_url = tokenizer_name[7:]+ # parts = repo_url.split('/')+ # filename = parts[-1]+ # repo_id = '/'.join(parts[:-1])+ # vocab_file = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=cache_dir)+ # self.tokenizer = BertWordPieceTokenizer(lowercase=True)+ # self.tokenizer = self.tokenizer.from_file(vocab_file)+ # else:+ # # Assume tokenizer_name is a local path to a vocab file+ # self.tokenizer = BertWordPieceTokenizer(lowercase=True)+ # self.tokenizer = self.tokenizer.from_file(tokenizer_name)++ # Standard HuggingFace tokenizer initialization+ from transformers import AutoTokenizer+ self.tokenizer = AutoTokenizer.from_pretrained(+ tokenizer_name,+ cache_dir=cache_dir,+ **kwargs+ )++ # Set language function if available+ set_lang_fn = getattr(self.tokenizer, 'set_src_lang_special_tokens', None)+ if callable(set_lang_fn):+ self.set_lang_fn = set_lang_fn+ if language is not None:+ self.set_language(language)++ def save_pretrained(self, dest):+ self.tokenizer.save_pretrained(dest)++ def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.Tensor:+ # same cleaning as for default tokenizer, except lowercasing+ # adding lower (for case-sensitive tokenizers) will make it more robust but less sensitive to nuance+ if isinstance(texts, str):+ texts = [texts]++ context_length = context_length or self.context_length+ assert context_length, 'Please set a valid context length in class init or call.'++ texts = [self.clean_fn(text) for text in texts]++ # Handle different tokenization modes+ if self.tokenizer_mode == 'clips':+ return self._clips_tokenize(texts, context_length)+ else:+ # Standard tokenization+ input_ids = self.tokenizer.batch_encode_plus(+ texts,+ return_tensors='pt',+ max_length=context_length,+ padding='max_length',+ truncation=True,+ ).input_ids++ if self.strip_sep_token:+ input_ids = torch.where(+ input_ids == self.tokenizer.sep_token_id,+ torch.zeros_like(input_ids),+ input_ids,+ )++ return input_ids++ def set_language(self, src_lang):+ if hasattr(self, 'set_lang_fn'):+ self.set_lang_fn(src_lang)+ else:+ warnings.warn('Cannot set language for the tokenizer.')++ def _clips_tokenize(self, texts: List[str], context_length: int) -> torch.Tensor:+ """Use standard HF tokenizer but apply custom post-processing"""+ # Use standard tokenizer without special tokens - we'll add our own+ encoded_outputs = self.tokenizer.batch_encode_plus(+ texts,+ add_special_tokens=False,+ padding=False,+ truncation=False,+ return_tensors=None+ )++ encoded = []+ for tokens in encoded_outputs["input_ids"]:+ tokens = tokens[:context_length - 3] # Leave room for special tokens+ tokens = [self.tokenizer.bos_token_id] + tokens + [self.tokenizer.eos_token_id]+ encoded.append(tokens)++ # Create result tensor and handle padding + class token+ result = torch.zeros(len(encoded), context_length, dtype=torch.long)+ for i, tokens in enumerate(encoded):+ padded_tokens = self._pad_and_add_class_token(+ tokens,+ max_length=context_length,+ pad_token_id=self.tokenizer.pad_token_id,+ cls_token_id=self.tokenizer.cls_token_id,+ )+ result[i, :len(padded_tokens)] = torch.tensor(padded_tokens)++ return result++ def _pad_and_add_class_token(+ self,+ tokens: List[int],+ max_length: int,+ pad_token_id: int = 0,+ cls_token_id: int = 101,+ ) -> List[int]:+ """ Add padding with class token at the end """+ if len(tokens) > max_length - 1:+ tokens = tokens[:max_length - 1]++ # Add padding to reach max_length-1+ if len(tokens) < max_length - 1:+ tokens = tokens + [pad_token_id] * (max_length - 1 - len(tokens))++ # Add class token at the end+ tokens = tokens + [cls_token_id]+ return tokens+++class SigLipTokenizer:+ """HuggingFace tokenizer wrapper for SigLIP T5 compatible sentencepiece vocabs++ NOTE: this is not needed in normal library use, but is used to import new sentencepiece tokenizers+ into OpenCLIP. Leaving code here in case future models use new tokenizers.+ """+ VOCAB_FILES = {+ # english, vocab_size=32_000+ "c4-en": "http://storage.googleapis.com/t5-data/vocabs/cc_en.32000/sentencepiece.model",+ # used in multilingual models (mT5, PaLI), vocab_size=250_000+ "mc4": "http://storage.googleapis.com/t5-data/vocabs/mc4.250000.100extra/sentencepiece.model",+ # used in SigLIP2 models, vocab_size=256000+ "gemma": "http://storage.googleapis.com/big_vision/gemma_tokenizer.model",+ }++ def __init__(+ self,+ tokenizer_name: str,+ context_length: Optional[int] = 64,+ ):+ if 'gemma' in tokenizer_name:+ from transformers import GemmaTokenizerFast+ tokenizer_cls = partial(+ GemmaTokenizerFast, padding_side='right', add_bos_token=False, add_eos_token=True)+ else:+ from transformers import T5TokenizerFast+ tokenizer_cls = partial(T5TokenizerFast, extra_ids=0)++ if tokenizer_name in self.VOCAB_FILES:+ # FIXME temporary hack?+ import tempfile+ import fsspec+ vocab_file = self.VOCAB_FILES[tokenizer_name]+ with tempfile.NamedTemporaryFile('wb') as dst:+ with fsspec.open(vocab_file, 'rb') as src:+ dst.write(src.read())+ self.tokenizer = tokenizer_cls(dst.name, legacy=False)+ else:+ self.tokenizer = tokenizer_cls(tokenizer_name, legacy=False)++ self.tokenizer.pad_token_id = 0 if 'gemma' in tokenizer_name else 1+ self.tokenizer.eos_token_id = 1+ self.context_length = context_length++ def save_pretrained(self, dest):+ self.tokenizer.save_pretrained(dest)++ def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.Tensor:+ # same cleaning as for default tokenizer, except lowercasing+ # adding lower (for case-sensitive tokenizers) will make it more robust but less sensitive to nuance+ if isinstance(texts, str):+ texts = [texts]++ context_length = context_length or self.context_length+ assert context_length, 'Please set a valid context length in class init or call.'++ texts = [canonicalize_text(basic_clean(text)) for text in texts]+ output = self.tokenizer(+ texts,+ return_tensors='pt',+ max_length=context_length,+ padding='max_length',+ truncation=True,+ )+ return output.input_ids\ No newline at end of file