Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

Tiny clip3 (#105)

Rory Clearac1dd96a378d
diff --git a/test/clip_images/embeddings.pkl b/test/clip_images/embeddings.pkl
index b5fd91c..0bbbc6a 100644
Binary files a/test/clip_images/embeddings.pkl and b/test/clip_images/embeddings.pkl differ
diff --git a/tiny_clip.py b/tiny_clip.py
index 1217c25..8bbb2a1 100644
--- a/tiny_clip.py
+++ b/tiny_clip.py
@@ -1,103 +1,81 @@
-import torch
from PIL import Image
-import open_clip
import os
import pickle
-import time
from datetime import datetime
-from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit
+from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit, Device
+from tinygrad.helpers import fetch
from torchvision.transforms import functional as F
from tinygrad.dtype import dtypes
-# DO NOT USE WITH BEAM
+import numpy as np
+import gc
+
+class TinyModel: pass
+
+
class CachedCLIPSearch:
def __init__(self, model_name="ViT-L-14", pretrained_name="laion2b_s32b_b82k"):
- print(f"Loading OpenCLIP model: {model_name} ({pretrained_name})")
-
- if os.path.exists("laion2b_s32b_b82k.pkl"):
- # Load from existing pickle file
- with open("laion2b_s32b_b82k.pkl", 'rb') as f:
- self.model = pickle.load(f)
- print("Loaded data from pickle file")
- else:
- self.model, _, _ = open_clip.create_model_and_transforms(
- model_name,
- pretrained=pretrained_name
- )
-
- self.model = self.model.to("cpu").eval()
-
- self.tokenizer = open_clip.get_tokenizer(model_name)
-
- self.image_embeddings = {}
- self.image_paths = {}
-
- # convert
- self.model.visual.tiny_vc = tiny_nn.Conv2d(in_channels=self.model.visual.conv1.in_channels, out_channels=self.model.visual.conv1.out_channels, kernel_size=self.model.visual.conv1.kernel_size, stride=self.model.visual.conv1.stride,\
- padding=self.model.visual.conv1.padding, dilation=self.model.visual.conv1.dilation, groups=self.model.visual.conv1.groups, bias=False)
- self.model.visual.tiny_vc.weight = tiny_Tensor(self.model.visual.conv1.weight.detach().numpy().copy())
-
- self.model.visual.tiny_class_embedding = tiny_Tensor(self.model.visual.class_embedding.detach().numpy().copy())
- self.model.visual.tiny_positional_embedding = tiny_Tensor(self.model.visual.positional_embedding.detach().numpy().copy())
-
- self.model.visual.tiny_ln_pre = tiny_nn.LayerNorm(self.model.visual.ln_pre.weight.shape[0])
- self.model.visual.tiny_ln_pre.weight = tiny_Tensor(self.model.visual.ln_pre.weight.detach().numpy().copy())
- self.model.visual.tiny_ln_pre.bias = tiny_Tensor(self.model.visual.ln_pre.bias.detach().numpy().copy())
-
- self.model.visual.tiny_ln_post = tiny_nn.LayerNorm(self.model.visual.ln_post.weight.shape[0])
- self.model.visual.tiny_ln_post.weight = tiny_Tensor(self.model.visual.ln_post.weight.detach().numpy().copy())
- self.model.visual.tiny_ln_post.bias = tiny_Tensor(self.model.visual.ln_post.bias.detach().numpy().copy())
-
- self.model.tiny_ln_1 = []
- self.model.tiny_ln_2 = []
- self.model.tiny_in_proj_weight = []
- self.model.tiny_in_proj_bias = []
-
- self.model.tiny_out_proj_weight = []
- self.model.tiny_out_proj_bias = []
-
- self.model.tiny_c_fc = []
- self.model.tiny_c_proj = []
-
- self.model.visual.tiny_proj = tiny_Tensor(self.model.visual.proj.detach().numpy().copy())
-
- for resblock in self.model.visual.transformer.resblocks:
- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_1.normalized_shape, eps=resblock.ln_1.eps, elementwise_affine=True)
- layernorm.weight = tiny_Tensor(resblock.ln_1.weight.detach().numpy().copy())
- layernorm.bias = tiny_Tensor(resblock.ln_1.bias.detach().numpy().copy())
- self.model.tiny_ln_1.append(layernorm)
-
- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_2.normalized_shape, eps=resblock.ln_2.eps, elementwise_affine=True)
- layernorm.weight = tiny_Tensor(resblock.ln_2.weight.detach().numpy().copy())
- layernorm.bias = tiny_Tensor(resblock.ln_2.bias.detach().numpy().copy())
- self.model.tiny_ln_2.append(layernorm)
-
- w = tiny_Tensor(resblock.attn.in_proj_weight.detach().numpy().copy())
- self.model.tiny_in_proj_weight.append(w)
-
- b = tiny_Tensor(resblock.attn.in_proj_bias.detach().numpy().copy())
- self.model.tiny_in_proj_bias.append(b)
-
- w = tiny_Tensor(resblock.attn.out_proj.weight.detach().numpy().copy())
- self.model.tiny_out_proj_weight.append(w)
-
- b = tiny_Tensor(resblock.attn.out_proj.bias.detach().numpy().copy())
- self.model.tiny_out_proj_bias.append(b)
-
- linear = tiny_nn.Linear(resblock.mlp.c_fc.weight.shape[0], resblock.mlp.c_fc.weight.shape[1])
- linear.weight = tiny_Tensor(resblock.mlp.c_fc.weight.detach().numpy().copy())
- linear.bias = tiny_Tensor(resblock.mlp.c_fc.bias.detach().numpy().copy())
- self.model.tiny_c_fc.append(linear)
-
- linear = tiny_nn.Linear(resblock.mlp.c_proj.weight.shape[0], resblock.mlp.c_proj.weight.shape[1])
- linear.weight = tiny_Tensor(resblock.mlp.c_proj.weight.detach().numpy().copy())
- linear.bias = tiny_Tensor(resblock.mlp.c_proj.bias.detach().numpy().copy())
- self.model.tiny_c_proj.append(linear)
-
- with open('laion2b_s32b_b82k.pkl', 'wb') as f: pickle.dump(self.model, f)
+ self.image_embeddings = {}
+ self.image_paths = {}
+
+ self.tiny_model = TinyModel()
+ device = Device.DEFAULT
+ # convert
+ weights = tiny_nn.state.safe_load(fetch("http://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K/resolve/main/open_clip_pytorch_model.safetensors"))
+
+ self.tiny_model.visual_conv1 = tiny_nn.Conv2d(3, 1024, (14, 14), (14, 14), (0, 0), (1, 1), 1, bias=False)
+ self.tiny_model.visual_conv1.weight = weights["visual.conv1.weight"].to(device)
+
+ self.tiny_model.class_embedding = weights["visual.class_embedding"].to(device)
+ self.tiny_model.positional_embedding = weights["visual.positional_embedding"].to(device)
+
+
+ self.tiny_model.ln_pre = tiny_nn.LayerNorm(1024)
+ self.tiny_model.ln_pre.weight = weights["visual.ln_pre.weight"].to(device)
+ self.tiny_model.ln_pre.bias = weights["visual.ln_pre.bias"].to(device)
+
+
+ self.tiny_model.ln_post = tiny_nn.LayerNorm(1024)
+ self.tiny_model.ln_post.weight = weights["visual.ln_post.weight"].to(device)
+ self.tiny_model.ln_post.bias = weights["visual.ln_post.bias"].to(device)
+
+ self.tiny_model.proj = weights["visual.proj"].to(device)
+
+ self.tiny_model.resblocks = []
+
+ for i in range(24):
+ tiny_resblock = TinyModel()
+
+ tiny_resblock.ln_1 = tiny_nn.LayerNorm(1024, 1e-05, elementwise_affine=True)
+ tiny_resblock.ln_1.weight = weights[f"visual.transformer.resblocks.{i}.ln_1.weight"].to(device)
+ tiny_resblock.ln_1.bias = weights[f"visual.transformer.resblocks.{i}.ln_1.bias"].to(device)
+
+ tiny_resblock.ln_2 = tiny_nn.LayerNorm(1024, 1e-05, elementwise_affine=True)
+ tiny_resblock.ln_2.weight = weights[f"visual.transformer.resblocks.{i}.ln_2.weight"].to(device)
+ tiny_resblock.ln_2.bias = weights[f"visual.transformer.resblocks.{i}.ln_2.bias"].to(device)
+
+ tiny_resblock.in_proj_weight = weights[f"visual.transformer.resblocks.{i}.attn.in_proj_weight"].to(device)
+ tiny_resblock.in_proj_bias = weights[f"visual.transformer.resblocks.{i}.attn.in_proj_bias"].to(device)
+
+ tiny_resblock.out_proj_weight = weights[f"visual.transformer.resblocks.{i}.attn.out_proj.weight"].to(device)
+ tiny_resblock.out_proj_bias = weights[f"visual.transformer.resblocks.{i}.attn.out_proj.bias"].to(device)
+
+
+ tiny_resblock.mlp_c_fc = tiny_nn.Linear(4096, 1024)
+ tiny_resblock.mlp_c_fc.weight = weights[f"visual.transformer.resblocks.{i}.mlp.c_fc.weight"].to(device)
+ tiny_resblock.mlp_c_fc.bias = weights[f"visual.transformer.resblocks.{i}.mlp.c_fc.bias"].to(device)
+
+ tiny_resblock.mlp_c_proj = tiny_nn.Linear(1024, 4096)
+ tiny_resblock.mlp_c_proj.weight = weights[f"visual.transformer.resblocks.{i}.mlp.c_proj.weight"].to(device)
+ tiny_resblock.mlp_c_proj.bias = weights[f"visual.transformer.resblocks.{i}.mlp.c_proj.bias"].to(device)
+
+ self.tiny_model.resblocks.append(tiny_resblock)
+
+ weights = None
+ gc.collect()
# for BEAM
- tiny_precompute_embeddings(self.model, tiny_Tensor.rand((1, 3, 224, 224), dtype=dtypes.float32))
- tiny_precompute_embeddings(self.model, tiny_Tensor.rand((16, 3, 224, 224), dtype=dtypes.float32))
+ tiny_precompute_embeddings(self.tiny_model, tiny_Tensor.rand((1, 3, 224, 224), dtype=dtypes.float32))
+ tiny_precompute_embeddings(self.tiny_model, tiny_Tensor.rand((16, 3, 224, 224), dtype=dtypes.float32))
def find_object_folders(self, base_path="data/cameras"):
object_folders = []
@@ -170,19 +148,18 @@ class CachedCLIPSearch:
print(f"Error loading {img_path}: {e}")
continue
- batch_tensors = torch.stack([inline_preprocess(img) for img in batch_images])
+ batch_tensors = np.stack([inline_preprocess(img) for img in batch_images])
if len(batch_images) == batch_size:
- x = tiny_Tensor(batch_tensors.detach().numpy())
- embeddings = tiny_precompute_embeddings(self.model, x)
- embeddings = torch.Tensor(embeddings.numpy())
+ x = tiny_Tensor(batch_tensors)
+ embeddings = tiny_precompute_embeddings(self.tiny_model, x)
+ embeddings = embeddings.numpy()
else:
embeddings = []
for i in range(len(batch_images)): # one by one if not batch_size
- single_x = tiny_Tensor(batch_tensors[i:i+1].detach().numpy())
- single_embedding = tiny_precompute_embedding(self.model, single_x)
- single_embedding_torch = torch.Tensor(single_embedding.numpy())
+ single_x = tiny_Tensor(batch_tensors[i:i+1])
+ single_embedding = tiny_precompute_embedding(self.tiny_model, single_x)
+ single_embedding_torch = single_embedding.numpy()
embeddings.append(single_embedding_torch)
- embeddings = torch.stack(embeddings)
for path, embedding in zip(batch_paths, embeddings):
folder_embeddings[path] = embedding
@@ -200,25 +177,24 @@ class CachedCLIPSearch:
print(f"{datetime.now()}: Updated cache for {folder_path}. Total images stored: {len(folder_embeddings)}")
@TinyJit
-def tiny_precompute_embeddings(model, x): return tiny_precompute_embedding(model, x)
+def tiny_precompute_embeddings(tiny_model, x): return tiny_precompute_embedding(tiny_model, x)
-def tiny_precompute_embedding(model, x):
- visual = model.visual
- x = visual.tiny_vc(x)
+def tiny_precompute_embedding(tiny_model, x):
+ x = tiny_model.visual_conv1(x)
x = x.reshape(x.shape[0], x.shape[1], -1)
x = x.permute(0, 2, 1)
- cls_emb = visual.tiny_class_embedding
+ cls_emb = tiny_model.class_embedding
cls_emb = cls_emb.unsqueeze(0).expand(x.shape[0], -1, -1)
x = tiny_Tensor.cat(cls_emb, x, dim=1)
- x = x + visual.tiny_positional_embedding
- x = visual.tiny_ln_pre(x)
+ x = x + tiny_model.positional_embedding
+ x = tiny_model.ln_pre(x)
# https://github.com/pytorch/pytorch/blob/v2.9.1/torch/nn/modules/activation.py#L1252
- for i, block in enumerate(visual.transformer.resblocks):
- x_ln1 = model.tiny_ln_1[i](x)
+ for i in range(len(tiny_model.resblocks)):
+ x_ln1 = tiny_model.resblocks[i].ln_1(x)
B, L, D = x_ln1.shape
- H = block.attn.num_heads
+ H = 16 #block.attn.num_heads
d_head = D // H
- qkv = x_ln1 @ model.tiny_in_proj_weight[i].T + model.tiny_in_proj_bias[i]
+ qkv = x_ln1 @ tiny_model.resblocks[i].in_proj_weight.T + tiny_model.resblocks[i].in_proj_bias
q, k, v = qkv.split(D, dim=-1)
def shape(x): return x.view(B, L, H, d_head).transpose(1, 2)
q = shape(q)
@@ -229,18 +205,18 @@ def tiny_precompute_embedding(model, x):
attn_probs = tiny_Tensor.softmax(attn_scores)
context = attn_probs.matmul(v)
context = context.transpose(1, 2).contiguous().view(B, L, D)
- attn_out = context @ model.tiny_out_proj_weight[i].T + model.tiny_out_proj_bias[i]
- attn_scaled = block.ls_1(attn_out)
+ attn_out = context @ tiny_model.resblocks[i].out_proj_weight.T + tiny_model.resblocks[i].out_proj_bias
+ attn_scaled = attn_out
x = x + attn_scaled
- x_ln2 = model.tiny_ln_2[i](x)
- ff = model.tiny_c_fc[i](x_ln2)
+ x_ln2 = tiny_model.resblocks[i].ln_2(x)
+ ff = tiny_model.resblocks[i].mlp_c_fc(x_ln2)
ff = ff.gelu()
- ff = model.tiny_c_proj[i](ff)
+ ff = tiny_model.resblocks[i].mlp_c_proj(ff)
x = x + ff
- x = visual.tiny_ln_post(x)
+ x = tiny_model.ln_post(x)
image_embeds = x[:, 0, :]
- embeddings = image_embeds @ visual.tiny_proj
+ embeddings = image_embeds @ tiny_model.proj
embeddings = embeddings / (embeddings.pow(2).sum(axis=-1, keepdim=True).sqrt() + 1e-8)
return embeddings
@@ -270,4 +246,4 @@ if __name__ == "__main__":
print(f"{datetime.now()}: Sleeping for 15 mins...")
time.sleep(900)
-'''
\ No newline at end of file
+'''
diff --git a/tiny_clip_search.py b/tiny_clip_search.py
index 4011361..dcc2e0e 100644
--- a/tiny_clip_search.py
+++ b/tiny_clip_search.py
@@ -1,63 +1,80 @@
import os
import pickle
-import torch
-import open_clip
-from typing import Optional
-from torch.nn import functional as F
-import torch.nn as nn
-from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit
+from tinygrad import nn as tiny_nn, Tensor as tiny_Tensor, TinyJit, Device
+from tinygrad.helpers import fetch
+from utils.clip_tokenizer import SimpleTokenizer
+import numpy as np
+import gc
+
+class TinyModel:
+ pass
class CLIPSearch:
def __init__(self, base_path="data/cameras"):
self.base_path = base_path
self.image_embeddings = {}
self.image_paths = {}
- self.device = torch.device("cpu")
- self.model, _, self.preprocess = open_clip.create_model_and_transforms(
- 'ViT-L-14',
- pretrained='laion2b_s32b_b82k'
- )
- self.model = self.model.to(self.device).eval()
-
- # convert here
- # .copy() or trouble
- self.model.tiny_text_projection = tiny_Tensor(self.model.text_projection.detach().numpy().copy())
- self.model.tiny_token_embedding = tiny_nn.Embedding(self.model.token_embedding.num_embeddings, self.model.token_embedding.embedding_dim)
- self.model.tiny_token_embedding.weight = tiny_Tensor(self.model.token_embedding.weight.data.detach().numpy().copy())
+ self.tokenizer = SimpleTokenizer()
+
+ device = Device.DEFAULT
+
+
+ self.model = TinyModel()
+
+ weights = tiny_nn.state.safe_load(fetch("http://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K/resolve/main/open_clip_pytorch_model.safetensors"))
+
+ self.model.text_projection = weights["text_projection"].to(device)
+ self.model.positional_embedding = weights["positional_embedding"].to(device)
+
+ self.model.token_embedding = tiny_nn.Embedding(49408, 768) # todo unhardcode
+ self.model.token_embedding.weight = weights["token_embedding.weight"].to(device)
+
+ self.model.ln_final = tiny_nn.LayerNorm(768, eps=1e-5, elementwise_affine=True)
+ self.model.ln_final.weight = weights["ln_final.weight"].to(device)
+ self.model.ln_final.bias = weights["ln_final.bias"].to(device)
+
+ attn_mask = np.where(np.tri(77, dtype=bool), 0.0, -np.inf).astype(np.float32)
+ self.model.attn_mask = tiny_Tensor(attn_mask)
+
+ self.model.resblocks = []
- self.model.tiny_ln_1 = []
- self.model.tiny_ln_2 = []
- self.model.out_proj_bias_tiny = []
- self.model.mlp_c_fc = []
- self.model.mlp_c_proj = []
- for resblock in self.model.transformer.resblocks:
- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_1.normalized_shape, eps=resblock.ln_1.eps, elementwise_affine=True)
- layernorm.weight = tiny_Tensor(resblock.ln_1.weight.detach().numpy().copy())
- layernorm.bias = tiny_Tensor(resblock.ln_1.bias.detach().numpy().copy())
- self.model.tiny_ln_1.append(layernorm)
-
- layernorm = tiny_nn.LayerNorm(normalized_shape=resblock.ln_2.normalized_shape, eps=resblock.ln_2.eps, elementwise_affine=True)
- layernorm.weight = tiny_Tensor(resblock.ln_2.weight.detach().numpy().copy())
- layernorm.bias = tiny_Tensor(resblock.ln_2.bias.detach().numpy().copy())
- self.model.tiny_ln_2.append(layernorm)
-
- opb = tiny_Tensor(resblock.attn.out_proj.bias.detach().numpy().copy())
- self.model.out_proj_bias_tiny.append(opb)
-
- # todo
- mlpcfc = tiny_nn.Linear(resblock.mlp.c_fc.weight.shape[0], resblock.mlp.c_fc.weight.shape[1])
- mlpcfc.weight = tiny_Tensor(resblock.mlp.c_fc.weight.detach().numpy().copy())
- mlpcfc.bias = tiny_Tensor(resblock.mlp.c_fc.bias.detach().numpy().copy())
- self.model.mlp_c_fc.append(mlpcfc)
-
- mlpcp = tiny_nn.Linear(resblock.mlp.c_proj.weight.shape[0], resblock.mlp.c_proj.weight.shape[1])
- mlpcp.weight = tiny_Tensor(resblock.mlp.c_proj.weight.detach().numpy().copy())
- mlpcp.bias = tiny_Tensor(resblock.mlp.c_proj.bias.detach().numpy().copy())
- self.model.mlp_c_proj.append(mlpcp)
-
- self.model.tiny_ln_final = tiny_nn.LayerNorm(normalized_shape=self.model.ln_final.normalized_shape, eps=self.model.ln_final.eps, elementwise_affine=True)
- self.model.tiny_ln_final .weight = tiny_Tensor(self.model.ln_final.weight.detach().numpy().copy())
- self.model.tiny_ln_final .bias = tiny_Tensor(self.model.ln_final.bias.detach().numpy().copy())
+ for i in range(12):
+ resblock = TinyModel()
+ layernorm = tiny_nn.LayerNorm(768, 1e-5, elementwise_affine=True)
+ layernorm.weight = weights[f"transformer.resblocks.{i}.ln_1.weight"].to(device)
+ layernorm.bias = weights[f"transformer.resblocks.{i}.ln_1.bias"].to(device)
+ resblock.ln_1 = layernorm
+
+ layernorm = tiny_nn.LayerNorm(768, 1e-5, elementwise_affine=True)
+ layernorm.weight = weights[f"transformer.resblocks.{i}.ln_2.weight"].to(device)
+ layernorm.bias = weights[f"transformer.resblocks.{i}.ln_2.bias"].to(device)
+ resblock.ln_2 = layernorm
+
+ weight = weights[f"transformer.resblocks.{i}.attn.out_proj.weight"].to(device)
+ resblock.attn_out_proj_weight = weight
+
+ bias = weights[f"transformer.resblocks.{i}.attn.out_proj.bias"].to(device)
+ resblock.attn_out_proj_bias = bias
+
+ mlpcfc = tiny_nn.Linear(3072, 768)
+ mlpcfc.weight = weights[f"transformer.resblocks.{i}.mlp.c_fc.weight"].to(device)
+ mlpcfc.bias = weights[f"transformer.resblocks.{i}.mlp.c_fc.bias"].to(device)
+ resblock.mlp_c_fc = mlpcfc
+
+ mlpcp = tiny_nn.Linear(768, 3072)
+ mlpcp.weight = weights[f"transformer.resblocks.{i}.mlp.c_proj.weight"].to(device)
+ mlpcp.bias = weights[f"transformer.resblocks.{i}.mlp.c_proj.bias"].to(device)
+ resblock.mlp_c_proj = mlpcp
+
+ weight = weights[f"transformer.resblocks.{i}.attn.in_proj_weight"].to(device)
+ resblock.in_proj_weight = weight
+
+ bias = weights[f"transformer.resblocks.{i}.attn.in_proj_bias"].to(device)
+ resblock.in_proj_bias = bias
+
+ self.model.resblocks.append(resblock)
+
+ weights = None
def _load_single_embeddings_file(self, cache_file):
try:
@@ -107,8 +124,11 @@ class CLIPSearch:
print(f"\nTotal images loaded: {total_loaded}")
def _encode_text(self, query):
- tokens = open_clip.tokenize([query]).to(self.device)
- tokens = tiny_Tensor(tokens.detach().numpy())
+ tokens = [49406]
+ tokens += self.tokenizer.encode(query)
+ tokens.append(49407)
+ if len(tokens) < 77: tokens += [0] * (77 - len(tokens))
+ tokens = tiny_Tensor([tokens])
text_emb = encode_text(self.model, tokens)
return text_emb
@@ -117,7 +137,7 @@ class CLIPSearch:
print("No embeddings available.")
return []
text_embedding = self._encode_text(query)
- text_embedding = torch.Tensor(text_embedding.numpy())
+ text_embedding = text_embedding.numpy()
all_similarities = []
for path, img_embedding in self.image_embeddings.items():
normalized_path = path.replace("\\", "/")
@@ -148,25 +168,18 @@ class CLIPSearch:
@TinyJit
def encode_text(model, text):
x = text
- x = model.tiny_token_embedding(x)
- if not hasattr(model, 'tiny_positional_embedding'): model.tiny_positional_embedding = tiny_Tensor(model.positional_embedding.detach().numpy())
- x = x + model.tiny_positional_embedding
+ x = model.token_embedding(x)
+ x = x + model.positional_embedding
- for i, resblock in enumerate(model.transformer.resblocks):
+ for i in range(len(model.resblocks)):
residual = x
- x = model.tiny_ln_1[i](x)
+ x = model.resblocks[i].ln_1(x)
B, L, D = x.shape
- H = resblock.attn.num_heads
+ H = 12 #resblock.attn.num_heads
d_head = D // H
- in_proj_weight_tiny = tiny_Tensor(resblock.attn.in_proj_weight.detach().numpy()) # todo store these
- in_proj_bias_tiny = tiny_Tensor(resblock.attn.in_proj_bias.detach().numpy())
- out_proj_weight_tiny = tiny_Tensor(resblock.attn.out_proj.weight.detach().numpy())
- attn_mask = tiny_Tensor(model.attn_mask.detach().numpy())
-
-
- qkv = x.matmul(in_proj_weight_tiny.T) + in_proj_bias_tiny
+ qkv = x.matmul(model.resblocks[i].in_proj_weight.T) + model.resblocks[i].in_proj_bias
q, k, v = qkv.split(D, dim=-1)
q = q.view(B, L, H, d_head).transpose(1, 2)
k = k.view(B, L, H, d_head).transpose(1, 2)
@@ -174,25 +187,25 @@ def encode_text(model, text):
scale = 1.0 / (d_head ** 0.5)
attn_scores = q.matmul(k.transpose(-2, -1)) * scale # (B,H,L,L)
- bool_mask = attn_mask < 0
+ bool_mask = model.attn_mask < 0
attn_scores = attn_scores.masked_fill(bool_mask, float("-inf"))
attn_probs = tiny_Tensor.softmax(attn_scores)
context = attn_probs.matmul(v)
context = context.transpose(1, 2).contiguous().view(B, L, D)
- x = context.matmul(out_proj_weight_tiny.T) + model.out_proj_bias_tiny[i]
+ x = context.matmul(model.resblocks[i].attn_out_proj_weight.T) + model.resblocks[i].attn_out_proj_bias
x += residual
residual = x
- x = model.tiny_ln_2[i](x)
- x = model.mlp_c_fc[i](x)
+ x = model.resblocks[i].ln_2(x)
+ x = model.resblocks[i].mlp_c_fc(x)
x = x.gelu()
- x = model.mlp_c_proj[i](x)
+ x = model.resblocks[i].mlp_c_proj(x)
x += residual
- x = model.tiny_ln_final(x) # [batch_size, n_ctx, transformer.width]
+ x = model.ln_final(x) # [batch_size, n_ctx, transformer.width]
argmax = text.argmax()
x = x[0][argmax]
- x = x @ model.tiny_text_projection
+ x = x @ model.text_projection
return x / (x * x).sum(axis=-1, keepdim=True).sqrt()
'''
@@ -205,4 +218,3 @@ if __name__ == "__main__":
for i, (path, score) in enumerate(results, 1):
print(f"{i}. Score: {score:.3f} - \"{path}\"")
'''
-
diff --git a/utils/__pycache__/clip_tokenizer.cpython-311.pyc b/utils/__pycache__/clip_tokenizer.cpython-311.pyc
new file mode 100644
index 0000000..85203b6
Binary files /dev/null and b/utils/__pycache__/clip_tokenizer.cpython-311.pyc differ
diff --git a/utils/bpe_simple_vocab_16e6.txt.gz b/utils/bpe_simple_vocab_16e6.txt.gz
new file mode 100644
index 0000000..7b5088a
Binary files /dev/null and b/utils/bpe_simple_vocab_16e6.txt.gz differ
diff --git a/utils/clip_tokenizer.py b/utils/clip_tokenizer.py
new file mode 100644
index 0000000..c83133d
--- /dev/null
+++ b/utils/clip_tokenizer.py
@@ -0,0 +1,621 @@
+""" CLIP tokenizer
+
+Copied from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI.
+"""
+#https://github.com/mlfoundations/open_clip/blob/d3cdb734a2710feeb4c6307df037afa5f786a3e1/src/open_clip/tokenizer.py
+import gzip
+import html
+import os
+import random
+import string
+from functools import lru_cache, partial
+from typing import Callable, List, Optional, Union, Dict
+import warnings
+
+import ftfy
+import numpy as np
+import regex as re
+import torch
+
+# https://stackoverflow.com/q/62691279
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+_nltk_init = False
+
+DEFAULT_CONTEXT_LENGTH = 77 # default context length for OpenAI CLIP
+
+
+@lru_cache()
+def default_bpe():
+ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+@lru_cache()
+def bytes_to_unicode():
+ """
+ Returns list of utf-8 byte and a corresponding list of unicode strings.
+ The reversible bpe codes work on unicode strings.
+ This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+ When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+ This is a significant percentage of your normal, say, 32K bpe vocab.
+ To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+ And avoids mapping to whitespace/control characters the bpe code barfs on.
+ """
+ bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+ cs = bs[:]
+ n = 0
+ for b in range(2**8):
+ if b not in bs:
+ bs.append(b)
+ cs.append(2**8+n)
+ n += 1
+ cs = [chr(n) for n in cs]
+ return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+ """Return set of symbol pairs in a word.
+ Word is represented as tuple of symbols (symbols being variable-length strings).
+ """
+ pairs = set()
+ prev_char = word[0]
+ for char in word[1:]:
+ pairs.add((prev_char, char))
+ prev_char = char
+ return pairs
+
+
+def basic_clean(text):
+ text = ftfy.fix_text(text)
+ text = html.unescape(html.unescape(text))
+ return text.strip()
+
+
+def whitespace_clean(text):
+ text = " ".join(text.split())
+ text = text.strip()
+ return text
+
+
+def _clean_canonicalize(x):
+ # basic, remove whitespace, remove punctuation, lower case
+ return canonicalize_text(basic_clean(x))
+
+
+def _clean_lower(x):
+ # basic, remove whitespace, lower case
+ return whitespace_clean(basic_clean(x)).lower()
+
+
+def _clean_whitespace(x):
+ # basic, remove whitespace
+ return whitespace_clean(basic_clean(x))
+
+
+def get_clean_fn(type: str):
+ if type == 'canonicalize':
+ return _clean_canonicalize
+ elif type == 'lower':
+ return _clean_lower
+ elif type == 'whitespace':
+ return _clean_whitespace
+ else:
+ assert False, f"Invalid clean function ({type})."
+
+
+def canonicalize_text(
+ text,
+ *,
+ keep_punctuation_exact_string=None,
+ trans_punctuation: dict = str.maketrans("", "", string.punctuation),
+):
+ """Returns canonicalized `text` (lowercase and punctuation removed).
+
+ From: https://github.com/google-research/big_vision/blob/53f18caf27a9419231bbf08d3388b07671616d3d/big_vision/evaluators/proj/image_text/prompt_engineering.py#L94
+
+ Args:
+ text: string to be canonicalized.
+ keep_punctuation_exact_string: If provided, then this exact string kept.
+ For example providing '{}' will keep any occurrences of '{}' (but will
+ still remove '{' and '}' that appear separately).
+ """
+ text = text.replace("_", " ")
+ if keep_punctuation_exact_string:
+ text = keep_punctuation_exact_string.join(
+ part.translate(trans_punctuation)
+ for part in text.split(keep_punctuation_exact_string)
+ )
+ else:
+ text = text.translate(trans_punctuation)
+ text = text.lower()
+ text = " ".join(text.split())
+ return text.strip()
+
+
+class SimpleTokenizer(object):
+ def __init__(
+ self,
+ bpe_path: str = default_bpe(),
+ additional_special_tokens: Optional[List[str]] = None,
+ context_length: Optional[int] = DEFAULT_CONTEXT_LENGTH,
+ clean: str = 'lower',
+ reduction_mask: str = ''
+ ):
+ self.byte_encoder = bytes_to_unicode()
+ self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+ merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
+ merges = merges[1:49152-256-2+1]
+ merges = [tuple(merge.split()) for merge in merges]
+ vocab = list(bytes_to_unicode().values())
+ vocab = vocab + [v+'</w>' for v in vocab]
+ for merge in merges:
+ vocab.append(''.join(merge))
+ special_tokens = ['<start_of_text>', '<end_of_text>']
+ if additional_special_tokens:
+ special_tokens += additional_special_tokens
+ vocab.extend(special_tokens)
+ self.encoder = dict(zip(vocab, range(len(vocab))))
+ self.decoder = {v: k for k, v in self.encoder.items()}
+ self.bpe_ranks = dict(zip(merges, range(len(merges))))
+ self.cache = {t:t for t in special_tokens}
+ special = "|".join(special_tokens)
+ self.pat = re.compile(
+ special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
+ re.IGNORECASE,
+ )
+ self.vocab_size = len(self.encoder)
+ self.all_special_ids = [self.encoder[t] for t in special_tokens]
+ self.sot_token_id = self.all_special_ids[0]
+ self.eot_token_id = self.all_special_ids[1]
+ self.context_length = context_length
+ self.clean_fn = get_clean_fn(clean)
+ self.reduction_fn = get_reduction_mask_fn(reduction_mask) if reduction_mask else None
+
+ def bpe(self, token):
+ if token in self.cache:
+ return self.cache[token]
+ word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+ pairs = get_pairs(word)
+
+ if not pairs:
+ return token+'</w>'
+
+ while True:
+ bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+ if bigram not in self.bpe_ranks:
+ break
+ first, second = bigram
+ new_word = []
+ i = 0
+ while i < len(word):
+ try:
+ j = word.index(first, i)
+ new_word.extend(word[i:j])
+ i = j
+ except Exception:
+ new_word.extend(word[i:])
+ break
+
+ if word[i] == first and i < len(word)-1 and word[i+1] == second:
+ new_word.append(first+second)
+ i += 2
+ else:
+ new_word.append(word[i])
+ i += 1
+ new_word = tuple(new_word)
+ word = new_word
+ if len(word) == 1:
+ break
+ else:
+ pairs = get_pairs(word)
+ word = ' '.join(word)
+ self.cache[token] = word
+ return word
+
+ def encode(self, text):
+ bpe_tokens = []
+ text = self.clean_fn(text)
+ for token in re.findall(self.pat, text):
+ token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+ bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+ return bpe_tokens
+
+ def decode(self, tokens):
+ text = ''.join([self.decoder[token] for token in tokens])
+ text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+ return text
+
+ def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.LongTensor:
+ """ Returns the tokenized representation of given input string(s)
+
+ Parameters
+ ----------
+ texts : Union[str, List[str]]
+ An input string or a list of input strings to tokenize
+ context_length : int
+ The context length to use; all CLIP models use 77 as the context length
+
+ Returns
+ -------
+ A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
+ """
+ if isinstance(texts, str):
+ texts = [texts]
+
+ context_length = context_length or self.context_length
+ assert context_length, 'Please set a valid context length'
+
+ if self.reduction_fn is not None:
+ # use reduction strategy for tokenize if set, otherwise default to truncation below
+ return self.reduction_fn(
+ texts,
+ context_length=context_length,
+ sot_token_id=self.sot_token_id,
+ eot_token_id=self.eot_token_id,
+ encode_fn=self.encode,
+ )
+
+ all_tokens = [[self.sot_token_id] + self.encode(text) + [self.eot_token_id] for text in texts]
+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+ for i, tokens in enumerate(all_tokens):
+ if len(tokens) > context_length:
+ tokens = tokens[:context_length] # Truncate
+ tokens[-1] = self.eot_token_id
+ result[i, :len(tokens)] = torch.tensor(tokens)
+
+ return result
+
+
+_tokenizer = SimpleTokenizer()
+
+
+def decode(output_ids: torch.Tensor):
+ output_ids = output_ids.cpu().numpy()
+ return _tokenizer.decode(output_ids)
+
+
+def tokenize(texts: Union[str, List[str]], context_length: int = DEFAULT_CONTEXT_LENGTH) -> torch.LongTensor:
+ return _tokenizer(texts, context_length=context_length)
+
+
+def random_mask_tokenize(
+ texts: Union[str, List[str]],
+ context_length: int,
+ sot_token_id: int,
+ eot_token_id: int,
+ encode_fn: Callable,
+ shuffle: bool = False,
+):
+ all_tokens = [encode_fn(text) for text in texts]
+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+ for i, tokens in enumerate(all_tokens):
+ tokens = torch.tensor(tokens)
+ num_tokens = len(tokens)
+ if num_tokens > context_length - 2: # 2 for sot and eot token
+ num_keep = context_length - 2
+ indices = torch.randperm(len(tokens))
+ indices = indices[:num_keep]
+ if not shuffle:
+ indices = indices.msort()
+ tokens = tokens[indices]
+ num_tokens = num_keep
+ result[i, 0] = sot_token_id
+ result[i, 1:num_tokens + 1] = tokens
+ result[i, num_tokens + 1] = eot_token_id
+
+ return result
+
+
+def simple_mask_tokenize(
+ texts: Union[str, List[str]],
+ context_length: int,
+ sot_token_id: int,
+ eot_token_id: int,
+ encode_fn: Callable,
+):
+ all_tokens = [encode_fn(text) for text in texts]
+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+ for i, tokens in enumerate(all_tokens):
+ num_tokens = len(tokens)
+ if num_tokens > context_length - 2: # 2 for sot and eot token
+ num_keep = context_length - 2
+ start_index = random.randint(0, num_tokens - num_keep) # high is incl
+ tokens = tokens[start_index: start_index + num_keep]
+ tokens = [sot_token_id] + tokens + [eot_token_id]
+ result[i, :len(tokens)] = torch.tensor(tokens)
+
+ return result
+
+
+def syntax_mask_tokenize(
+ texts: Union[str, List[str]],
+ context_length: int,
+ sot_token_id: int,
+ eot_token_id: int,
+ encode_fn: Callable,
+) -> torch.LongTensor:
+ """ Returns the tokenized representation of given input string(s).
+ Apply syntax masking before tokenize.
+ """
+ import nltk
+ global _nltk_init
+ if not _nltk_init:
+ # run them for the first time
+ nltk.download('punkt')
+ nltk.download('averaged_perceptron_tagger')
+ _nltk_init = True
+
+ def get_order(x):
+ if x.startswith('NN'):
+ return 1
+ elif x.startswith('JJ'):
+ return 2
+ elif x.startswith('VB'):
+ return 3
+ else:
+ return 4
+
+ # syntax masking
+ new_texts = []
+ for text in texts:
+ list_tokens = nltk.tokenize.word_tokenize(text)
+ pos_tags = nltk.pos_tag(list_tokens)
+ # sample the words by get_order method
+ order_list = [get_order(tag) for _, tag in pos_tags]
+ sorted_ids = np.argsort(np.array(order_list))
+ sampled_ids = sorted(sorted_ids[:context_length - 2]) # need 2 slots for sot and eot tokens
+ sampled_tokens = np.take(np.array(list_tokens), sampled_ids, axis=0) # sample the tokens
+
+ new_text = ''
+ for token in sampled_tokens:
+ new_text = new_text + str(token) + ' '
+ new_text = new_text.strip()
+ new_texts.append(new_text)
+ texts = new_texts
+
+ all_tokens = [[sot_token_id] + encode_fn(text) + [eot_token_id] for text in texts]
+ result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+ for i, tokens in enumerate(all_tokens):
+ # still need first truncate because some words produces two tokens
+ if len(tokens) > context_length:
+ tokens = tokens[:context_length] # Truncate
+ tokens[-1] = eot_token_id
+ result[i, :len(tokens)] = torch.tensor(tokens)
+
+ return result
+
+
+def get_reduction_mask_fn(type: str):
+ """ Choose strategy for dropping (masking) tokens to achieve target context length"""
+ assert type in ('simple', 'random', 'shuffle', 'syntax')
+ if type == 'simple':
+ return simple_mask_tokenize # randomly select block [start:end]
+ elif type == 'random':
+ return random_mask_tokenize # randomly drop tokens (keep order)
+ elif type == 'shuffle':
+ return partial(random_mask_tokenize, shuffle=True) # randomly drop tokens (shuffle order)
+ elif type == 'syntax':
+ return syntax_mask_tokenize # randomly drop prioritized by syntax
+ else:
+ assert False, F'Unknown type {type}.'
+
+
+class HFTokenizer:
+ """HuggingFace tokenizer wrapper with support for custom tokenization modes"""
+
+ def __init__(
+ self,
+ tokenizer_name: str,
+ context_length: Optional[int] = DEFAULT_CONTEXT_LENGTH,
+ clean: str = 'whitespace',
+ strip_sep_token: bool = False,
+ language: Optional[str] = None,
+ cache_dir: Optional[str] = None,
+ tokenizer_mode: Optional[str] = None, # None, 'clips'
+ **kwargs
+ ):
+ self.tokenizer_mode = tokenizer_mode or ''
+ self.context_length = context_length
+ self.clean_fn = get_clean_fn(clean)
+ self.strip_sep_token = strip_sep_token
+
+ # NOTE: Left as example of loading custom tokenizer from file for experimentation
+ # if self.tokenizer_mode == 'bert_clips':
+ # self.special_tokens = {
+ # "bos_token": 1,
+ # "eos_token": 2,
+ # "cls_token": 101,
+ # "pad_token": 0
+ # }
+ #
+ # # For BERT CLIPS mode with vocab file
+ # from tokenizers import BertWordPieceTokenizer
+ # if tokenizer_name.startswith('hf-hub:'):
+ # from huggingface_hub import hf_hub_download
+ # # Format: hf-hub:repo_id/filename
+ # repo_url = tokenizer_name[7:]
+ # parts = repo_url.split('/')
+ # filename = parts[-1]
+ # repo_id = '/'.join(parts[:-1])
+ # vocab_file = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=cache_dir)
+ # self.tokenizer = BertWordPieceTokenizer(lowercase=True)
+ # self.tokenizer = self.tokenizer.from_file(vocab_file)
+ # else:
+ # # Assume tokenizer_name is a local path to a vocab file
+ # self.tokenizer = BertWordPieceTokenizer(lowercase=True)
+ # self.tokenizer = self.tokenizer.from_file(tokenizer_name)
+
+ # Standard HuggingFace tokenizer initialization
+ from transformers import AutoTokenizer
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ tokenizer_name,
+ cache_dir=cache_dir,
+ **kwargs
+ )
+
+ # Set language function if available
+ set_lang_fn = getattr(self.tokenizer, 'set_src_lang_special_tokens', None)
+ if callable(set_lang_fn):
+ self.set_lang_fn = set_lang_fn
+ if language is not None:
+ self.set_language(language)
+
+ def save_pretrained(self, dest):
+ self.tokenizer.save_pretrained(dest)
+
+ def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.Tensor:
+ # same cleaning as for default tokenizer, except lowercasing
+ # adding lower (for case-sensitive tokenizers) will make it more robust but less sensitive to nuance
+ if isinstance(texts, str):
+ texts = [texts]
+
+ context_length = context_length or self.context_length
+ assert context_length, 'Please set a valid context length in class init or call.'
+
+ texts = [self.clean_fn(text) for text in texts]
+
+ # Handle different tokenization modes
+ if self.tokenizer_mode == 'clips':
+ return self._clips_tokenize(texts, context_length)
+ else:
+ # Standard tokenization
+ input_ids = self.tokenizer.batch_encode_plus(
+ texts,
+ return_tensors='pt',
+ max_length=context_length,
+ padding='max_length',
+ truncation=True,
+ ).input_ids
+
+ if self.strip_sep_token:
+ input_ids = torch.where(
+ input_ids == self.tokenizer.sep_token_id,
+ torch.zeros_like(input_ids),
+ input_ids,
+ )
+
+ return input_ids
+
+ def set_language(self, src_lang):
+ if hasattr(self, 'set_lang_fn'):
+ self.set_lang_fn(src_lang)
+ else:
+ warnings.warn('Cannot set language for the tokenizer.')
+
+ def _clips_tokenize(self, texts: List[str], context_length: int) -> torch.Tensor:
+ """Use standard HF tokenizer but apply custom post-processing"""
+ # Use standard tokenizer without special tokens - we'll add our own
+ encoded_outputs = self.tokenizer.batch_encode_plus(
+ texts,
+ add_special_tokens=False,
+ padding=False,
+ truncation=False,
+ return_tensors=None
+ )
+
+ encoded = []
+ for tokens in encoded_outputs["input_ids"]:
+ tokens = tokens[:context_length - 3] # Leave room for special tokens
+ tokens = [self.tokenizer.bos_token_id] + tokens + [self.tokenizer.eos_token_id]
+ encoded.append(tokens)
+
+ # Create result tensor and handle padding + class token
+ result = torch.zeros(len(encoded), context_length, dtype=torch.long)
+ for i, tokens in enumerate(encoded):
+ padded_tokens = self._pad_and_add_class_token(
+ tokens,
+ max_length=context_length,
+ pad_token_id=self.tokenizer.pad_token_id,
+ cls_token_id=self.tokenizer.cls_token_id,
+ )
+ result[i, :len(padded_tokens)] = torch.tensor(padded_tokens)
+
+ return result
+
+ def _pad_and_add_class_token(
+ self,
+ tokens: List[int],
+ max_length: int,
+ pad_token_id: int = 0,
+ cls_token_id: int = 101,
+ ) -> List[int]:
+ """ Add padding with class token at the end """
+ if len(tokens) > max_length - 1:
+ tokens = tokens[:max_length - 1]
+
+ # Add padding to reach max_length-1
+ if len(tokens) < max_length - 1:
+ tokens = tokens + [pad_token_id] * (max_length - 1 - len(tokens))
+
+ # Add class token at the end
+ tokens = tokens + [cls_token_id]
+ return tokens
+
+
+class SigLipTokenizer:
+ """HuggingFace tokenizer wrapper for SigLIP T5 compatible sentencepiece vocabs
+
+ NOTE: this is not needed in normal library use, but is used to import new sentencepiece tokenizers
+ into OpenCLIP. Leaving code here in case future models use new tokenizers.
+ """
+ VOCAB_FILES = {
+ # english, vocab_size=32_000
+ "c4-en": "http://storage.googleapis.com/t5-data/vocabs/cc_en.32000/sentencepiece.model",
+ # used in multilingual models (mT5, PaLI), vocab_size=250_000
+ "mc4": "http://storage.googleapis.com/t5-data/vocabs/mc4.250000.100extra/sentencepiece.model",
+ # used in SigLIP2 models, vocab_size=256000
+ "gemma": "http://storage.googleapis.com/big_vision/gemma_tokenizer.model",
+ }
+
+ def __init__(
+ self,
+ tokenizer_name: str,
+ context_length: Optional[int] = 64,
+ ):
+ if 'gemma' in tokenizer_name:
+ from transformers import GemmaTokenizerFast
+ tokenizer_cls = partial(
+ GemmaTokenizerFast, padding_side='right', add_bos_token=False, add_eos_token=True)
+ else:
+ from transformers import T5TokenizerFast
+ tokenizer_cls = partial(T5TokenizerFast, extra_ids=0)
+
+ if tokenizer_name in self.VOCAB_FILES:
+ # FIXME temporary hack?
+ import tempfile
+ import fsspec
+ vocab_file = self.VOCAB_FILES[tokenizer_name]
+ with tempfile.NamedTemporaryFile('wb') as dst:
+ with fsspec.open(vocab_file, 'rb') as src:
+ dst.write(src.read())
+ self.tokenizer = tokenizer_cls(dst.name, legacy=False)
+ else:
+ self.tokenizer = tokenizer_cls(tokenizer_name, legacy=False)
+
+ self.tokenizer.pad_token_id = 0 if 'gemma' in tokenizer_name else 1
+ self.tokenizer.eos_token_id = 1
+ self.context_length = context_length
+
+ def save_pretrained(self, dest):
+ self.tokenizer.save_pretrained(dest)
+
+ def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.Tensor:
+ # same cleaning as for default tokenizer, except lowercasing
+ # adding lower (for case-sensitive tokenizers) will make it more robust but less sensitive to nuance
+ if isinstance(texts, str):
+ texts = [texts]
+
+ context_length = context_length or self.context_length
+ assert context_length, 'Please set a valid context length in class init or call.'
+
+ texts = [canonicalize_text(basic_clean(text)) for text in texts]
+ output = self.tokenizer(
+ texts,
+ return_tensors='pt',
+ max_length=context_length,
+ padding='max_length',
+ truncation=True,
+ )
+ return output.input_ids
\ No newline at end of file