Add object detection, tracking, mobile notifications, and search to any security camera.
clean
roryclear9c7352ed0f7e
diff --git a/llm/qwen3vl.py b/llm/qwen3vl.pyindex 1195676..b970188 100644--- a/llm/qwen3vl.py+++ b/llm/qwen3vl.py@@ -198,7 +198,9 @@ def get_vision_position_ids(h: int, w:int, merge_size: int):return pos_idsclass Qwen3VLVis():- def __init__(self, tok:SimpleTokenizer, size="2B", res=[640, 640]):+ def __init__(self, tok:SimpleTokenizer, size="2B", res:list=[640, 640]):+ assert len(res) == 2, f"Invalid qwen resolution: {res}"+ res = [math.ceil(x / 32) * 32 for x in res] # make divisible by 32self.res = resself.toks_per_img = (self.res[0] * self.res[1]) // (32*32) # 32x32 tokens per pixel https://www.alibabacloud.com/help/en/model-studio/visionkv, state_dict = gguf_load(fetch(f"https://huggingface.co/Qwen/Qwen3-VL-{size}-Instruct-GGUF/resolve/main/mmproj-Qwen3VL-{size}-Instruct-F16.gguf"))@@ -218,7 +220,7 @@ class Qwen3VLVis():self.suffix = Tensor(tok.encode("<|vision_end|>\n<|im_end|>\n"))# https://github.com/huggingface/transformers/blob/15bb519bd4277f4ab5309154aedf3c231e8b4ca8/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L679- def forward(self, pixel_values, image_grid_size):+ def forward(self, pixel_values:Tensor, image_grid_size:list):grid_hs, grid_ws = image_grid_sizeidx_tensor, weight_tensor = get_vision_bilinear_indices_and_weights(h=grid_hs, w=grid_ws, num_grid_per_side=self.v.num_grid_per_side, merge_size=self.merge_size)pos_ids = get_vision_position_ids(h=grid_hs, w=grid_ws, merge_size=self.merge_size)@@ -239,7 +241,7 @@ class Qwen3VLVis():deepstack_feature_lists = []for i in range(len(self.v.blk)):- hidden_states = self.v.blk[i](hidden_states, cos, sin)+ hidden_states = self.v.blk[i](hidden_states=hidden_states, position_embeddings=(cos, sin))if i in self.v.deepstack_idx: deepstack_feature_lists.append(self.v.deepstack[i](hidden_states))image_embeds = self.v.post_ln(hidden_states)@@ -249,7 +251,7 @@ class Qwen3VLVis():image_embeds = self.mm[2](image_embeds)return image_embeds, hidden_states, deepstack_feature_lists- def __call__(self, lang, image, start_pos):+ def __call__(self, lang:Transformer, image:Tensor|bytes, start_pos:int):if type(image) == bytes: image = cv2.cvtColor(cv2.imdecode(np.frombuffer(image, np.uint8), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)if image.shape[:2] != self.res:target_h, target_w = self.res[:2]@@ -259,7 +261,7 @@ class Qwen3VLVis():self.prefill(lang=lang, image=Tensor(image), start_pos=start_pos)@TinyJit- def prefill(self, lang, image, start_pos):+ def prefill(self, lang:Transformer, image, start_pos):image = image.permute(2, 0, 1)image = image.unsqueeze(0).float()image = ((image / 255) - Tensor(self.image_mean).view(1, 3, 1, 1)) / Tensor(self.image_std).view(1, 3, 1, 1)@@ -300,13 +302,13 @@ class Qwen3VLVis():hidden_states.realize()class Qwen3PatchEmbed:- def __init__(self, kv=None, weights=None):+ def __init__(self, kv:dict, weights:dict):self.weight = Tensor.zeros(weights["v.patch_embd.weight"].shape)self.weight1 = Tensor.zeros(weights["v.patch_embd.weight.1"].shape)self.bias = Tensor.zeros(kv["clip.vision.embedding_length"])class Qwen3VisBlocks:- def __init__(self, kv=None, weights=None):+ def __init__(self, kv:dict, weights:dict):self.blk = []for _ in range(kv["clip.vision.block_count"]): self.blk.append(Qwen3VisBlock(kv, weights=weights))self.patch_embd = Qwen3PatchEmbed(kv=kv, weights=weights)@@ -324,14 +326,14 @@ class Qwen3VisBlocks:self.post_ln = nn.LayerNorm(weights["v.post_ln.weight"].shape[0], eps=1e-6, elementwise_affine=True)class DeepstackLayer:- def __init__(self, index, weights):+ def __init__(self, index:int, weights:dict):self.fc1 = nn.Linear(*weights[f"v.deepstack.{index}.fc1.weight"].shape[::-1])self.fc2 = nn.Linear(*weights[f"v.deepstack.{index}.fc2.weight"].shape[::-1])self.norm = nn.LayerNorm(weights[f"v.deepstack.{index}.norm.weight"].shape[0], eps=1e-6, elementwise_affine=True)self.hidden_size = weights[f"v.deepstack.{index}.norm.weight"].shape[0]#https://github.com/huggingface/transformers/blob/027d1a97025295a1346c2eb5c361259e69eedfe7/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L112- def __call__(self, hidden_states):+ def __call__(self, hidden_states:Tensor):deepstack_feature = (hidden_states.view(-1, self.hidden_size)).view(-1, self.hidden_size)return self.fc2(Tensor.gelu(self.fc1(deepstack_feature)))@@ -345,19 +347,18 @@ class Qwen3VisBlock:self.attn_out = nn.Linear(kv["clip.vision.embedding_length"], kv["clip.vision.embedding_length"])self.attn_qkv = nn.Linear(*weights["v.blk.0.attn_qkv.weight"].shape[::-1])- def __call__(self, hidden_states, cos, sin):+ #https://github.com/huggingface/transformers/blob/1316cd76c0ce328228e08d55dc257484961b074c/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L280+ def __call__(self, hidden_states:Tensor, position_embeddings:tuple[Tensor, Tensor]):hidden_states_input = self.ln1(hidden_states)- qkv = self.attn_qkv(hidden_states_input)# https://github.com/huggingface/transformers/blob/1316cd76c0ce328228e08d55dc257484961b074c/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L186- qkv = qkv.reshape(qkv.shape[0], 3, self.num_heads, -1).permute(1, 0, 2, 3)- query, key, value = qkv[0], qkv[1], qkv[2]+ query, key, value = self.attn_qkv(hidden_states_input).reshape(hidden_states.shape[0], 3, self.num_heads, -1).permute(1, 0, 2, 3)+ cos, sin = position_embeddingsquery, key = apply_rotary_pos_emb_vision(query, key, cos, sin)query = query.transpose(0, 1).unsqueeze(0)value = value.transpose(0, 1).unsqueeze(0)key = key.transpose(0, 1).unsqueeze(0)attn_output = query.scaled_dot_product_attention(key, value)-attn_output = attn_output.transpose(1, 2)attn_output = attn_output.reshape(attn_output.shape[1], -1)attn_output = self.attn_out(attn_output)