Add object detection, tracking, mobile notifications, and search to any security camera.
update qwen (#157)
Rory Clear39d43d313502
diff --git a/clearcam.py b/clearcam.pyindex c5acacb..fd8ed2b 100644--- a/clearcam.py+++ b/clearcam.py@@ -1377,7 +1377,7 @@ if __name__ == "__main__":print("prewarming Qwen3VL....")qwen = Qwen3VL(size=f"{qwen_size}B")qwen_prompt = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>\nWhat has been detected on my CCTV camera? Write in one short sentence<|im_end|>\n<|im_start|>assistant\n"- qwen.prewarm(res=(540, 960, 3), prompt=qwen_prompt)+ qwen.prewarm(res=(540, 960, 3))print("DONE")use_qwen = Trueelse: userID = Nonediff --git a/llm/qwen3vl.py b/llm/qwen3vl.pyindex 98ef082..c8c35d2 100644--- a/llm/qwen3vl.py+++ b/llm/qwen3vl.py@@ -153,6 +153,7 @@ class Qwen3VL():self.lang, kv = Transformer.from_gguf(fetch(f"https://huggingface.co/Qwen/Qwen3-VL-{size}-Instruct-GGUF/resolve/main/Qwen3VL-{size}-Instruct-F16.gguf"), self.max_context) # max contextself.tok = SimpleTokenizer.from_gguf_kv(kv)self.start_pos = 0+ self.img_done = False # todo, different format for first text after img, is it needed?def preprocess(self, image, prompt):pixel_values, image_grid_thw = self.vis.preprocess_img(image=Tensor(image))@@ -166,7 +167,8 @@ class Qwen3VL():input_ids = Tensor([text_inputs])return pixel_values, input_ids, seq_len, image_grid_thw- def prewarm(self, res, prompt):+ def prewarm(self, res):+ prompt = f"<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>\n"pixel_values, input_ids, seq_len, image_grid_thw = self.preprocess(image=np.random.randint(0, 256, size=res, dtype=np.uint8), prompt=prompt)for _ in range(2):self.vis.preprocess_img(image=Tensor.rand(res).cast(dtypes.uint8))@@ -175,18 +177,23 @@ class Qwen3VL():self.lang.prefill_jit(tokens=Tensor([[42]*self.max_context]).clone()[:, :Variable("len",1,self.max_context).bind(42)], \start_pos=Variable("pos",1,self.max_context).bind(42), temperature=Tensor(0.7).clone())- def generate(self, prompt, image=None):+ def generate(self, prompt=None, image=None):if image is not None:- pixel_values, input_ids, seq_len, image_grid_thw = self.preprocess(image=image, prompt=prompt)+ img_prompt = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>\n"+ pixel_values, input_ids, seq_len, image_grid_thw = self.preprocess(image=image, prompt=img_prompt)self.start_pos = seq_lentoken = self.prefill(pixel_values=pixel_values, input_ids=input_ids, image_grid_thw=image_grid_thw)- else:- prompt = self.tok.encode(prompt)- prompt_len = len(prompt)- prompt = prompt + [0] * (self.max_context - prompt_len)- tokens = Tensor(prompt).unsqueeze(0)- token = self.lang.prefill_jit(tokens=tokens[:, :Variable("len",1,self.max_context).bind(prompt_len)], start_pos=Variable("pos",1,self.max_context).bind(self.start_pos), temperature=Tensor(0.7).clone())[0]- self.start_pos += prompt_len+ self.img_done = False+ if prompt is None: return+ prompt = f"{prompt}<|im_end|>\n<|im_start|>assistant\n"+ if self.img_done: prompt = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>\n" + prompt+ self.img_done = True+ prompt = self.tok.encode(prompt)+ prompt_len = len(prompt)+ prompt = prompt + [0] * (self.max_context - prompt_len)+ tokens = Tensor(prompt).unsqueeze(0)+ token = self.lang.prefill_jit(tokens=tokens[:, :Variable("len",1,self.max_context).bind(prompt_len)], start_pos=Variable("pos",1,self.max_context).bind(self.start_pos), temperature=Tensor(0.7).clone())[0]+ self.start_pos += prompt_lentoks_out = []decoded = ""@@ -231,10 +238,8 @@ class Qwen3VL():if i in self.vis.v.deepstack_idx:hidden_states = deepstack_process(hidden_states=hidden_states, visual_pos_masks=image_mask.squeeze(0), visual_embeds=(deepstack_feature_lists[self.vis.v.deepstack_idx.index(i)])).unsqueeze(0)- hidden_states = self.lang.output_norm(hidden_states)- outputs = hidden_states[:, -1:, :] @ self.lang.token_embd.weight.T-- next_token_logits = outputs[:, -1, :]+ hidden_states = self.lang.output_norm(hidden_states[:, -1, :])+ next_token_logits = hidden_states @ self.lang.token_embd.weight.Tscores = next_token_logits / TEMPtoken = sample(scores[0], temp=TEMP, k=TOP_K, p=TOP_P, af=None, ap=None)return token@@ -252,31 +257,14 @@ class Qwen3VLVis():kv, state_dict = gguf_load(fetch(f"https://huggingface.co/Qwen/Qwen3-VL-{size}-Instruct-GGUF/resolve/main/mmproj-Qwen3VL-{size}-Instruct-F16.gguf"))self.merge_size = kv["clip.vision.spatial_merge_size"]self.patch_size = kv["clip.vision.patch_size"]+ self.temporal_patch_size = 2self.v = Qwen3VisBlocks(kv=kv, weights=state_dict)self.mm = [nn.Linear(*state_dict["mm.0.weight"].shape[::-1], bias=True), None, nn.Linear(*state_dict["mm.2.weight"].shape[::-1], bias=True)]state_dict["v.patch_embd.weight1"] = state_dict["v.patch_embd.weight.1"]load_state_dict(self, state_dict)self.inv_freq = 1.0 / (10000.0 ** (Tensor.arange(0, 32, 2, dtype=dtypes.float) / 32))- def __call__(self, pixel_values, image_grid_thw):- hidden_states = pixel_values.view(-1, 3, 2, 16, 16)- B, C, D, H, W = hidden_states.shape- x = hidden_states.reshape(B, C * D, H, W)- w = Tensor.stack(self.v.patch_embd.weight, self.v.patch_embd.weight1, dim=2)- out_C, in_C, kD, kH, kW = w.shape- w2d = w.reshape(out_C, in_C * kD, kH, kW)-- hidden_states = x.conv2d(- weight=w2d,- bias=self.v.patch_embd.bias,- stride=(16, 16),- padding=(0, 0),- dilation=(1, 1),- groups=1- )-- hidden_states = hidden_states.view(-1, 1024)-+ def __call__(self, pixel_values, image_grid_thw):grid_ts = image_grid_thw[0]grid_hs = image_grid_thw[1]grid_ws = image_grid_thw[2]@@ -318,7 +306,6 @@ class Qwen3VLVis():merge_size = 2pos_embeds = patch_pos_embeds.repeat(grid_ts, 1)pos_embeds = (pos_embeds.view(grid_ts, grid_hs // merge_size, merge_size, grid_ws // merge_size, merge_size, -1).permute(0, 1, 3, 2, 4, 5).flatten(0, 4))- hidden_states = hidden_states + pos_embedshpos_ids = Tensor.arange(image_grid_thw[1]).unsqueeze(1).expand(-1, image_grid_thw[2])hpos_ids = hpos_ids.reshape(image_grid_thw[1] // merge_size, merge_size, image_grid_thw[2] // merge_size, merge_size).transpose(1, 2).flatten()@@ -330,6 +317,24 @@ class Qwen3VLVis():rotary_pos_emb = (pos_ids.unsqueeze(-1) * self.inv_freq).flatten(1)+ hidden_states = pixel_values.view(-1, 3, 2, 16, 16)+ hidden_states = hidden_states.flatten(1, 2)+ w = Tensor.stack(self.v.patch_embd.weight, self.v.patch_embd.weight1, dim=2)+ out_C, in_C, kD, kH, kW = w.shape+ w2d = w.reshape(out_C, in_C * kD, kH, kW)++ hidden_states = hidden_states.conv2d(+ weight=w2d,+ bias=self.v.patch_embd.bias,+ stride=(16, 16),+ padding=(0, 0),+ dilation=(1, 1),+ groups=1+ )++ hidden_states = hidden_states.view(-1, 1024)+ hidden_states = hidden_states + pos_embeds+sqlen, _ = hidden_states.size()hidden_states = hidden_states.reshape(sqlen, -1)rotary_pos_emb = rotary_pos_emb.reshape(sqlen, -1)@@ -352,7 +357,6 @@ class Qwen3VLVis():@TinyJitdef preprocess_img(self, image):image = image.permute(2, 0, 1)- temporal_patch_size = 2height, width = image.shape[-2:]resized_height, resized_width = smart_resize(height,@@ -365,7 +369,7 @@ class Qwen3VLVis():image = image.interpolate(size=(resized_height, resized_width))resized_height, resized_width = image.shape[-2:]patches = (image - 127.5) / 127.5- batch_size, channel = patches.shape[:2]+ batch_size, channel = 1, 3grid_h, grid_w = resized_height // self.patch_size, resized_width // self.patch_sizepatches = patches.reshape(batch_size,@@ -380,11 +384,11 @@ class Qwen3VLVis():patches = patches.permute(0, 2, 5, 3, 6, 1, 4, 7)pixel_values = (patches.unsqueeze(6)- .expand(-1, -1, -1, -1, -1, -1, temporal_patch_size, -1, -1)+ .expand(-1, -1, -1, -1, -1, -1, self.temporal_patch_size, -1, -1).reshape(batch_size,grid_h * grid_w,- channel * temporal_patch_size * self.patch_size * self.patch_size,+ channel * self.temporal_patch_size * self.patch_size * self.patch_size, # 1536))[0]return pixel_values.cast(dtypes.bfloat16), Tensor([1, grid_h, grid_w])