Skip to content

Add object detection, tracking, mobile notifications, and search to any security camera.

Qwen (#156)

Rory Clearfb9e1752961f
diff --git a/clearcam.py b/clearcam.py
index f33a9d5..c5acacb 100644
--- a/clearcam.py
+++ b/clearcam.py
@@ -1,14 +1,13 @@
from tinygrad.tensor import Tensor
-from tinygrad import TinyJit
from tinygrad.helpers import fetch
-from detection.yolov9 import safe_load, load_state_dict, YOLOv9
+from detection.yolov9 import YOLOv9
+from llm.qwen3vl import Qwen3VL
from detection.rfdetr import RFDETR, detr_to_yolo
import numpy as np
from pathlib import Path
import cv2
from collections import defaultdict, deque
import time, sys
-from tinygrad.nn.state import safe_load, load_state_dict
import json
import http
from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -491,9 +490,12 @@ class VideoCapture:
if (plain := filepath / f"{ts}.jpg").exists() and (filepath / f"{ts}_notif.jpg").exists():
plain.unlink() # only one image per event
self.filename[cam_name] = filepath / f"{ts}_notif.jpg"
- text = f"Event Detected ({cam_name})"
if userID is not None and not self.vod[cam_name] and alert.is_notif:
- threading.Thread(target=send_notif, args=(userID,text,), daemon=True).start()
+ title = f"Event Detected ({cam_name})"
+ threading.Thread(target=send_notif, args=(userID,title,None), daemon=True).start()
+ if use_qwen: # extra notif if qwen
+ text = qwen.generate(prompt=qwen_prompt, image=cv2.resize(cv2.cvtColor(cv2.imread(self.filename[cam_name]), cv2.COLOR_BGR2RGB), (960, 540)))
+ threading.Thread(target=send_notif, args=(userID,f"AI Summary ({cam_name}):",text), daemon=True).start()
threading.Thread(target=export_and_upload, kwargs={"cam_name": cam_name, "thumbnail": self.filename[cam_name], "userID": userID, "key": key, "start": ts, "wait":True}, daemon=True).start()
self.last_det[cam_name] = time.time()
alert.last_det = time.time()
@@ -1345,8 +1347,6 @@ if __name__ == "__main__":
multiprocessing.set_start_method("spawn", force=True)
database = db()
cams = database.run_get("links", None)
- url = next((arg.split("=", 1)[1] for arg in sys.argv[1:] if arg.startswith("--rtsp=")), None)
- is_file = url.endswith(('.mp4', '.avi', '.mov', '.mkv', '.webm')) if url is not None else False
classes = {"0","1","2","7"} # person, bike, car, truck, bird (14)
userID = next((arg.split("=", 1)[1] for arg in sys.argv[1:] if arg.startswith("--userid=")), None)
@@ -1367,20 +1367,27 @@ if __name__ == "__main__":
use_face = input("Would you like to enable (experimental) face recognition search? (y/n), or press enter to skip:") or False
use_face = use_face in ["y", "Y"]
- if url is None and userID is None:
- userID = input("enter your Clearcam user id or press Enter to skip: ")
- if len(userID) > 0:
- key = ""
- while len(key) < 1: key = input("enter a password for encryption: ")
- sys.argv.extend([f"--use_clip={use_clip}", f"--use_face={use_face}" ,f"--userid={userID}", f"--key={key}", f"--yolo_size={model_variant}"])
- else: userID = None
+ userID = input("enter your Clearcam user id or press Enter to skip: ")
+ use_qwen = False
+ if len(userID) > 0:
+ key = ""
+ while len(key) < 1: key = input("enter a password for encryption: ")
+ qwen_size = input("Select a Qwen3VL model for AI Summaries from \n2: 2B\n4: 4B\nor press enter to skip:") or False
+ if qwen_size in ["2", "4"]:
+ print("prewarming Qwen3VL....")
+ qwen = Qwen3VL(size=f"{qwen_size}B")
+ qwen_prompt = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>\nWhat has been detected on my CCTV camera? Write in one short sentence<|im_end|>\n<|im_start|>assistant\n"
+ qwen.prewarm(res=(540, 960, 3), prompt=qwen_prompt)
+ print("DONE")
+ use_qwen = True
+ else: userID = None
if userID is not None and key is None:
print("Error: key is required when userID is provided")
sys.exit(1)
- if use_clip or use_face:
- from objects import ObjectFinder
+ if use_clip or use_face: from objects import ObjectFinder
+
object_queue = []
cam_name = next((arg.split("=", 1)[1] for arg in sys.argv[1:] if arg.startswith("--cam_name=")), "my_camera")
diff --git a/llm/__pycache__/qwen3vl.cpython-311.pyc b/llm/__pycache__/qwen3vl.cpython-311.pyc
new file mode 100644
index 0000000..e94661c
Binary files /dev/null and b/llm/__pycache__/qwen3vl.cpython-311.pyc differ
diff --git a/llm/qwen3vl.py b/llm/qwen3vl.py
new file mode 100644
index 0000000..98ef082
--- /dev/null
+++ b/llm/qwen3vl.py
@@ -0,0 +1,478 @@
+import unicodedata, re, math, typing, sys, cv2, time
+import numpy as np
+from tinygrad import Tensor, nn, TinyJit, Variable, dtypes
+Tensor.manual_seed(42)
+from tinygrad.nn.state import safe_load, load_state_dict
+from tinygrad.helpers import partition, fetch
+from utils.gguf import gguf_load
+from utils.model import Transformer
+
+TEMP = 0.7
+TOP_K = 20
+TOP_P = 0.8
+
+class SimpleTokenizer:
+ def __init__(self, normal_tokens:dict[str, int], special_tokens:dict[str, int], preset:str="llama3",
+ bos_id:int|None=None, eos_id:int=0, eot_id:int|None=None):
+ preset = {"qwen35":"qwen2","qwen35moe":"qwen2"}.get(preset, preset)
+ if preset not in ("llama3","llama-v3","llama-bpe","qwen2","olmo","kimi-k2","tekken","glm4"):
+ raise ValueError(f"Invalid tokenizer preset '{preset}'")
+ # https://github.com/openai/gpt-2/blob/9b63575ef42771a015060c964af2c3da4cf7c8ab/src/encoder.py#L9
+ bs = [*range(33, 127), *range(161, 173), *range(174, 256)] # bytes that map to themselves
+ self._byte_decoder = {chr(b): b for b in bs} | {chr(256+i): b for i,b in enumerate(b for b in range(256) if b not in bs)}
+
+ # https://github.com/ggml-org/llama.cpp/blob/94933c8c2eeaa9a7983e3f6c08af76bd86724094/src/llama-vocab.cpp#L286
+ # 0x323b0 is one past the max codepoint in unicode categories L/N/Z (0x323af is max L)
+ def ucat_range(pre: str): return "".join(re.escape(chr(cp)) for cp in range(0x323b0) if unicodedata.category(chr(cp)).startswith(pre))
+ r_ws, r_p_N, r_p_L = r"\t\n\x0b\x0c\r\x85" + ucat_range("Z"), ucat_range("N"), ucat_range("L")
+ self._split_to_word = re.compile("(?i:'s|'t|'re|'ve|'m|'ll|'d)|" + \
+ f"[^\\r\\n{r_p_N}{r_p_L}]?[{r_p_L}]+|[{r_p_N}]{{1,3}}| ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*|[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+")
+ self._split_to_sentence = re.compile("|".join(re.escape(tok) for tok in special_tokens.keys()) if special_tokens else r"(?!)")
+
+ self._normal_tokens = {bytes(self._byte_decoder[c] for c in tok): tid for tok, tid in normal_tokens.items()}
+ self._special_tokens = special_tokens
+ self._tok2bytes = {tid: tok for tok, tid in self._normal_tokens.items()} | {tid: tok.encode() for tok, tid in self._special_tokens.items()}
+ self.preset = preset
+ self.bos_id, self.eos_id, self.eot_id = bos_id, eos_id, eot_id
+
+ @staticmethod
+ def from_gguf_kv(kv:dict):
+ # https://github.com/ggml-org/llama.cpp/blob/94933c8c2eeaa9a7983e3f6c08af76bd86724094/src/llama-vocab.cpp#L1818-L1820
+ vocab: typing.Iterable[tuple[str, int]] = ((tok, idx) for idx, tok in enumerate(kv["tokenizer.ggml.tokens"]))
+ normal_tokens, special_tokens = partition(vocab, lambda e: kv["tokenizer.ggml.token_type"][e[1]] == 1)
+ return SimpleTokenizer(dict(normal_tokens), dict(special_tokens), kv["tokenizer.ggml.pre"],
+ bos_id=kv.get('tokenizer.ggml.bos_token_id') if kv.get('tokenizer.ggml.add_bos_token', True) else None,
+ eos_id=kv.get('tokenizer.ggml.eos_token_id', 0), eot_id=kv.get('tokenizer.ggml.eot_token_id'))
+
+ def _encode_word(self, word:bytes) -> list[int]:
+ if (early_token:=self._normal_tokens.get(word)) is not None: return [early_token]
+ parts = [bytes([b]) for b in word]
+ # greedily merge any parts that we can
+ while True:
+ i = min([(sys.maxsize, -1)] + [(self._normal_tokens.get(parts[j]+parts[j+1], sys.maxsize), j) for j in range(len(parts)-1)])[1]
+ if i == -1: break
+ parts[i:i+2] = [parts[i] + parts[i+1]]
+ try: return [self._normal_tokens[p] for p in parts]
+ except KeyError: raise RuntimeError("token not found")
+ def _encode_sentence(self, chunk:str) -> list[int]:
+ return [tok for word in self._split_to_word.findall(chunk) for tok in self._encode_word(word.encode())]
+ def encode(self, text:str) -> list[int]:
+ tokens: list[int] = []
+ pos = 0
+ for match in self._split_to_sentence.finditer(text):
+ tokens.extend(self._encode_sentence(text[pos:match.start(0)]) + [self._special_tokens[text[match.start(0):match.end(0)]]])
+ pos = match.end(0)
+ ret = tokens + self._encode_sentence(text[pos:])
+ return ret
+
+ def decode(self, ids:list[int]) -> str: return b''.join(self._tok2bytes[tid] for tid in ids).decode(errors='replace')
+ def stream_decoder(self) -> typing.Callable[..., str]:
+ dec = codecs.getincrementaldecoder('utf-8')('replace')
+ def _decode(tid:int|None=None) -> str: return dec.decode(self._tok2bytes[tid]) if tid is not None else dec.decode(b'', final=True)
+ return _decode
+ def role(self, role:str):
+ if self.preset == 'olmo': return self.encode("<|" + role + "|>\n") # OLMoE Instruct format
+ if self.preset == 'kimi-k2': return self.encode("<|im_" + role + "|>" + role + "<|im_middle|>")
+ if self.preset == 'qwen2': return self.encode("<|im_start|>" + role + "\n")
+ if self.preset == 'glm4': return self.encode("<|" + role + "|>")
+ if self.preset == 'tekken':
+ if role == 'user': return self.encode("[INST]")
+ if role == 'assistant': return []
+ raise ValueError(f"Unsupported role '{role}' for tokenizer preset '{self.preset}'")
+ return self.encode("<|start_header_id|>" + role + "<|end_header_id|>\n\n")
+ def end_turn(self):
+ if self.preset == 'olmo': return self.encode("\n")
+ if self.preset == 'kimi-k2': return [self.eos_id]
+ if self.preset == 'qwen2': return [self.eos_id] + self.encode("\n")
+ if self.preset == 'glm4': return []
+ if self.preset == 'tekken': return self.encode("[/INST]")
+ return [self.eos_id]
+ def prefix(self) -> list[int]:
+ return ([] if self.bos_id is None else [self.bos_id]) + (self.encode("<sop>") if self.preset == 'glm4' else [])
+ def is_end(self, token_id:int) -> bool: return token_id in (self.eos_id, self.eot_id)
+
+def rotate_half(x):
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2 :]
+ ret = Tensor.cat(-x2, x1, dim=-1)
+ return ret
+
+
+def sample(logits, temp: float, k: int, p: float, af: float, ap: float):
+ assert logits.ndim == 1, "only works on 1d tensors"
+ assert 0 <= p <= 1, "p must be between 0 and 1"
+ assert 0 <= k <= logits.numel(), "k must be between 0 and numel"
+
+ # if temperature is very low just use argmax
+ if temp < 1e-6: return logits.argmax()
+
+ # alpha sampling
+ if af or ap:
+ if not hasattr(sample, "alpha_counter"):
+ setattr(sample, "alpha_counter", Tensor.zeros_like(logits, dtype=dtypes.int32).contiguous())
+ logits = logits - (sample.alpha_counter * af + (sample.alpha_counter > 0) * ap)
+
+ # replace NaNs with -inf
+ logits = (logits != logits).where(-float("inf"), logits)
+
+ # softmax
+ t = (logits / temp).softmax()
+
+ counter, counter2 = Tensor.arange(t.numel(), device=logits.device).contiguous(), Tensor.arange(t.numel() - 1, -1, -1, device=logits.device).contiguous()
+ # top k
+ if k:
+ output, output_indices = Tensor.zeros(k, device=logits.device).contiguous(), Tensor.zeros(k, device=logits.device, dtype=dtypes.int32).contiguous()
+ for i in range(k):
+ t_argmax = (t.numel() - ((t == (t_max := t.max())) * counter2).max() - 1).cast(dtypes.default_int)
+ output = output + t_max.unsqueeze(0).pad(((i, k - i - 1),))
+ output_indices = output_indices + t_argmax.unsqueeze(0).pad(((i, k - i - 1),))
+ t = (counter == t_argmax).where(0, t)
+
+ # approximate top p
+ # because we are already limited to top k elements we can do top p "without sorting"
+ output_cumsum = output[::-1].cumsum()[::-1] + t.sum()
+ output = (output_cumsum >= (1 - p)) * output
+ output_indices = (output_cumsum >= (1 - p)) * output_indices
+
+ # sample
+ output_idx = output.multinomial()
+ output_token = output_indices[output_idx]
+ else:
+ output_token = t.multinomial()
+
+ # increase alpha counter
+ if af or ap:
+ sample.alpha_counter = (counter == output_token).where(sample.alpha_counter + 1, sample.alpha_counter)
+
+ return output_token
+
+class Qwen3VL():
+ def __init__(self, size="2B"):
+ self.max_context = 2000
+ self.vis = Qwen3VLVis(size=size)
+ self.lang, kv = Transformer.from_gguf(fetch(f"https://huggingface.co/Qwen/Qwen3-VL-{size}-Instruct-GGUF/resolve/main/Qwen3VL-{size}-Instruct-F16.gguf"), self.max_context) # max context
+ self.tok = SimpleTokenizer.from_gguf_kv(kv)
+ self.start_pos = 0
+
+ def preprocess(self, image, prompt):
+ pixel_values, image_grid_thw = self.vis.preprocess_img(image=Tensor(image))
+ image_grid_thw = image_grid_thw.numpy().tolist()
+ text_inputs = self.tok.encode(prompt)
+ image_token_id = 151655
+ image_token_positions = [i for i, tid in enumerate(text_inputs) if tid == image_token_id]
+ num_image_tokens = ((image_grid_thw[0]*image_grid_thw[1]*image_grid_thw[2]) / 4)
+ for pos in reversed(image_token_positions): text_inputs[pos:pos+1] = [image_token_id] * int(num_image_tokens)
+ seq_len=len(text_inputs)
+ input_ids = Tensor([text_inputs])
+ return pixel_values, input_ids, seq_len, image_grid_thw
+
+ def prewarm(self, res, prompt):
+ pixel_values, input_ids, seq_len, image_grid_thw = self.preprocess(image=np.random.randint(0, 256, size=res, dtype=np.uint8), prompt=prompt)
+ for _ in range(2):
+ self.vis.preprocess_img(image=Tensor.rand(res).cast(dtypes.uint8))
+ self.prefill(pixel_values=pixel_values, input_ids=input_ids, image_grid_thw=image_grid_thw)
+ self.lang(tokens=Tensor([[42]]).clone(), start_pos=Variable("pos",1,self.max_context).bind(seq_len), temperature=Tensor(0.7).clone())
+ self.lang.prefill_jit(tokens=Tensor([[42]*self.max_context]).clone()[:, :Variable("len",1,self.max_context).bind(42)], \
+ start_pos=Variable("pos",1,self.max_context).bind(42), temperature=Tensor(0.7).clone())
+
+ def generate(self, prompt, image=None):
+ if image is not None:
+ pixel_values, input_ids, seq_len, image_grid_thw = self.preprocess(image=image, prompt=prompt)
+ self.start_pos = seq_len
+ token = self.prefill(pixel_values=pixel_values, input_ids=input_ids, image_grid_thw=image_grid_thw)
+ else:
+ prompt = self.tok.encode(prompt)
+ prompt_len = len(prompt)
+ prompt = prompt + [0] * (self.max_context - prompt_len)
+ tokens = Tensor(prompt).unsqueeze(0)
+ token = self.lang.prefill_jit(tokens=tokens[:, :Variable("len",1,self.max_context).bind(prompt_len)], start_pos=Variable("pos",1,self.max_context).bind(self.start_pos), temperature=Tensor(0.7).clone())[0]
+ self.start_pos += prompt_len
+ toks_out = []
+ decoded = ""
+
+ while True:
+ ts = time.time()
+ if toks_out:
+ token = self.lang(tokens=next_token_tensor.clone(), start_pos=Variable("pos",1,self.max_context).bind(self.start_pos), temperature=Tensor(0.7).clone())[0]
+ self.start_pos += 1
+ next_token = int(token.numpy()[0])
+ next_token_tensor = Tensor([[next_token]])
+ if next_token == 151645: break
+ toks_out.append(next_token)
+ new_text = self.tok.decode([next_token])
+ decoded += new_text
+ tok_s = f" ({1/(time.time()-ts):.1f} tok/s)"
+ print(new_text + tok_s, end="", flush=True)
+ print("\b" * len(tok_s), end="", flush=True)
+ print("\n")
+ return self.tok.decode(toks_out)
+
+ @TinyJit
+ def prefill(self, pixel_values, input_ids, image_grid_thw):
+ image_embeds, hidden_states, deepstack_feature_lists = self.vis(pixel_values, image_grid_thw)
+ image_mask = input_ids == 151655
+
+ inputs_embeds = self.lang.token_embd(input_ids)
+ image_mask = image_mask.unsqueeze(-1).expand(inputs_embeds.shape)
+ image_embeds = image_embeds.view(-1)
+ flat_mask = image_mask.view(-1)
+ idx = (flat_mask.cumsum(0) - 1).clamp(0)
+ expanded = image_embeds[idx] * flat_mask
+ flat_inputs = inputs_embeds.view(-1)
+ flat_inputs = flat_inputs * (~flat_mask) + expanded
+ hidden_states = flat_inputs.view(inputs_embeds.shape)
+
+ # https://github.com/huggingface/transformers/blob/08692e3c31654e4825b4c078a3c70b86efa70a46/src/transformers/models/qwen3_vl/modular_qwen3_vl.py#L626
+ # https://github.com/huggingface/transformers/blob/08692e3c31654e4825b4c078a3c70b86efa70a46/src/transformers/models/qwen3_vl/modular_qwen3_vl.py#L543
+ for i in range(len(self.lang.blk)):
+ self.lang.blk[i]._init_state(Tensor.zeros(1, 1))
+ hidden_states = self.lang.blk[i](hidden_states, start_pos=0)
+ # https://github.com/huggingface/transformers/blob/08692e3c31654e4825b4c078a3c70b86efa70a46/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L692
+ if i in self.vis.v.deepstack_idx:
+ hidden_states = deepstack_process(hidden_states=hidden_states, visual_pos_masks=image_mask.squeeze(0), visual_embeds=(deepstack_feature_lists[self.vis.v.deepstack_idx.index(i)])).unsqueeze(0)
+
+ hidden_states = self.lang.output_norm(hidden_states)
+ outputs = hidden_states[:, -1:, :] @ self.lang.token_embd.weight.T
+
+ next_token_logits = outputs[:, -1, :]
+ scores = next_token_logits / TEMP
+ token = sample(scores[0], temp=TEMP, k=TOP_K, p=TOP_P, af=None, ap=None)
+ return token
+
+def deepstack_process(hidden_states, visual_pos_masks, visual_embeds):
+ mask_float = visual_pos_masks.any(axis=1)
+ positions = mask_float.cumsum(axis=0) - 1
+ positions = positions.clamp(0)
+ expanded = visual_embeds[positions]
+ expanded = expanded * mask_float.unsqueeze(-1)
+ return hidden_states[0] + expanded
+
+class Qwen3VLVis():
+ def __init__(self, size="2B"):
+ kv, state_dict = gguf_load(fetch(f"https://huggingface.co/Qwen/Qwen3-VL-{size}-Instruct-GGUF/resolve/main/mmproj-Qwen3VL-{size}-Instruct-F16.gguf"))
+ self.merge_size = kv["clip.vision.spatial_merge_size"]
+ self.patch_size = kv["clip.vision.patch_size"]
+ self.v = Qwen3VisBlocks(kv=kv, weights=state_dict)
+ self.mm = [nn.Linear(*state_dict["mm.0.weight"].shape[::-1], bias=True), None, nn.Linear(*state_dict["mm.2.weight"].shape[::-1], bias=True)]
+ state_dict["v.patch_embd.weight1"] = state_dict["v.patch_embd.weight.1"]
+ load_state_dict(self, state_dict)
+ self.inv_freq = 1.0 / (10000.0 ** (Tensor.arange(0, 32, 2, dtype=dtypes.float) / 32))
+
+ def __call__(self, pixel_values, image_grid_thw):
+ hidden_states = pixel_values.view(-1, 3, 2, 16, 16)
+ B, C, D, H, W = hidden_states.shape
+ x = hidden_states.reshape(B, C * D, H, W)
+ w = Tensor.stack(self.v.patch_embd.weight, self.v.patch_embd.weight1, dim=2)
+ out_C, in_C, kD, kH, kW = w.shape
+ w2d = w.reshape(out_C, in_C * kD, kH, kW)
+
+ hidden_states = x.conv2d(
+ weight=w2d,
+ bias=self.v.patch_embd.bias,
+ stride=(16, 16),
+ padding=(0, 0),
+ dilation=(1, 1),
+ groups=1
+ )
+
+ hidden_states = hidden_states.view(-1, 1024)
+
+ grid_ts = image_grid_thw[0]
+ grid_hs = image_grid_thw[1]
+ grid_ws = image_grid_thw[2]
+
+ h_idxs = Tensor.linspace(0, self.v.num_grid_per_side - 1, grid_hs)
+ w_idxs = Tensor.linspace(0, self.v.num_grid_per_side - 1, grid_ws)
+
+ h_idxs_floor = h_idxs.cast(dtypes.int32)
+ w_idxs_floor = w_idxs.cast(dtypes.int32)
+ h_idxs_ceil = (h_idxs_floor.int() + 1).clip(self.v.num_grid_per_side - 1)
+ w_idxs_ceil = (w_idxs_floor.int() + 1).clip(self.v.num_grid_per_side - 1)
+ dh = h_idxs - h_idxs_floor
+ dw = w_idxs - w_idxs_floor
+
+ base_h = h_idxs_floor * self.v.num_grid_per_side
+ base_h_ceil = h_idxs_ceil * self.v.num_grid_per_side
+
+
+ idx_tensor = Tensor.stack(
+ (base_h[None].T + w_idxs_floor[None]).flatten(),
+ (base_h[None].T + w_idxs_ceil[None]).flatten(),
+ (base_h_ceil[None].T + w_idxs_floor[None]).flatten(),
+ (base_h_ceil[None].T + w_idxs_ceil[None]).flatten(),
+ ).cast(dtypes.int32)
+
+ weight_tensor = Tensor.stack(
+ ((1 - dh)[None].T * (1 - dw)[None]).flatten(),
+ ((1 - dh)[None].T * dw[None]).flatten(),
+ (dh[None].T * (1 - dw)[None]).flatten(),
+ (dh[None].T * dw[None]).flatten(),
+ ).cast(dtypes.bfloat16)
+
+ pos_embeds = self.v.position_embd(idx_tensor)
+ pos_embeds *= weight_tensor[:, :, None]
+ patch_pos_embeds = pos_embeds[0] + pos_embeds[1] + pos_embeds[2] + pos_embeds[3]
+
+ patch_pos_embeds = patch_pos_embeds[:grid_hs * grid_ws]
+
+ merge_size = 2
+ pos_embeds = patch_pos_embeds.repeat(grid_ts, 1)
+ pos_embeds = (pos_embeds.view(grid_ts, grid_hs // merge_size, merge_size, grid_ws // merge_size, merge_size, -1).permute(0, 1, 3, 2, 4, 5).flatten(0, 4))
+ hidden_states = hidden_states + pos_embeds
+
+ hpos_ids = Tensor.arange(image_grid_thw[1]).unsqueeze(1).expand(-1, image_grid_thw[2])
+ hpos_ids = hpos_ids.reshape(image_grid_thw[1] // merge_size, merge_size, image_grid_thw[2] // merge_size, merge_size).transpose(1, 2).flatten()
+
+ wpos_ids = Tensor.arange(image_grid_thw[2]).unsqueeze(0).expand(image_grid_thw[1], -1)
+ wpos_ids = wpos_ids.reshape(image_grid_thw[1] // merge_size, merge_size, image_grid_thw[2] // merge_size, merge_size).transpose(1, 2).flatten()
+
+ pos_ids = Tensor.stack(hpos_ids, wpos_ids, dim=-1).repeat(image_grid_thw[0], 1)
+
+ rotary_pos_emb = (pos_ids.unsqueeze(-1) * self.inv_freq).flatten(1)
+
+ sqlen, _ = hidden_states.size()
+ hidden_states = hidden_states.reshape(sqlen, -1)
+ rotary_pos_emb = rotary_pos_emb.reshape(sqlen, -1)
+ emb = Tensor.cat(rotary_pos_emb, rotary_pos_emb, dim=-1)
+ cos, sin = emb.cos(), emb.sin()
+ cos, sin = cos.unsqueeze(-2), sin.unsqueeze(-2)
+
+ deepstack_feature_lists = []
+ for i in range(len(self.v.blk)):
+ hidden_states = self.v.blk[i](hidden_states, cos, sin)
+ if i in self.v.deepstack_idx: deepstack_feature_lists.append(self.v.deepstack[i](hidden_states))
+
+ image_embeds = self.v.post_ln(hidden_states)
+ image_embeds = image_embeds.view(-1, 4096)
+ image_embeds = self.mm[0](image_embeds)
+ image_embeds = Tensor.gelu(image_embeds)
+ image_embeds = self.mm[2](image_embeds)
+ return image_embeds, hidden_states, deepstack_feature_lists
+
+ @TinyJit
+ def preprocess_img(self, image):
+ image = image.permute(2, 0, 1)
+ temporal_patch_size = 2
+ height, width = image.shape[-2:]
+ resized_height, resized_width = smart_resize(
+ height,
+ width,
+ factor=self.patch_size * self.merge_size,
+ min_pixels=65536,
+ max_pixels=16777216,
+ )
+ image = image.unsqueeze(0).float()
+ image = image.interpolate(size=(resized_height, resized_width))
+ resized_height, resized_width = image.shape[-2:]
+ patches = (image - 127.5) / 127.5
+ batch_size, channel = patches.shape[:2]
+ grid_h, grid_w = resized_height // self.patch_size, resized_width // self.patch_size
+ patches = patches.reshape(
+ batch_size,
+ channel,
+ grid_h // self.merge_size,
+ self.merge_size,
+ self.patch_size,
+ grid_w // self.merge_size,
+ self.merge_size,
+ self.patch_size,
+ )
+ patches = patches.permute(0, 2, 5, 3, 6, 1, 4, 7)
+ pixel_values = (
+ patches.unsqueeze(6)
+ .expand(-1, -1, -1, -1, -1, -1, temporal_patch_size, -1, -1)
+ .reshape(
+ batch_size,
+ grid_h * grid_w,
+ channel * temporal_patch_size * self.patch_size * self.patch_size,
+ )
+ )[0]
+ return pixel_values.cast(dtypes.bfloat16), Tensor([1, grid_h, grid_w])
+
+# https://github.com/huggingface/transformers/blob/90e3c4fa7200a9c8bb9756bf7bf43381d10850c0/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L62
+def smart_resize(height, width, factor, min_pixels, max_pixels):
+ h_bar = round(height / factor) * factor
+ w_bar = round(width / factor) * factor
+ if h_bar * w_bar > max_pixels:
+ beta = math.sqrt((height * width) / max_pixels)
+ h_bar = max(factor, math.floor(height / beta / factor) * factor)
+ w_bar = max(factor, math.floor(width / beta / factor) * factor)
+ elif h_bar * w_bar < min_pixels:
+ beta = math.sqrt(min_pixels / (height * width))
+ h_bar = math.ceil(height * beta / factor) * factor
+ w_bar = math.ceil(width * beta / factor) * factor
+ return h_bar, w_bar
+
+class Qwen3PatchEmbed():
+ def __init__(self, kv=None):
+ self.weight = Tensor.zeros(kv["clip.vision.embedding_length"], 3, 16, 16)
+ self.weight1 = Tensor.zeros(kv["clip.vision.embedding_length"], 3, 16, 16)
+ self.bias = Tensor.zeros(kv["clip.vision.embedding_length"])
+
+class Qwen3VisBlocks():
+ def __init__(self, kv=None, weights=None):
+ self.blk = []
+ for _ in range(kv["clip.vision.block_count"]): self.blk.append(Qwen3VisBlock(kv, weights=weights))
+ self.patch_embd = Qwen3PatchEmbed(kv=kv)
+ self.num_grid_per_side = 48
+ self.deepstack_layers = kv["clip.vision.is_deepstack_layers"]
+ self.deepstack_idx = [i for i, val in enumerate(self.deepstack_layers) if val]
+ self.deepstack = []
+ for i in range(len(self.deepstack_layers)):
+ if i in self.deepstack_idx:
+ self.deepstack.append(DeepstackLayer(i, weights))
+ else:
+ self.deepstack.append(None)
+ self.position_embd = nn.Embedding(*weights["v.position_embd.weight"].shape)
+ self.post_ln = nn.LayerNorm(weights["v.post_ln.weight"].shape[0], eps=1e-6, elementwise_affine=True)
+
+class DeepstackLayer:
+ def __init__(self, index, weights):
+ self.fc1 = nn.Linear(*weights[f"v.deepstack.{index}.fc1.weight"].shape[::-1])
+ self.fc2 = nn.Linear(*weights[f"v.deepstack.{index}.fc2.weight"].shape[::-1])
+ self.norm = nn.LayerNorm(weights[f"v.deepstack.{index}.norm.weight"].shape[0], eps=1e-6, elementwise_affine=True)
+ self.hidden_size = weights[f"v.deepstack.{index}.norm.weight"].shape[0]
+
+ #https://github.com/huggingface/transformers/blob/027d1a97025295a1346c2eb5c361259e69eedfe7/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L112
+ def __call__(self, hidden_states):
+ deepstack_feature = (hidden_states.view(-1, self.hidden_size)).view(-1, self.hidden_size)
+ return self.fc2(Tensor.gelu(self.fc1(deepstack_feature)))
+
+class Qwen3VisBlock():
+ def __init__(self, kv=None, weights=None):
+ self.ffn_up = nn.Linear(kv["clip.vision.embedding_length"], kv["clip.vision.feed_forward_length"])
+ self.ffn_down = nn.Linear(kv["clip.vision.feed_forward_length"], kv["clip.vision.embedding_length"])
+ self.ln1 = nn.LayerNorm(kv["clip.vision.embedding_length"], eps=1e-6, elementwise_affine=True)
+ self.ln2 = nn.LayerNorm(kv["clip.vision.embedding_length"], eps=1e-6, elementwise_affine=True)
+ self.attn_out = nn.Linear(kv["clip.vision.embedding_length"], kv["clip.vision.embedding_length"])
+ self.attn_qkv = nn.Linear(*weights["v.blk.0.attn_qkv.weight"].shape[::-1])
+
+ def __call__(self, hidden_states, cos, sin):
+ hidden_states_input = self.ln1(hidden_states)
+ seq_length = hidden_states_input.shape[0]
+ qkv = self.attn_qkv(hidden_states_input)
+ qkv_reshaped = qkv.reshape(seq_length, 3, 16, -1)
+ qkv_permuted = qkv_reshaped.permute(1, 0, 2, 3)
+ query, key, value = qkv_permuted.chunk(3, dim=0)
+ query = query.squeeze(0)
+ key = key.squeeze(0)
+ value = value.squeeze(0)
+ query = (query * cos) + (rotate_half(query) * sin)
+ key = (key * cos) + (rotate_half(key) * sin)
+
+ query = query.transpose(0, 1).unsqueeze(0)
+ key = key.transpose(0, 1).unsqueeze(0)
+ value = value.transpose(0, 1).unsqueeze(0)
+
+ attn_weight = query @ key.transpose(-2, -1) * 0.125
+ attn_weight = Tensor.softmax(attn_weight)
+ attn_output = attn_weight @ value
+ attn_output = attn_output.transpose(1, 2)
+ attn_output = attn_output.reshape(seq_length, -1)
+ attn_output = self.attn_out(attn_output)
+ hidden_states += attn_output
+ norm = self.ln2(hidden_states)
+ x = self.ffn_up(norm)
+ x = Tensor.gelu(x)
+ norm = self.ffn_down(x)
+ return hidden_states + norm
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index b4ac979..ddb110f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-git+https://github.com/tinygrad/tinygrad.git@2041945f4b4971b984fbfd73b860523e1513aab6
+git+https://github.com/tinygrad/tinygrad.git@2d48fe8b7bd9acfa00e91a7f89b28b3ded370c27
numpy==2.0.0
opencv-python-headless==4.10.0.84
\ No newline at end of file
diff --git a/utils/gguf.py b/utils/gguf.py
new file mode 100644
index 0000000..74c1a15
--- /dev/null
+++ b/utils/gguf.py
@@ -0,0 +1,177 @@
+import functools, io, pathlib, re, struct
+from typing import Any, Callable
+
+from tinygrad.tensor import Tensor
+from tinygrad.dtype import dtypes
+from tinygrad.helpers import prod, round_up
+from tinygrad.nn.state import TensorIO
+
+# ggml packs each iq grid entry as N bytes (N=4 for uint32 grids, N=8 for uint64 grids) in a single word. See ggml-common.h.
[email protected]_cache(None)
+def _ggml_iq_grid(device: str, grid: tuple[int, ...], grid_shape: tuple[int, int]) -> Tensor:
+ values = [float((w >> (8*i)) & 0xFF) for w in grid for i in range(grid_shape[1])]
+ return Tensor(values, dtype=dtypes.float32, device=device).reshape(grid_shape)
+
+# native types {ggml_type: dtype}
+_GGML_NATIVE = {0: dtypes.float32, 1: dtypes.float16, 24: dtypes.int8, 25: dtypes.int16,
+ 26: dtypes.int32, 27: dtypes.int64, 28: dtypes.float64, 30: dtypes.bfloat16}
+
+# quant types {ggml_type: (number of elements, number of bytes)}
+_GGML_QUANT = {2:(32,18), 3:(32,20), 6:(32,22), 7:(32,24), 8:(32,34),
+ 12:(256,144), 13:(256,176), 14:(256,210), 18:(256,98), 21:(256,110), 22:(256,82), 23:(256,136), 39:(32,17), 41:(128,18)}
+
+def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor:
+ """
+ Converts ggml tensor data to a tinygrad tensor.
+
+ Supported native types: float32 (id: 0), float16 (id: 1), int8 (id: 24),
+ int16 (id: 25), int32 (id: 26), int64 (id: 27), float64 (id: 28), bfloat16 (id: 30)
+ Supported quantized types: Q4_0 (id: 2), Q4_1 (id: 3), Q5_0 (id: 6),
+ Q5_1 (id: 7), Q8_0 (id: 8), Q4_K (id: 12), Q5_K (id: 13),
+ Q6_K (id: 14), IQ3_XXS (id: 18), IQ3_S (id: 21), IQ2_S (id: 22), IQ4_XS (id: 23), MXFP4 (id: 39), Q1_0 (id: 41)
+ """
+ # https://github.com/ggerganov/ggml/blob/323951f1bdcdfbd5b5ff3a9a7c3770e63b1a560e/include/ggml.h#L356
+
+ if (dtype := _GGML_NATIVE.get(ggml_type)) is not None:
+ return t[:dtype.itemsize * n].contiguous().bitcast(dtype)
+
+ def q_to_uint8(t: Tensor, b: int) -> Tensor:
+ # TODO: rewrite with arange?
+ shift_tensor, bitmask = Tensor.stack(*[ Tensor(2**(i*b), device=t.device, dtype=t.dtype) for i in range(8//b) ]), 0xff >> (8 - b)
+ return t.unsqueeze(-1).expand((*t.shape,8//b)).div(shift_tensor, rounding_mode="trunc").bitwise_and(bitmask).transpose(-1, -2).flatten(-2)
+
+ if (nelements_nbytes := _GGML_QUANT.get(ggml_type)) is not None:
+ from tinygrad.runtime.autogen import ggml_common as _ggml
+ blocks = t[:(n//nelements_nbytes[0])*nelements_nbytes[1]].reshape((-1, nelements_nbytes[1])).contiguous()
+ if ggml_type == 2: return (q_to_uint8(blocks[:,2:], 4).bitcast(dtypes.int8) - 8) * blocks[:,:2].bitcast(dtypes.float16).cast(dtypes.float32)
+ if ggml_type == 3:
+ d, m = (blocks[:,s:s+2].bitcast(dtypes.float16).cast(dtypes.float32) for s in [ 0, 2 ])
+ return q_to_uint8(blocks[:,4:], 4).bitcast(dtypes.int8) * d + m
+ if ggml_type in (6, 7):
+ d = blocks[:,:2].bitcast(dtypes.float16).cast(dtypes.float32)
+ qh_off = 2 if ggml_type == 6 else 4
+ qh = q_to_uint8(blocks[:,qh_off:qh_off+4], 1).reshape((-1, 8, 4)).transpose(-1, -2).flatten(-2).bitcast(dtypes.int8)
+ q = q_to_uint8(blocks[:,qh_off+4:], 4).bitcast(dtypes.int8) + qh * 16
+ return q * d + (blocks[:,2:4].bitcast(dtypes.float16).cast(dtypes.float32) if ggml_type == 7 else -16 * d)
+ if ggml_type == 8: return blocks[:,:2].bitcast(dtypes.float16).cast(dtypes.float32) * blocks[:,2:].bitcast(dtypes.int8)
+ # Q4_K: 256 elements per 144-byte block (d:2, dmin:2, scales:12, qs:128)
+ # Q5_K: 256 elements per 176-byte block (d:2, dmin:2, scales:12, qh:32, qs:128)
+ if ggml_type in (12, 13):
+ d, dmin = (blocks[:,i:i+2].bitcast(dtypes.float16).cast(dtypes.float32).unsqueeze(-1) for i in [0, 2])
+ s = blocks[:,4:16] # 12 bytes: 6-bit scales[0-3], 6-bit mins[0-3], high bits[4-7]
+ sc = s[:,0:4].bitwise_and(63).cat(s[:,8:12].bitwise_and(0xF).bitwise_or(s[:,0:4].rshift(6).lshift(4)), dim=-1)
+ mn = s[:,4:8].bitwise_and(63).cat(s[:,8:12].rshift(4).bitwise_or(s[:,4:8].rshift(6).lshift(4)), dim=-1)
+ qs_off = 48 if ggml_type == 13 else 16
+ q = Tensor.stack((qs:=blocks[:,qs_off:qs_off+128].reshape(-1,4,32)).bitwise_and(0xF), qs.rshift(4), dim=2).reshape(-1,8,32)
+ if ggml_type == 13: q = q + q_to_uint8(blocks[:,16:48], 1).reshape(-1, 8, 32) * 16
+ return (d * sc.unsqueeze(-1) * q - dmin * mn.unsqueeze(-1)).flatten(-2)
+ if ggml_type == 14:
+ xl, xh = q_to_uint8(blocks[:,:128].reshape((-1, 2, 64)), 4), q_to_uint8(blocks[:,128:192].reshape((-1, 2, 32)), 2).lshift(4)
+ scales = blocks[:,192:208].bitcast(dtypes.int8).unsqueeze(-1).expand((-1, 16, 16)).reshape((-1, 256))
+ d = blocks[:,-2:].bitcast(dtypes.float16).cast(dtypes.float32).expand((-1, 256))
+ return d * (xl.bitwise_or(xh).bitcast(dtypes.int8) - 32).flatten(-2) * scales
+ if ggml_type == 18:
+ d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1))
+ scale_words = blocks[:, 66:98].bitcast(dtypes.uint32)
+ db = d * (scale_words.rshift(28).cast(dtypes.float32) + 0.5).reshape((-1, 8, 1, 1)) * 0.5
+ sign_idx = scale_words.unsqueeze(-1).rshift(
+ Tensor([0, 7, 14, 21], device=t.device, dtype=dtypes.uint32)).bitwise_and(0x7F).reshape((-1, 32)).cast(dtypes.int32)
+ even_signs = Tensor([i | (0x80 if i.bit_count() % 2 else 0) for i in range(128)], dtype=dtypes.uint8, device=t.device)
+ signs = (q_to_uint8(even_signs[sign_idx].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 8, 4, 8))
+ grid = _ggml_iq_grid(t.device, _ggml.iq3xxs_grid, (256, 4))[blocks[:, 2:66]].reshape((-1, 8, 4, 8))
+ return (db * grid * signs).flatten(-3)
+ if ggml_type == 21:
+ d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1))
+ scales = (1 + 2 * q_to_uint8(blocks[:, 106:110].reshape((-1, 4, 1)), 4).reshape((-1, 8))).cast(dtypes.float32).reshape((-1, 8, 1, 1))
+ qh = q_to_uint8(blocks[:, 66:74].reshape((-1, 8, 1)), 1).reshape((-1, 64)).cast(dtypes.uint16)
+ signs = (q_to_uint8(blocks[:, 74:106].reshape((-1, 32, 1)), 1).reshape((-1, 256)) == 0).where(1.0, -1.0).reshape((-1, 8, 4, 8))
+ q = blocks[:, 2:66].cast(dtypes.uint16) + qh.lshift(8)
+ return (d * scales * _ggml_iq_grid(t.device, _ggml.iq3s_grid, (512, 4))[q].reshape((-1, 8, 4, 8)) * signs).flatten(-3)
+ if ggml_type == 22:
+ d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1))
+ db = d * (q_to_uint8(blocks[:, 74:82].reshape((-1, 8, 1)), 4).reshape((-1, 16)).cast(dtypes.float32) + 0.5).reshape((-1, 16, 1, 1)) * 0.25
+ signs = (q_to_uint8(blocks[:, 34:66].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 16, 2, 8))
+ qh = q_to_uint8(blocks[:, 66:74].reshape((-1, 8, 1)), 2).reshape((-1, 32)).cast(dtypes.uint16)
+ q = blocks[:, 2:34].cast(dtypes.uint16) + qh.lshift(8)
+ return (db * _ggml_iq_grid(t.device, _ggml.iq2s_grid, (1024, 8))[q].reshape((-1, 16, 2, 8)) * signs).flatten(-3)
+ if ggml_type == 23:
+ d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1))
+ scale_shifts = Tensor([0, 2, 4, 6, 8, 10, 12, 14], device=t.device, dtype=dtypes.uint16)
+ iq4_xs_lut = Tensor(list(_ggml.kvalues_iq4nl), dtype=dtypes.float32, device=t.device)
+ scales_l = Tensor.stack((sl:=blocks[:, 4:8]).bitwise_and(0xF), sl.rshift(4), dim=2).reshape((-1, 8))
+ scales_h = blocks[:, 2:4].bitcast(dtypes.uint16).unsqueeze(-1).rshift(scale_shifts).bitwise_and(0x03).reshape((-1, 8)).cast(dtypes.uint8)
+ scales = (scales_l.bitwise_or(scales_h.lshift(4)).bitcast(dtypes.int8) - 32).cast(dtypes.float32).reshape((-1, 8, 1))
+ q = (qs:=blocks[:, 8:].reshape((-1, 8, 16))).bitwise_and(0xF).cat(qs.rshift(4), dim=2)
+ return (d * scales * iq4_xs_lut[q]).flatten(-2)
+ if ggml_type == 39:
+ e = blocks[:, 0].cast(dtypes.uint32)
+ small_bits = Tensor([0x00200000, 0x00400000], dtype=dtypes.uint32, device=t.device)[e.clip(0, 1).cast(dtypes.int32)] # e = 0 or e = 1 case
+ d = (e < 2).where(small_bits, ((e - 1) * 0x00800000).cast(dtypes.uint32)).bitcast(dtypes.float32).unsqueeze(-1)
+ codes = q_to_uint8(blocks[:, 1:17], 4)
+ fp4_lut = Tensor([0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0,
+ -0.0,-1.0,-2.0,-3.0,-4.0,-6.0,-8.0,-12.0],
+ dtype=dtypes.float32, device=t.device)
+ fp4_val = fp4_lut[codes]
+ return (fp4_val * d).flatten(-2)[:n]
+ if ggml_type == 41:
+ d = blocks[:,:2].bitcast(dtypes.float16)
+ bits = q_to_uint8(blocks[:,2:], 1).reshape(-1, 8, 16).transpose(-1, -2).flatten(-2).bitcast(dtypes.int8)
+ return d * (bits * 2 - 1)
+ raise ValueError(f"GGML type '{ggml_type}' is not supported!")
+
+def _read_unpack(fmt: str, n: int, r:io.BufferedIOBase): return struct.unpack(fmt, r.read(n))[0]
+def read_str(r:io.BufferedIOBase): return str(r.read(read_uint64(r)), "utf-8")
+def read_arr(r:io.BufferedIOBase):
+ item_reader, n = readers[read_int32(r)], read_uint64(r)
+ return [item_reader(r) for _ in range(n)]
+
+readers: dict[int, Callable[[io.BufferedIOBase], Any]] = { 8: read_str, 9: read_arr,
+ **{ t: functools.partial(_read_unpack, "<"+f, nb) for t,f,nb in \
+ [ (0,"c",1), (1,"b",1), (2,"H",2), (3,"h",2), (4,"I",4), (5,"i",4), (6,"f",4), (7,"?",1), (10,"Q",8), (11,"q",8), (12,"d",8) ] } }
+read_uint32, read_int32, read_uint64, read_int64 = readers[4], readers[5], readers[10], readers[11]
+
+def _gguf_parse(tensor: Tensor) -> tuple[dict, dict[str, Tensor]]:
+ # TODO: remove the need for copy to default device
+ tensor = tensor.to(None).realize()
+ r = io.BufferedReader(TensorIO(tensor), 1_000_000)
+ magic, version, n_tensors, n_kv = r.read(4), read_int32(r), read_int64(r), read_int64(r)
+ if magic != b"GGUF" or version not in [2, 3]: raise ValueError("Invalid GGUF format!")
+
+ kv_data = {}
+ for _ in range(n_kv):
+ k, typ = read_str(r), read_int32(r)
+ kv_data[k] = readers[typ](r)
+
+ t_infos = [ (read_str(r), tuple(read_uint64(r) for _ in range(read_uint32(r))), read_int32(r), read_uint64(r)) for _ in range(n_tensors) ]
+ alignment, pos = kv_data.get("general.alignment", 32), r.tell()
+ data_start = round_up(pos, alignment)
+
+ state_dict = {name: ggml_data_to_tensor(tensor[data_start + off:], prod(dims), typ).reshape(*reversed(dims)) for name, dims, typ, off in t_infos}
+ return kv_data, state_dict
+
+def _gguf_split_paths(path: pathlib.Path, kv: dict) -> list[pathlib.Path]:
+ if (total := kv.get('split.count', 1)) <= 1: return [path]
+ if kv.get('split.no', 0) != 0: raise ValueError(f"multi-part GGUF must be loaded from the first split, got split.no={kv['split.no']}")
+ if not (m := re.match(r"^(.*)-00001-of-\d{5}\.gguf$", str(path))): raise ValueError(f"first split path must end with -00001-of-NNNNN.gguf: {path}")
+ return [pathlib.Path(f"{m.group(1)}-{i:05d}-of-{total:05d}.gguf") for i in range(1, total+1)]
+
+def gguf_load(fn: Tensor|str|pathlib.Path) -> tuple[dict, dict[str, Tensor]]:
+ """
+ Loads a .gguf file, returning the `kv_data` and `state_dict`. Multi-part splits are auto-merged when loaded by path.
+
+ ```python
+ import pathlib
+ from tinygrad import Device, Tensor
+ from tinygrad.llm.gguf import gguf_load
+
+ gguf_tensor = Tensor(pathlib.Path("Meta-Llama-3-8B-Instruct.Q4_0.gguf")).to(Device.DEFAULT)
+ kv_data, state_dict = gguf_load(gguf_tensor)
+ ```
+
+ NOTE: The provided tensor must be on a device that supports execution.
+ """
+ kv, sd = _gguf_parse(fn if isinstance(fn, Tensor) else Tensor(pathlib.Path(fn)))
+ if kv.get('split.count', 1) <= 1: return kv, sd
+ if isinstance(fn, Tensor): raise ValueError("multi-part GGUF requires a path argument (got Tensor)")
+ for pp in _gguf_split_paths(pathlib.Path(fn), kv)[1:]: sd.update(_gguf_parse(Tensor(pp))[1])
+ return kv, sd
diff --git a/utils/model.py b/utils/model.py
new file mode 100644
index 0000000..bc7fe59
--- /dev/null
+++ b/utils/model.py
@@ -0,0 +1,416 @@
+from __future__ import annotations
+import functools, itertools, pathlib
+from dataclasses import dataclass, replace
+from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function
+from tinygrad.llm.gguf import gguf_load
+from tinygrad.uop.ops import resolve
+
+def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0, device:str|None=None) -> Tensor:
+ freqs = 1.0 / (theta ** (Tensor.arange(0, dim, 2, device=device)[:(dim // 2)] / dim))
+ freqs = Tensor.arange(end, device=device).unsqueeze(dim=1) * freqs.unsqueeze(dim=0)
+ return freqs.cos().cat(freqs.sin(), dim=-1).contiguous()
+
+class ExpertWeights:
+ """Like nn.Linear but with num_experts dimension. Weight shape: (num_experts, out_features, in_features)."""
+ def __init__(self, num_experts:int, in_features:int, out_features:int):
+ self.weight = Tensor.zeros(num_experts, out_features, in_features)
+ def __call__(self, sel:Tensor, x:Tensor) -> Tensor:
+ # sel: (B, T, k), x: (B, T, 1, in) or (B, T, k, in) -> output: (B, T, k, out)
+ return (x.unsqueeze(-2) @ self.weight[sel].transpose(-1, -2)).contiguous().squeeze(-2)
+
+def apply_rope(x:Tensor, freqs_cis:Tensor) -> Tensor:
+ assert x.shape[-1] % 2 == 0
+ cos, sin = freqs_cis.reshape(1, 1, x.shape[2], -1).chunk(2, dim=-1)
+ x1, x2 = x.chunk(2, dim=-1)
+ return (x1 * cos - x2 * sin).cat(x2 * cos + x1 * sin, dim=-1)
+
+def pairwise_topk(x: Tensor, k: int) -> tuple[Tensor, Tensor]:
+ n = x.shape[-1]
+ vals = Tensor.arange(n, device=x.device).reshape(1,1,n).cast(x.dtype).expand(x.shape)
+ cmp = (x.unsqueeze(-1) > x.unsqueeze(-2)) | ((x.unsqueeze(-1) == x.unsqueeze(-2)) & \
+ (Tensor.arange(n, device=x.device).reshape(1,1,n,1) < Tensor.arange(n, device=x.device).reshape(1,1,1,n)))
+ sel = x.const_like(0).scatter(-1, cmp.sum(axis=-1).cast('int32'), vals)[:,:,n-k:].cast('int32')
+ return x.gather(-1, sel), sel
+
+@dataclass(frozen=True)
+class SSMConfig:
+ conv_kernel: int
+ state_size: int
+ group_count: int
+ time_step_rank: int
+ inner_size: int
+
+@dataclass(frozen=True)
+class TransformerConfig:
+ num_blocks: int
+ dim: int
+ hidden_dim: int
+ n_heads: int
+ n_kv_heads: int
+ norm_eps: float
+ vocab_size: int
+ head_dim: int
+ rope_theta: float
+ rope_dim: int
+ v_head_dim: int
+ max_context: int = 0
+ qk_norm: int = 0
+ num_experts: int = 0
+ num_experts_per_tok: int = 0
+ norm_topk_prob: bool = False
+ q_lora_rank: int = 0
+ kv_lora_rank: int = 0
+ shared_expert_dim: int = 0
+ full_attention_interval: int = 0
+ attn_output_gate: bool = False
+ ssm: SSMConfig|None = None
+ shared_expert_gate: bool = True
+ leading_dense_blocks: int = 0
+ dense_hidden_dim: int = 0
+ routed_scaling_factor: float = 1.0
+ qkv_bias: bool = False
+ expert_bias: bool = False
+
+class FFNBlock:
+ def __init__(self, config:TransformerConfig):
+ self.config = config
+
+ # --- RMSNorms --------------------------------------------------------
+ self.attn_norm = nn.RMSNorm(config.dim, config.norm_eps)
+ self.ffn_norm = nn.RMSNorm(config.dim, config.norm_eps)
+
+ # --- feed-forward (MoE or dense) -------------------------------------
+ if config.num_experts > 0:
+ self.ffn_gate_inp = nn.Linear(config.dim, config.num_experts, bias=False) # router
+ if config.expert_bias: self.exp_probs_b = {"bias": Tensor.zeros(config.num_experts)}
+ self.ffn_gate_exps = ExpertWeights(config.num_experts, config.dim, config.hidden_dim)
+ self.ffn_up_exps = ExpertWeights(config.num_experts, config.dim, config.hidden_dim)
+ self.ffn_down_exps = ExpertWeights(config.num_experts, config.hidden_dim, config.dim)
+ if config.shared_expert_dim > 0:
+ self.ffn_gate_shexp = nn.Linear(config.dim, config.shared_expert_dim, bias=False)
+ self.ffn_up_shexp = nn.Linear(config.dim, config.shared_expert_dim, bias=False)
+ self.ffn_down_shexp = nn.Linear(config.shared_expert_dim, config.dim, bias=False)
+ if config.shared_expert_gate: self.ffn_gate_inp_shexp = {"weight": Tensor.zeros(config.dim)}
+ else:
+ self.ffn_gate = nn.Linear(config.dim, config.hidden_dim, bias=False)
+ self.ffn_up = nn.Linear(config.dim, config.hidden_dim, bias=False)
+ self.ffn_down = nn.Linear(config.hidden_dim, config.dim, bias=False)
+
+ def _feed_forward(self, x:Tensor) -> Tensor:
+ if hasattr(self, 'ffn_gate_exps'):
+ h = x.unsqueeze(2) # (B, T, 1, D) - add expert dim for broadcasting
+ logits = self.ffn_gate_inp(x)
+ if hasattr(self, 'exp_probs_b'):
+ probs = logits.sigmoid()
+ _, sel = pairwise_topk(probs + self.exp_probs_b["bias"], self.config.num_experts_per_tok)
+ probs = probs.gather(-1, sel)
+ if self.config.norm_topk_prob: probs = probs / probs.sum(axis=-1, keepdim=True)
+ else:
+ vals, sel = pairwise_topk(logits, self.config.num_experts_per_tok)
+ probs = vals.softmax(-1) if self.config.norm_topk_prob else logits.softmax(-1).gather(-1, sel)
+ probs = probs * self.config.routed_scaling_factor
+ x_down = self.ffn_down_exps(sel, (self.ffn_gate_exps(sel, h).silu() * self.ffn_up_exps(sel, h)).contiguous()) # (B, T, k, D)
+ out = (x_down * probs.unsqueeze(-1)).sum(axis=2) # (B, T, D)
+ if hasattr(self, 'ffn_gate_shexp'):
+ shexp = self.ffn_down_shexp(self.ffn_gate_shexp(x).silu().contiguous() * self.ffn_up_shexp(x))
+ if hasattr(self, 'ffn_gate_inp_shexp'): shexp = shexp * (x * self.ffn_gate_inp_shexp["weight"]).sum(axis=-1, keepdim=True).sigmoid()
+ out = out + shexp
+ return out
+ # TODO: remove the need for this contiguous
+ return self.ffn_down(self.ffn_gate(x).silu().contiguous() * self.ffn_up(x))
+
+ # given the token-prefix match, return how much cached state this block can still reuse
+ def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return prefix_len
+ # return writes that reset this block's state after a cache mismatch
+ def _state_reset_ops(self) -> list[Tensor]: return []
+ def _init_state(self, x:Tensor): raise NotImplementedError
+ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: raise NotImplementedError
+
+ def __call__(self, x: Tensor, start_pos: int|UOp):
+ self._init_state(x)
+ # we pass in the weights implicitly so we unpack the GGUF on the fly
+ @function(precompile=True, allow_implicit=True)
+ def _run(x:Tensor, start_pos:int|UOp):
+ h = x + self._attention(self.attn_norm(x), start_pos)
+ return (h + self._feed_forward(self.ffn_norm(h))).contiguous()
+ return _run(x, start_pos)
+
+class TransformerBlock(FFNBlock):
+ def __init__(self, config:TransformerConfig):
+ super().__init__(config)
+ assert config.v_head_dim == config.head_dim, "TransformerBlock requires v_head_dim == head_dim"
+
+ # --- attention projections (all linear, bias-free) ------------------
+ q_proj_out = config.head_dim * config.n_heads * (2 if config.attn_output_gate else 1)
+ kv_proj_out = config.head_dim * config.n_kv_heads
+ self.attn_q = nn.Linear(config.dim, q_proj_out, bias=config.qkv_bias)
+ self.attn_k = nn.Linear(config.dim, kv_proj_out, bias=config.qkv_bias)
+ self.attn_v = nn.Linear(config.dim, kv_proj_out, bias=config.qkv_bias)
+ self.attn_output = nn.Linear(config.head_dim * config.n_heads, config.dim, bias=False)
+ if config.qk_norm: self.attn_q_norm, self.attn_k_norm = nn.RMSNorm(config.qk_norm, config.norm_eps), nn.RMSNorm(config.qk_norm, config.norm_eps)
+
+ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor:
+ q, k, v = self.attn_q(x), self.attn_k(x), self.attn_v(x)
+ if self.config.qk_norm and self.config.qk_norm != self.config.head_dim: q, k = self.attn_q_norm(q), self.attn_k_norm(k)
+
+ B, T, _ = x.shape
+ if self.config.attn_output_gate:
+ qg = q.reshape(B, T, self.config.n_heads, 2, self.config.head_dim)
+ q, gate = qg[:, :, :, 0, :], qg[:, :, :, 1, :].reshape(B, T, self.config.n_heads * self.config.head_dim)
+ q = q.reshape(B, T, self.config.n_heads, self.config.head_dim).transpose(1, 2) # (B,H,T,Hd)
+ k = k.reshape(B, T, self.config.n_kv_heads, self.config.head_dim).transpose(1, 2) # (B,KvH,T,Hd)
+ v = v.reshape(B, T, self.config.n_kv_heads, self.config.head_dim).transpose(1, 2) # (B,KvH,T,Hd)
+ if self.config.qk_norm == self.config.head_dim: q, k = self.attn_q_norm(q), self.attn_k_norm(k)
+
+ q = apply_rope(q[..., :self.config.rope_dim], self.freqs_cis[start_pos:start_pos+T]).cat(q[..., self.config.rope_dim:], dim=-1)
+ k = apply_rope(k[..., :self.config.rope_dim], self.freqs_cis[start_pos:start_pos+T]).cat(k[..., self.config.rope_dim:], dim=-1)
+
+ # NOTE: we don't want to change self.cache_kv, the function API doesn't support this well
+ assigned_kv = Tensor(self.cache_kv.uop.after(self.cache_kv[:, :, :, start_pos:start_pos+T, :].uop.store(Tensor.stack(k, v).uop)))
+ k = assigned_kv[0, :, :, 0:start_pos+T, :]
+ v = assigned_kv[1, :, :, 0:start_pos+T, :]
+
+ #self.cache_kv[:, :, :, start_pos:start_pos+T, :].assign(Tensor.stack(k, v))
+ #k = self.cache_kv[0, :, :, 0:start_pos+T, :]
+ #v = self.cache_kv[1, :, :, 0:start_pos+T, :]
+
+ # NOTE: this mask is causal_lower_right, not the causal_upper_left generated by is_casual = True
+ # TODO: this if statement should be removed and it shouldn't generate extra kernels
+ mask = Tensor.full((1, 1, T, start_pos+T), float("-inf"), dtype=x.dtype, device=x.device, buffer=False).triu(start_pos+1) \
+ if resolve(T != 1) else None
+ attn = q.scaled_dot_product_attention(k, v, attn_mask=mask, enable_gqa=True) # (B,H,T,Hd)
+ attn = attn.transpose(1, 2).reshape(B, T, -1) # back to (B,T,D)
+ return self.attn_output(attn if not self.config.attn_output_gate else (attn * gate.sigmoid()))
+
+ def _init_state(self, x:Tensor):
+ if not hasattr(self, "cache_kv"):
+ # TODO: how is the dtype of this determined?
+ self.cache_kv = Tensor.empty(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, device=x.device)
+ self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device)
+
+class MLATransformerBlock(FFNBlock):
+ def __init__(self, config:TransformerConfig):
+ super().__init__(config)
+ qk_nope_head_dim = config.head_dim - config.rope_dim
+ if config.q_lora_rank > 0:
+ self.attn_q_a = nn.Linear(config.dim, config.q_lora_rank, bias=False)
+ self.attn_q_a_norm = nn.RMSNorm(config.q_lora_rank, config.norm_eps)
+ self.attn_q_b = nn.Linear(config.q_lora_rank, config.n_heads * config.head_dim, bias=False)
+ else:
+ self.attn_q = nn.Linear(config.dim, config.n_heads * config.head_dim, bias=False)
+ self.attn_kv_a_mqa = nn.Linear(config.dim, config.kv_lora_rank + config.rope_dim, bias=False)
+ self.attn_kv_a_norm = nn.RMSNorm(config.kv_lora_rank, config.norm_eps)
+ self.attn_k_b = {"weight": Tensor.zeros(config.n_heads, config.kv_lora_rank, qk_nope_head_dim)}
+ self.attn_v_b = {"weight": Tensor.zeros(config.n_heads, config.v_head_dim, config.kv_lora_rank)}
+ self.attn_output = nn.Linear(config.n_heads * config.v_head_dim, config.dim, bias=False)
+
+ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor:
+ B, T, _ = x.shape
+ q_nope_head_dim = self.config.head_dim - self.config.rope_dim
+ q_proj = self.attn_q_b(self.attn_q_a_norm(self.attn_q_a(x))) if self.config.q_lora_rank > 0 else self.attn_q(x)
+ q = q_proj.reshape(B, T, self.config.n_heads, self.config.head_dim).transpose(1, 2)
+ q_nope, q_rope = q[..., :q_nope_head_dim], q[..., q_nope_head_dim:]
+ q = (q_nope @ self.attn_k_b["weight"].transpose(-1, -2)).cat(apply_rope(q_rope, self.freqs_cis[start_pos:start_pos+T]), dim=-1)
+
+ kv_a = self.attn_kv_a_mqa(x)
+ c_kv = self.attn_kv_a_norm(kv_a[..., :self.config.kv_lora_rank])
+ k_rope = apply_rope(
+ kv_a[..., self.config.kv_lora_rank:].reshape(B, T, 1, self.config.rope_dim).transpose(1, 2),
+ self.freqs_cis[start_pos:start_pos+T])
+
+ k_store = c_kv.reshape(B, 1, T, self.config.kv_lora_rank).cat(k_rope.reshape(B, 1, T, self.config.rope_dim), dim=-1)
+ k = Tensor(self.cache_k.uop.after(self.cache_k[:, :, start_pos:start_pos+T, :].uop.store(k_store.uop)))[:, :, 0:start_pos+T, :]
+ v = k[..., :self.config.kv_lora_rank]
+
+ mask = Tensor.full((1, 1, T, start_pos+T), float("-inf"), dtype=x.dtype, device=x.device, buffer=False).triu(start_pos+1) \
+ if resolve(T != 1) else None
+ attn = q @ k.transpose(-1, -2) * (1.0 / self.config.head_dim ** 0.5)
+ if mask is not None: attn = attn + mask
+ attn = attn.softmax(-1)
+ attn = ((attn @ v) @ self.attn_v_b["weight"].transpose(-1, -2)).transpose(1, 2).reshape(B, T, -1)
+ return self.attn_output(attn)
+
+ def _init_state(self, x:Tensor):
+ if not hasattr(self, "cache_k"):
+ self.cache_k = Tensor.empty(x.shape[0], 1, self.config.max_context, self.config.kv_lora_rank + self.config.rope_dim, device=x.device)
+ self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device)
+
+class GatedDeltaNetBlock(FFNBlock):
+ def __init__(self, config:TransformerConfig, ssm:SSMConfig):
+ super().__init__(config)
+ self.head_k_dim, self.num_k_heads, self.num_v_heads = ssm.state_size, ssm.group_count, ssm.time_step_rank
+ assert self.num_v_heads % self.num_k_heads == 0
+ self.head_v_dim, self.ssm_conv_kernel = ssm.inner_size // ssm.time_step_rank, ssm.conv_kernel
+ self.conv_channels, self.q_dim = ssm.inner_size + 2*ssm.group_count*ssm.state_size, ssm.state_size*ssm.group_count
+ self.attn_qkv, self.attn_gate = nn.Linear(config.dim, self.conv_channels, bias=False), nn.Linear(config.dim, ssm.inner_size, bias=False)
+ self.ssm_alpha, self.ssm_beta = nn.Linear(config.dim, self.num_v_heads, bias=False), nn.Linear(config.dim, self.num_v_heads, bias=False)
+ self.ssm_conv1d = {"weight": Tensor.zeros(self.conv_channels, self.ssm_conv_kernel)}
+ self.ssm_dt = {"bias": Tensor.zeros(self.num_v_heads)}
+ self.ssm_a = Tensor.zeros(self.num_v_heads)
+ self.ssm_norm, self.ssm_out = nn.RMSNorm(self.head_v_dim, config.norm_eps), nn.Linear(ssm.inner_size, config.dim, bias=False)
+
+ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor:
+ B, T, _ = x.shape
+ assert T == 1, "GatedDeltaNetBlock currently only supports T=1"
+
+ # input processing
+ x = x.half()
+ out_gate = self.attn_gate(x).reshape(B, 1, self.num_v_heads, self.head_v_dim)
+ beta = self.ssm_beta(x).sigmoid().reshape(B, self.num_v_heads, 1, 1)
+ alpha = ((self.ssm_alpha(x).float() + self.ssm_dt["bias"]).softplus() * self.ssm_a).reshape(B, self.num_v_heads, 1, 1).exp()
+
+ # qkv conv
+ conv_window = self.conv_state.cat(self.attn_qkv(x), dim=1)
+ conv_out = (conv_window * self.ssm_conv1d["weight"].T.unsqueeze(0)).sum(1).silu()
+ q, k, v = conv_out.split([self.q_dim, self.q_dim, self.conv_channels - 2*self.q_dim], dim=-1)
+ q = q.reshape(B, self.num_k_heads, self.head_k_dim).normalize(dim=-1).repeat(1, self.num_v_heads//self.num_k_heads, 1)
+ k = k.reshape(B, self.num_k_heads, self.head_k_dim).normalize(dim=-1).repeat(1, self.num_v_heads//self.num_k_heads, 1)
+ v = v.reshape(B, self.num_v_heads, self.head_v_dim)
+ q, k, v = q.mul(self.head_k_dim**-0.5).unsqueeze(-1), k.unsqueeze(-1), v.unsqueeze(-1)
+
+ # recurrent
+ recurrent_state = self.recurrent_state * alpha
+ recurrent_state = recurrent_state + ((v - recurrent_state@k) * beta)@k.transpose(-1, -2)
+
+ # store the updated state
+ conv_state_store = self.conv_state.uop.store(conv_window[:, 1:, :].cast(self.conv_state.dtype).uop)
+ recurrent_state_store = self.recurrent_state.uop.store(recurrent_state.cast(self.recurrent_state.dtype).uop)
+ recurrent_state = Tensor(self.recurrent_state.uop.after(recurrent_state_store, conv_state_store))
+
+ # output
+ core_attn_out = self.ssm_norm((recurrent_state@q).squeeze(-1).reshape(B, 1, self.num_v_heads, self.head_v_dim))
+ return self.ssm_out((core_attn_out * out_gate.silu()).reshape(B, 1, -1).cast(x.dtype))
+
+ # recurrent state can't be partially reused after divergence, force a full rebuild
+ def _state_reset_ops(self):
+ return [self.conv_state.assign(self.conv_state.const_like(0)),
+ self.recurrent_state.assign(self.recurrent_state.const_like(0))] if hasattr(self, "conv_state") else []
+ def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return 0 if prefix_len != cached_len else prefix_len
+
+ def _init_state(self, x):
+ if not hasattr(self, "conv_state"):
+ self.conv_state = Tensor.zeros(x.shape[0], self.ssm_conv_kernel-1, self.conv_channels, device=x.device).clone()
+ self.recurrent_state = Tensor.zeros(x.shape[0], self.num_v_heads, self.head_v_dim, self.head_v_dim, device=x.device).clone()
+
+class Transformer:
+ def __init__(self, config:TransformerConfig):
+ dense_config = replace(config, num_experts=0, num_experts_per_tok=0, shared_expert_dim=0, hidden_dim=config.dense_hidden_dim or config.hidden_dim)
+ if config.ssm: config = replace(config, qk_norm=config.head_dim)
+ block_cls = MLATransformerBlock if config.kv_lora_rank > 0 else TransformerBlock
+ self.blk:list[FFNBlock] = [GatedDeltaNetBlock(config, config.ssm) if config.ssm and (i+1) % config.full_attention_interval != 0 else
+ block_cls(dense_config if i < config.leading_dense_blocks else config) for i in range(config.num_blocks)]
+ self.token_embd = nn.Embedding(config.vocab_size, config.dim)
+ self.output_norm = nn.RMSNorm(config.dim, config.norm_eps)
+ self.output = nn.Linear(config.dim, config.vocab_size, bias=False)
+ self.max_context = config.max_context
+ self.has_recurrent_block = any(isinstance(b, GatedDeltaNetBlock) for b in self.blk)
+ self._cached_tokens: list[int] = []
+ # we specialize the JIT for prefill and rollout
+ self.prefill_jit = TinyJit(self.forward)
+ self.rollout_jit = TinyJit(self.forward)
+
+ def forward(self, tokens:Tensor, start_pos:int|UOp, temperature:Tensor) -> Tensor:
+ x = self.token_embd(tokens).float() # (B, T, D)
+ for block in self.blk: x = block(x, start_pos)
+ logits = self.output(self.output_norm(x))[:, -1, :]
+ # Gumbel-max trick: argmax(logits/temp - log(-log(uniform))) is equivalent to sampling from softmax(logits/temp)
+ return (logits / temperature.maximum(1e-12) - (Tensor.rand_like(logits).maximum(1e-12).log().neg()).log()).argmax(-1, keepdim=True)
+
+ def __call__(self, tokens:Tensor, start_pos:int|UOp, temperature:Tensor) -> Tensor:
+ return (self.prefill_jit if resolve(tokens.shape[1] != 1) else self.rollout_jit)(tokens.contiguous(), start_pos, temperature)
+
+ @staticmethod
+ def from_gguf(gguf:Tensor|str|pathlib.Path, max_context:int|None=None,
+ realize=bool(getenv("REALIZE", 0))) -> tuple[Transformer, dict]:
+ # TODO: remove the need for copy to default device
+ kv, state_dict = gguf_load(gguf.to(None).realize() if isinstance(gguf, Tensor) else gguf)
+
+ # all state items should be float16, not float32
+ state_dict = {k:v.cast('float16') if getenv("HALF", 1) else v for k,v in state_dict.items()}
+
+ # some models like Llama 3.2 don't have an output.weight, they just tie to the token_embd.weight
+ if 'output.weight' not in state_dict: state_dict['output.weight'] = state_dict['token_embd.weight']
+
+ arch = kv['general.architecture']
+ max_context = min(max_context, kv[f'{arch}.context_length']) if max_context is not None else kv[f'{arch}.context_length']
+ n_heads, n_kv_heads = kv[f'{arch}.attention.head_count'], kv[f'{arch}.attention.head_count_kv']
+
+ ssm = None
+ if arch in ('qwen35', 'qwen35moe'):
+ ssm = SSMConfig(**{k: kv[f'{arch}.ssm.{k}'] for k in ('conv_kernel','state_size','group_count','time_step_rank','inner_size')})
+ if arch in ('qwen35', 'qwen35moe', 'glm4moe'):
+ state_dict = {k.replace('post_attention_norm', 'ffn_norm'):v for k,v in state_dict.items()}
+
+ kv_lora_rank = kv.get(f'{arch}.attention.kv_lora_rank', 0)
+ head_dim = kv.get(f'{arch}.attention.key_length_mla', kv.get(f'{arch}.attention.key_length', kv[f'{arch}.embedding_length'] // n_heads))
+ rope_dim = kv.get(f'{arch}.rope.dimension_count', head_dim)
+
+ # Permute RoPE weights from interleaved to half-split layout.
+ for name in state_dict:
+ if ('attn_q.weight' in name or 'attn_q_b.weight' in name) and (arch == 'llama' or kv_lora_rank):
+ w = state_dict[name].reshape(n_heads, state_dict[name].shape[0]//n_heads, -1)
+ prefix = head_dim-rope_dim
+ state_dict[name] = w[:, :prefix].cat(w[:, prefix:].rearrange("n (h two) d -> n (two h) d", two=2), dim=1).reshape(-1, w.shape[-1])
+ elif arch == 'llama' and 'attn_k.weight' in name:
+ w = state_dict[name].reshape(n_kv_heads, state_dict[name].shape[0]//n_kv_heads, -1)
+ state_dict[name] = w.rearrange("n (h two) d -> n (two h) d", two=2).reshape(-1, w.shape[-1])
+ elif kv_lora_rank and 'attn_kv_a_mqa.weight' in name:
+ state_dict[name] = state_dict[name][:kv_lora_rank].cat(state_dict[name][kv_lora_rank:].rearrange("(h two) d -> (two h) d", two=2), dim=0)
+ config = TransformerConfig(
+ num_blocks=kv[f'{arch}.block_count'] - kv.get(f'{arch}.nextn_predict_layers', 0), dim=kv[f'{arch}.embedding_length'],
+ hidden_dim=kv.get(f'{arch}.expert_feed_forward_length', kv.get(f'{arch}.feed_forward_length', 0)),
+ n_heads=n_heads, n_kv_heads=n_kv_heads, norm_eps=kv[f'{arch}.attention.layer_norm_rms_epsilon'],
+ vocab_size=len(kv['tokenizer.ggml.tokens']),
+ head_dim=head_dim,
+ rope_theta=kv[f'{arch}.rope.freq_base'],
+ rope_dim=rope_dim,
+ v_head_dim=kv.get(f'{arch}.attention.value_length_mla', kv.get(f'{arch}.attention.value_length', head_dim)),
+ max_context=max_context,
+ qk_norm=int(state_dict['blk.0.attn_q_norm.weight'].shape[0]) if 'blk.0.attn_q_norm.weight' in state_dict else 0,
+ num_experts=kv.get(f'{arch}.expert_count', 0), num_experts_per_tok=kv.get(f'{arch}.expert_used_count', 0),
+ norm_topk_prob=kv.get(f'{arch}.expert_weights_norm', arch in ('qwen3moe', 'qwen35moe')),
+ kv_lora_rank=kv_lora_rank, q_lora_rank=kv.get(f'{arch}.attention.q_lora_rank', 0),
+ leading_dense_blocks=kv.get(f'{arch}.leading_dense_block_count', 0),
+ shared_expert_dim=kv.get(
+ f'{arch}.expert_shared_feed_forward_length',
+ kv.get(f'{arch}.expert_shared_count', 0) * kv.get(f'{arch}.expert_feed_forward_length', 0)),
+ shared_expert_gate=f"blk.{kv.get(f'{arch}.leading_dense_block_count', 0)}.ffn_gate_inp_shexp.weight" in state_dict,
+ dense_hidden_dim=kv.get(f'{arch}.feed_forward_length', 0) if kv.get(f'{arch}.leading_dense_block_count', 0) else 0,
+ routed_scaling_factor=kv.get(f'{arch}.expert_weights_scale', 1.0), attn_output_gate=arch in ('qwen35', 'qwen35moe'), ssm=ssm,
+ full_attention_interval=kv.get(f'{arch}.full_attention_interval', 0),
+ qkv_bias='blk.0.attn_q.bias' in state_dict,
+ expert_bias=f"blk.{kv.get(f'{arch}.leading_dense_block_count', 0)}.exp_probs_b.bias" in state_dict)
+ model = Transformer(config)
+ nn.state.load_state_dict(model, state_dict, verbose=False, consume=True, realize=False) # NOTE: rope_freqs.weight (32,) is unused
+ # NOTE: without this contiguous, it unpacks the weights from the model every time. we shouldn't need this, but for now it's faster
+ if realize:
+ for s in (params:=nn.state.get_parameters(model)): s.replace(s.contiguous())
+ Tensor.realize(*params)
+ return model, kv
+
+ def get_start_pos(self, tokens:list[int]) -> int:
+ prefix_len = sum(1 for _ in itertools.takewhile(lambda ab: ab[0] == ab[1], zip(tokens[:-1], self._cached_tokens)))
+ return min(block._reusable_prefix_len(prefix_len, len(self._cached_tokens)) for block in self.blk)
+
+ def generate(self, tokens:list[int], chunk_size:int=32, temperature:float=0.0):
+ if self.has_recurrent_block: chunk_size = 1
+ v_start_pos = UOp.variable("start_pos", 0, self.max_context-1)
+ v_toks = UOp.variable("toks", 1, chunk_size)
+ # TODO: use UOp.variable for temperature once float variables are supported
+ temp = Tensor(temperature).contiguous()
+ # assign all input tokens once, then slice from start_pos for the model call
+ t = Tensor(tokens + [0] * (self.max_context - len(tokens)), dtype="int32").reshape(1, self.max_context)
+ # recompute start_pos from what's currently valid in the caches
+ start_pos = self.get_start_pos(tokens)
+ if start_pos < len(self._cached_tokens) and (resets := [r for b in self.blk for r in b._state_reset_ops()]): Tensor.realize(*resets)
+ out, prompt_len = None, len(tokens)
+ while len(tokens) < self.max_context:
+ sp, nt = v_start_pos.bind(start_pos), v_toks.bind(min(chunk_size, len(tokens) - start_pos))
+ out = self(t[:, sp:sp+nt] if start_pos < prompt_len or out is None else out, sp, temp).realize()
+ start_pos += nt.val
+ # chunked prefill: keep processing until all prompt tokens are consumed
+ if start_pos < len(tokens): continue
+ tokens.append(int(out.item()))
+ self._cached_tokens = tokens[:-1]
+ yield tokens[-1]