Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Refactor project structure and update dependencies

HenryNdubuaku43f10729d818
diff --git a/.gitignore b/.gitignore
index 1afe2fe..262dd0f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -91,4 +91,7 @@ wandb/
tokenizer/
data/
voice_samples/
-needle-ui.*
\ No newline at end of file
+needle-ui.*
+*.vocab
+*.model
+*.cact
\ No newline at end of file
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..8f5eacc
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,3 @@
+include README.md LICENSE
+recursive-include needle/model *.model *.vocab
+include needle/agent/index.html
diff --git a/README.md b/README.md
index ec1dcfe..937380d 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ Needle 2 is an open 45M-parameter model for tool calling, device use and structu
- **Tool retrieval**: declare a large catalogue and a built-in retrieval head renders only the top five tools per turn, with the grammar constrained to that subset.
- **Bounded memory**: a 256-token sliding window with the tools pinned as KV sinks, so total memory stays near 28MB no matter how long the conversation runs.
-Weights on: [github.com/cactus-compute/needle](Cactus-Compute/needle2).
+Weights: [huggingface.co/Cactus-Compute/needle2](https://huggingface.co/Cactus-Compute/needle2) · source: [github.com/cactus-compute/needle](https://github.com/cactus-compute/needle).
![Size-quality frontier: mobile-class and below](assets/frontier.png)
@@ -243,7 +243,28 @@ The `confidence` field is the minimum of two signals: a calibrated post-hoc head
## Fine-tuning
-Needle is open and trainable end to end. Fine-tune it on your own tools and domains, then export to a `.cact` and ship it like the base model. See the [needle repo](https://github.com/cactus-compute/needle) for training and export.
+Needle fine-tunes with LoRA on the frozen base and merges the adapter at export, so a run is cheap and the adapter is tiny. Your data is a JSONL of tool-call examples, one per line: `{"query": ..., "tools": [...], "answers": [{"name": ..., "arguments": {...}}], "reasoning": "..."}`.
+
+```sh
+# optionally synthesize training data first (needs OPENROUTER_API_KEY)
+needle generate-data --tools my_tools.json --num-samples 500 --output data.jsonl
+
+# LoRA fine-tune; --generate N expands your set with more examples before training
+needle finetune data.jsonl --epochs 3 --generate 300
+
+# merge the adapter into the base and export a tuned .cact
+needle build checkpoints/needle2.pkl --lora checkpoints/needle_lora.pkl --out my_needle.cact
+```
+
+The engine binary is downloaded from Hugging Face and is weights-agnostic, so a tuned `.cact` runs on it directly - no recompilation:
+
+```python
+import needle
+agent = needle.Needle(weights="my_needle.cact", tools=[...])
+agent.run("...")
+```
+
+Set `NEEDLE_HF_REPO=<you>/<model>` and pass `--upload` to `needle build` to publish your tuned `.cact`. Try any model in the browser playground with `needle playground` (http://127.0.0.1:7860).
## Extraction
diff --git a/launch_train.sh b/launch_train.sh
deleted file mode 100644
index 2e7ab7c..0000000
--- a/launch_train.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-# Launch multi-host training on all TPU workers.
-# Run this from your LOCAL machine (not from the TPU VM).
-#
-# Usage: ./launch_train.sh [extra needle train args...]
-# Example: ./launch_train.sh --epochs 3 --wandb --batch-size 64
-
-set -euo pipefail
-
-TPU_NAME="${TPU_NAME:-large}"
-ZONE="${ZONE:-asia-northeast1-b}"
-PROJECT="${PROJECT:-$(gcloud config get-value project 2>/dev/null)}"
-
-if [ -z "$PROJECT" ] || [ "$PROJECT" = "(unset)" ]; then
- echo "[launch] ERROR: no GCP project set. Run 'gcloud config set project <PROJECT>' or export PROJECT=..."
- exit 1
-fi
-
-TRAIN_ARGS="${@}"
-
-echo "[launch] Distributing SSH keys across workers..."
-# Generate keypair on worker 0 if not present, then distribute to all workers
-gcloud compute tpus tpu-vm ssh "$TPU_NAME" \
- --zone="$ZONE" --project="$PROJECT" --worker=0 \
- --command='[ -f ~/.ssh/id_rsa ] || ssh-keygen -t rsa -b 2048 -f ~/.ssh/id_rsa -N "" -q; cat ~/.ssh/id_rsa.pub' \
- 2>/dev/null | tail -1 > /tmp/_tpu_pubkey.txt
-
-# Add worker 0's public key to all workers' authorized_keys
-PUBKEY=$(cat /tmp/_tpu_pubkey.txt)
-gcloud compute tpus tpu-vm ssh "$TPU_NAME" \
- --zone="$ZONE" --project="$PROJECT" --worker=all \
- --command="grep -qF '$(echo $PUBKEY | cut -d' ' -f2)' ~/.ssh/authorized_keys 2>/dev/null || echo '$PUBKEY' >> ~/.ssh/authorized_keys"
-
-echo "[launch] Starting training on all workers..."
-gcloud compute tpus tpu-vm ssh "$TPU_NAME" \
- --zone="$ZONE" --project="$PROJECT" --worker=all \
- --command="cd ~/needle && .venv/bin/needle train $TRAIN_ARGS" \
- 2>&1
diff --git a/needle/__init__.py b/needle/__init__.py
index f582872..22dffed 100644
--- a/needle/__init__.py
+++ b/needle/__init__.py
@@ -1,25 +1,124 @@
-"""Needle - a 26M parameter function calling model."""
-
-from needle.model.architecture import (
- SimpleAttentionNetwork,
- TransformerConfig,
-)
-from needle.model.run import (
- generate,
- generate_batch,
- load_checkpoint,
- encode_for_retrieval,
- retrieve_tools,
-)
-from needle.dataset.dataset import get_tokenizer
-
-__all__ = [
- "SimpleAttentionNetwork",
- "TransformerConfig",
- "generate",
- "generate_batch",
- "load_checkpoint",
- "encode_for_retrieval",
- "retrieve_tools",
- "get_tokenizer",
-]
+import ctypes
+import json
+import os
+import sys
+
+from .agent.tools import Field, build_schema, pydantic_schema, tool, _is_pydantic_model
+
+__version__ = "2.0.0"
+__all__ = ["Needle", "tool", "Field", "extract", "__version__"]
+
+
+def _library_path():
+ from .agent import fetch
+
+ here = os.path.dirname(os.path.abspath(__file__))
+ local = os.path.join(here, fetch._lib_name())
+ if os.path.exists(local):
+ return local
+ cache = os.path.join(os.path.expanduser("~"), ".cache", "cactus-needle", __version__)
+ cached = os.path.join(cache, fetch._lib_name())
+ if os.path.exists(cached):
+ return cached
+ os.makedirs(cache, exist_ok=True)
+ return fetch.fetch_library(__version__, cache)
+
+
+_lib_handle = None
+
+
+def _lib():
+ global _lib_handle
+ if _lib_handle is None:
+ lib = ctypes.CDLL(_library_path())
+ lib.needle_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]
+ lib.needle_init.restype = ctypes.c_int
+ lib.needle_complete.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
+ lib.needle_complete.restype = ctypes.c_int
+ lib.needle_reset.argtypes = []
+ lib.needle_reset.restype = None
+ lib.needle_load.argtypes = [ctypes.c_char_p, ctypes.c_uint64]
+ lib.needle_load.restype = ctypes.c_int
+ _lib_handle = lib
+ return _lib_handle
+
+
+class Needle:
+ def __init__(self, tools=None, system=None, weights=None, tool_index_path=None, buffer_size=65536):
+ self._functions = {}
+ if weights:
+ with open(weights, "rb") as handle:
+ blob = handle.read()
+ if _lib().needle_load(blob, len(blob)) != 0:
+ raise RuntimeError(f"failed to load weights from {weights}")
+ tools_json = tools if isinstance(tools, str) else json.dumps(self._resolve(tools))
+ if _lib().needle_init((system or "").encode("utf-8"), tools_json.encode("utf-8"),
+ tool_index_path.encode("utf-8") if tool_index_path else None) < 0:
+ raise RuntimeError("needle_init failed")
+ self._buffer = ctypes.create_string_buffer(buffer_size)
+
+ def _resolve(self, tools):
+ schemas = []
+ for entry in tools or []:
+ if _is_pydantic_model(entry):
+ schema = pydantic_schema(entry)
+ self._functions[schema["name"]] = entry
+ schemas.append(schema)
+ elif callable(entry):
+ schema = getattr(entry, "_needle_tool", None) or build_schema(entry)
+ self._functions[schema["name"]] = entry
+ schemas.append(schema)
+ elif isinstance(entry, dict):
+ schemas.append(entry)
+ return schemas
+
+ def complete(self, text, max_new_tokens=256):
+ _lib().needle_complete(text.encode("utf-8"), int(max_new_tokens),
+ self._buffer, len(self._buffer))
+ return json.loads(self._buffer.value.decode("utf-8"))
+
+ def run(self, query, max_steps=8, max_new_tokens=256):
+ response = self.complete(query, max_new_tokens)
+ for _ in range(max_steps):
+ calls = response.get("function_calls") or []
+ if response.get("type") != "call" or not calls:
+ return response
+ results = []
+ for call in calls:
+ fn = self._functions.get(call.get("name"))
+ if fn is None:
+ results.append({"error": "unknown tool: " + str(call.get("name"))})
+ continue
+ try:
+ results.append(fn(**(call.get("arguments") or {})))
+ except Exception as exc:
+ results.append({"error": str(exc)})
+ response = self.complete(json.dumps(results, default=_jsonable), max_new_tokens)
+ return response
+
+ def extract(self, text, schema, max_new_tokens=256):
+ return extract(text, schema, max_new_tokens=max_new_tokens)
+
+ def reset(self):
+ _lib().needle_reset()
+
+
+def _jsonable(value):
+ if hasattr(value, "model_dump"):
+ return value.model_dump()
+ if hasattr(value, "dict") and _is_pydantic_model(type(value)):
+ return value.dict()
+ return str(value)
+
+
+def extract(text, schema, system=None, max_new_tokens=256):
+ """One-shot structured extraction: declare `schema` as the only tool and return
+ the parsed object (a Pydantic instance if `schema` is a model, else a dict).
+ Re-initializes the shared engine with this single schema."""
+ agent = Needle(tools=[schema], system=system)
+ response = agent.complete(text, max_new_tokens)
+ calls = response.get("function_calls") or []
+ if not calls:
+ return None
+ arguments = calls[0].get("arguments") or {}
+ return schema(**arguments) if _is_pydantic_model(schema) else arguments
diff --git a/needle/ui/__init__.py b/needle/agent/__init__.py
similarity index 100%
rename from needle/ui/__init__.py
rename to needle/agent/__init__.py
diff --git a/needle/agent/fetch.py b/needle/agent/fetch.py
new file mode 100644
index 0000000..d1c5957
--- /dev/null
+++ b/needle/agent/fetch.py
@@ -0,0 +1,54 @@
+import os
+import platform
+import sys
+import zipfile
+
+HF_REPO = "Cactus-Compute/needle2"
+
+
+def _lib_name():
+ if sys.platform == "darwin":
+ return "libneedle.dylib"
+ if sys.platform == "win32":
+ return "libneedle.dll"
+ return "libneedle.so"
+
+
+def _is_musl():
+ if sys.platform != "linux":
+ return False
+ libc, _ = platform.libc_ver()
+ if libc:
+ return False
+ try:
+ with open("/proc/self/maps", "rb") as maps:
+ return b"musl" in maps.read()
+ except OSError:
+ return True
+
+
+def _platform_tag():
+ machine = platform.machine().lower()
+ if sys.platform == "darwin":
+ arch = "arm64" if machine in ("arm64", "aarch64") else "x86_64"
+ return "macosx_11_0_" + arch
+ if sys.platform == "win32":
+ return "win_arm64" if machine in ("arm64", "aarch64") else "win_amd64"
+ arch = "aarch64" if machine in ("aarch64", "arm64") else "x86_64"
+ family = "musllinux_1_2_" if _is_musl() else "manylinux2014_"
+ return family + arch
+
+
+def fetch_library(version, dest_dir):
+ from huggingface_hub import hf_hub_download
+
+ tag = _platform_tag()
+ wheel = "cactus_needle-{}-py3-none-{}.whl".format(version, tag)
+ path = hf_hub_download(repo_id=HF_REPO, filename="python/" + wheel, repo_type="model")
+ lib = _lib_name()
+ with zipfile.ZipFile(path) as archive:
+ data = archive.read("needle/" + lib)
+ out = os.path.join(dest_dir, lib)
+ with open(out, "wb") as handle:
+ handle.write(data)
+ return out
diff --git a/needle/agent/index.html b/needle/agent/index.html
new file mode 100644
index 0000000..7affee3
--- /dev/null
+++ b/needle/agent/index.html
@@ -0,0 +1,94 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Needle 2 playground</title>
+<style>
+ :root { --bg:#0d0d0f; --panel:#161619; --line:#26262b; --fg:#e7e7ea; --dim:#8a8a92; --red:#ff5a36; }
+ * { box-sizing: border-box; }
+ body { margin:0; background:var(--bg); color:var(--fg); font:14px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace; }
+ header { padding:20px 24px; border-bottom:1px solid var(--line); }
+ header b { color:var(--red); }
+ .wrap { display:grid; grid-template-columns:1fr 1fr; gap:0; height:calc(100vh - 62px); }
+ .col { padding:20px 24px; overflow:auto; }
+ .col.left { border-right:1px solid var(--line); }
+ .lbl { color:var(--dim); text-transform:uppercase; font-size:11px; letter-spacing:.08em; margin:0 0 8px; }
+ textarea, input { width:100%; background:var(--panel); color:var(--fg); border:1px solid var(--line);
+ border-radius:8px; padding:12px; font:inherit; resize:vertical; }
+ textarea { height:340px; }
+ #query { margin:14px 0; }
+ button { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:999px;
+ padding:6px 12px; margin:0 6px 8px 0; cursor:pointer; font:inherit; }
+ button:hover { border-color:var(--red); }
+ #run { background:var(--red); border-color:var(--red); color:#fff; padding:8px 22px; border-radius:8px; }
+ pre { background:var(--panel); border:1px solid var(--line); border-radius:8px; padding:14px; white-space:pre-wrap;
+ word-break:break-word; margin:0; }
+ .meta { color:var(--dim); margin-top:10px; font-size:12px; }
+</style>
+</head>
+<body>
+<header>Play with <b>Needle 2</b> right here. Pick a preset, edit the tools or prompt, and Run.</header>
+<div class="wrap">
+ <div class="col left">
+ <p class="lbl">Tools JSON</p>
+ <textarea id="tools"></textarea>
+ <div id="presets" style="margin-top:14px"></div>
+ </div>
+ <div class="col">
+ <p class="lbl">Query</p>
+ <input id="query" placeholder="what do you want Needle to do?">
+ <button id="run">Run</button>
+ <p class="lbl" style="margin-top:22px">Result</p>
+ <pre id="out">-</pre>
+ <div class="meta" id="meta"></div>
+ </div>
+</div>
+<script>
+var PRESETS = {"home": {"tools": [{"name": "set_lights", "description": "Turn lights on or off or dim them.", "parameters": {"type": "object", "properties": {"room": {"type": "string", "description": "Which room."}, "state": {"type": "string", "enum": ["on", "off"]}, "brightness": {"type": "integer", "description": "Percent 1-100."}}, "required": ["room", "state"]}}, {"name": "set_thermostat", "description": "Set the target temperature.", "parameters": {"type": "object", "properties": {"room": {"type": "string"}, "temperature": {"type": "number"}, "mode": {"type": "string", "enum": ["heat", "cool", "auto"]}}, "required": ["room", "temperature"]}}, {"name": "lock_door", "description": "Lock a door.", "parameters": {"type": "object", "properties": {"door": {"type": "string", "description": "Which door."}}, "required": ["door"]}}], "q": "dim the bedroom lights to 20 percent and lock the front door"}, "robot": {"tools": [{"name": "move", "description": "Drive the robot in a direction.", "parameters": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["forward", "backward", "left", "right"]}, "distance_m": {"type": "number", "description": "Distance in meters."}}, "required": ["direction", "distance_m"]}}, {"name": "rotate", "description": "Rotate the robot in place.", "parameters": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["left", "right"]}, "degrees": {"type": "number"}}, "required": ["direction", "degrees"]}}, {"name": "gripper", "description": "Open or close the gripper.", "parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["open", "close"]}}, "required": ["action"]}}], "q": "move forward 2 meters, turn left 90 degrees, then close the gripper"}, "device": {"tools": [{"name": "open_website", "description": "Open a website in a new tab.", "parameters": {"type": "object", "properties": {"url": {"type": "string", "description": "The site to open."}}, "required": ["url"]}}, {"name": "set_theme", "description": "Change the page accent color.", "parameters": {"type": "object", "properties": {"accent_color": {"type": "string", "description": "A color name or hex."}}, "required": ["accent_color"]}}, {"name": "speak", "description": "Say something out loud through the device speakers.", "parameters": {"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]}}], "q": "make the accent green, open github.com, and say hello"}, "gallery": {"tools": [{"name": "create_album", "description": "Create a new photo album.", "parameters": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}}, {"name": "move_photos", "description": "Move photos into an album.", "parameters": {"type": "object", "properties": {"album": {"type": "string"}, "filter": {"type": "string", "description": "Which photos, e.g. 'last weekend', 'screenshots'."}}, "required": ["album", "filter"]}}, {"name": "delete_photos", "description": "Delete photos.", "parameters": {"type": "object", "properties": {"filter": {"type": "string"}}, "required": ["filter"]}}], "q": "create an album called Summer 2026 and move the photos from last weekend into it"}, "email": {"tools": [{"name": "send_email", "description": "Send an email.", "parameters": {"type": "object", "properties": {"to": {"type": "string", "description": "Email address."}, "subject": {"type": "string"}, "body": {"type": "string"}}, "required": ["to", "subject", "body"]}}], "q": "send the receipt to [email protected] with subject expenses: Blue Bottle Coffee, $14.50, August 4th"}, "currency": {"tools": [{"name": "convert_currency", "description": "Convert an amount between currencies.", "parameters": {"type": "object", "properties": {"amount": {"type": "number"}, "from_currency": {"type": "string", "description": "ISO code, e.g. USD."}, "to_currency": {"type": "string", "description": "ISO code, e.g. EUR."}}, "required": ["amount", "from_currency", "to_currency"]}}], "q": "how much is 250 dollars in euros"}, "document": {"tools": [{"name": "record_booking", "description": "Record the details of a hotel booking.", "parameters": {"type": "object", "properties": {"hotel": {"type": "string"}, "confirmation_number": {"type": "string"}, "guest_name": {"type": "string"}, "room_type": {"type": "string"}, "check_in": {"type": "string"}, "check_out": {"type": "string"}, "total": {"type": "number"}, "currency": {"type": "string"}}, "required": ["confirmation_number", "check_in", "check_out", "total"]}}], "q": "extract the booking from this email: Dear Mr Okafor, thank you for choosing the Harbor Light Hotel. This email confirms reservation HL-88213 for a deluxe sea-view room, checking in on 14 September 2026 and checking out on 18 September 2026. The total for your stay is 642.50 euros, payable at checkout. Breakfast is included from 7am, and our shuttle meets the 9:40 ferry on request. We look forward to welcoming you. Warm regards, Reception."}, "sentiment": {"tools": [{"name": "classify_sentiment", "description": "Classify the sentiment of a message.", "parameters": {"type": "object", "properties": {"sentiment": {"type": "string", "enum": ["positive", "negative", "neutral", "mixed"]}}, "required": ["sentiment"]}}], "q": "classify the sentiment of this message: this is the worst purchase I have ever made, it broke in a day and support ignored me"}, "crowded": {"tools": [{"name": "set_lights", "description": "Control lights.", "parameters": {"type": "object", "properties": {"room": {"type": "string"}, "state": {"type": "string", "enum": ["on", "off"]}}, "required": ["room", "state"]}}, {"name": "set_thermostat", "description": "Set temperature.", "parameters": {"type": "object", "properties": {"temperature": {"type": "number"}}, "required": ["temperature"]}}, {"name": "lock_door", "description": "Lock a door.", "parameters": {"type": "object", "properties": {"door": {"type": "string"}}, "required": ["door"]}}, {"name": "play_music", "description": "Play music.", "parameters": {"type": "object", "properties": {"genre": {"type": "string"}}, "required": []}}, {"name": "set_timer", "description": "Set a timer.", "parameters": {"type": "object", "properties": {"time_human": {"type": "string"}}, "required": ["time_human"]}}, {"name": "send_message", "description": "Send a text.", "parameters": {"type": "object", "properties": {"recipient": {"type": "string"}, "message": {"type": "string"}}, "required": ["recipient", "message"]}}, {"name": "create_note", "description": "Create a note.", "parameters": {"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]}}, {"name": "start_vacuum", "description": "Start the robot vacuum.", "parameters": {"type": "object", "properties": {"room": {"type": "string"}}, "required": []}}, {"name": "water_plants", "description": "Run the irrigation system.", "parameters": {"type": "object", "properties": {"zone": {"type": "string"}}, "required": []}}, {"name": "open_blinds", "description": "Open or close the blinds.", "parameters": {"type": "object", "properties": {"room": {"type": "string"}, "position": {"type": "string", "enum": ["open", "closed", "half"]}}, "required": ["room", "position"]}}, {"name": "preheat_oven", "description": "Preheat the oven.", "parameters": {"type": "object", "properties": {"temperature_c": {"type": "number"}}, "required": ["temperature_c"]}}, {"name": "charge_car", "description": "Start charging the car.", "parameters": {"type": "object", "properties": {"limit_percent": {"type": "integer"}}, "required": []}}], "q": "close the bedroom blinds halfway"}, "repeat": {"tools": [{"name": "add_to_list", "description": "Add one item to the shopping list.", "parameters": {"type": "object", "properties": {"item": {"type": "string"}, "quantity": {"type": "integer"}}, "required": ["item"]}}], "q": "add 2 milk and 6 eggs to the shopping list"}, "array": {"tools": [{"name": "add_items", "description": "Add items to the shopping list.", "parameters": {"type": "object", "properties": {"items": {"type": "array", "items": {"type": "string"}}}, "required": ["items"]}}], "q": "add milk, eggs, and bread to the shopping list"}, "flight": {"tools": [{"name": "search_flights", "description": "Search for flights.", "parameters": {"type": "object", "properties": {"from_city": {"type": "string"}, "to_city": {"type": "string"}, "date": {"type": "string"}, "direct_only": {"type": "boolean", "description": "Only non-stop flights."}, "passengers": {"type": "integer"}}, "required": ["from_city", "to_city"]}}], "q": "find non-stop flights from lagos to nairobi on december 3rd for two passengers"}, "refuse": {"tools": [{"name": "move", "description": "Drive the robot in a direction.", "parameters": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["forward", "backward", "left", "right"]}, "distance_m": {"type": "number", "description": "Distance in meters."}}, "required": ["direction", "distance_m"]}}, {"name": "rotate", "description": "Rotate the robot in place.", "parameters": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["left", "right"]}, "degrees": {"type": "number"}}, "required": ["direction", "degrees"]}}, {"name": "gripper", "description": "Open or close the gripper.", "parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["open", "close"]}}, "required": ["action"]}}], "q": "write me a short poem about the moon"}};
+var LABELS = {"home": "Smart home", "robot": "Robot", "gallery": "Gallery", "device": "Device control", "email": "Extract email", "currency": "Currency", "document": "Document extraction", "sentiment": "Sentiment", "crowded": "12 tools routing", "repeat": "Repeated calls", "array": "Array argument", "flight": "Flight form", "refuse": "Off-topic refusal"};
+var toolsBox = document.getElementById("tools");
+var queryBox = document.getElementById("query");
+var out = document.getElementById("out");
+var meta = document.getElementById("meta");
+
+function load(key) {
+ var p = PRESETS[key];
+ toolsBox.value = JSON.stringify(p.tools, null, 2);
+ queryBox.value = p.q;
+}
+var box = document.getElementById("presets");
+Object.keys(LABELS).forEach(function (key) {
+ if (!PRESETS[key]) return;
+ var b = document.createElement("button");
+ b.textContent = LABELS[key];
+ b.onclick = function () { load(key); };
+ box.appendChild(b);
+});
+
+document.getElementById("run").onclick = async function () {
+ out.textContent = "running...";
+ meta.textContent = "";
+ try {
+ var tools = JSON.parse(toolsBox.value);
+ var res = await fetch("/complete", {
+ method: "POST", headers: {"Content-Type": "application/json"},
+ body: JSON.stringify({tools: tools, query: queryBox.value}),
+ });
+ var d = await res.json();
+ out.textContent = JSON.stringify(d.function_calls || d, null, 2);
+ if (d.confidence !== undefined) {
+ meta.textContent = "confidence " + d.confidence.toFixed(4)
+ + " \u00b7 " + (d.decode_tps || 0).toFixed(0) + " tok/s"
+ + " \u00b7 " + (d.peak_ram_mb || 0).toFixed(1) + " MB"
+ + (d.reasoning ? " \u2014 " + d.reasoning : "");
+ }
+ } catch (e) {
+ out.textContent = "error: " + e;
+ }
+};
+
+load("home");
+</script>
+</body>
+</html>
diff --git a/needle/agent/server.py b/needle/agent/server.py
new file mode 100644
index 0000000..cb49986
--- /dev/null
+++ b/needle/agent/server.py
@@ -0,0 +1,56 @@
+import json
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+_INDEX = Path(__file__).parent / "index.html"
+_CACHE = {}
+
+
+def _agent(tools_json):
+ from .. import Needle
+ if tools_json not in _CACHE:
+ _CACHE.clear()
+ _CACHE[tools_json] = Needle(tools=tools_json)
+ return _CACHE[tools_json]
+
+
+class _Handler(BaseHTTPRequestHandler):
+ def _send(self, code, body, ctype="application/json"):
+ data = body if isinstance(body, bytes) else body.encode("utf-8")
+ self.send_response(code)
+ self.send_header("Content-Type", ctype)
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def do_GET(self):
+ if self.path.split("?")[0] in ("/", "/index.html"):
+ self._send(200, _INDEX.read_bytes(), "text/html")
+ else:
+ self._send(404, b"not found", "text/plain")
+
+ def do_POST(self):
+ if self.path != "/complete":
+ self._send(404, b"not found", "text/plain")
+ return
+ length = int(self.headers.get("Content-Length", 0))
+ try:
+ request = json.loads(self.rfile.read(length) or b"{}")
+ tools = request.get("tools", [])
+ tools_json = tools if isinstance(tools, str) else json.dumps(tools)
+ result = _agent(tools_json).complete(request.get("query", ""))
+ self._send(200, json.dumps(result))
+ except Exception as exc:
+ self._send(500, json.dumps({"error": str(exc)}))
+
+ def log_message(self, *args):
+ pass
+
+
+def main(args):
+ server = ThreadingHTTPServer((args.host, args.port), _Handler)
+ print(f"needle playground: http://{args.host}:{args.port}")
+ try:
+ server.serve_forever()
+ except KeyboardInterrupt:
+ server.shutdown()
diff --git a/needle/agent/tools.py b/needle/agent/tools.py
new file mode 100644
index 0000000..c2446ef
--- /dev/null
+++ b/needle/agent/tools.py
@@ -0,0 +1,159 @@
+import enum
+import inspect
+import re
+import typing
+
+_JSON_TYPES = {str: "string", int: "integer", float: "number", bool: "boolean",
+ list: "array", dict: "object"}
+
+_MISSING = object()
+
+
+class Field:
+ def __init__(self, default=_MISSING, *, description=None, enum=None, const=_MISSING,
+ ge=None, le=None, gt=None, lt=None, multiple_of=None,
+ min_length=None, max_length=None, pattern=None, format=None,
+ min_items=None, max_items=None, unique_items=None):
+ self.default = default
+ self.description = description
+ self.enum = enum
+ self.const = const
+ self.ge, self.le, self.gt, self.lt = ge, le, gt, lt
+ self.multiple_of = multiple_of
+ self.min_length, self.max_length = min_length, max_length
+ self.pattern, self.format = pattern, format
+ self.min_items, self.max_items, self.unique_items = min_items, max_items, unique_items
+
+ def has_default(self):
+ return self.default is not _MISSING
+
+ def apply(self, schema):
+ pairs = (("description", self.description), ("enum", self.enum),
+ ("minimum", self.ge), ("maximum", self.le),
+ ("exclusiveMinimum", self.gt), ("exclusiveMaximum", self.lt),
+ ("multipleOf", self.multiple_of), ("minLength", self.min_length),
+ ("maxLength", self.max_length), ("pattern", self.pattern),
+ ("format", self.format), ("minItems", self.min_items),
+ ("maxItems", self.max_items), ("uniqueItems", self.unique_items))
+ for key, value in pairs:
+ if value is not None:
+ schema[key] = list(value) if key == "enum" else value
+ if self.const is not _MISSING:
+ schema["const"] = self.const
+ return schema
+
+
+def _is_optional(annotation):
+ return (typing.get_origin(annotation) is typing.Union
+ and type(None) in typing.get_args(annotation))
+
+
+def _json_type(annotation):
+ if annotation is inspect.Parameter.empty or annotation is None:
+ return {"type": "string"}
+ origin = typing.get_origin(annotation)
+ if origin is typing.Annotated:
+ return _json_type(typing.get_args(annotation)[0])
+ if origin is None:
+ if annotation in _JSON_TYPES:
+ return {"type": _JSON_TYPES[annotation]}
+ if isinstance(annotation, type) and issubclass(annotation, enum.Enum):
+ return {"type": "string", "enum": [e.value for e in annotation]}
+ if _is_pydantic_model(annotation):
+ return pydantic_schema(annotation)["parameters"]
+ return {"type": "string"}
+ if origin is typing.Literal:
+ return {"type": "string", "enum": list(typing.get_args(annotation))}
+ if origin in (list, typing.List):
+ args = typing.get_args(annotation)
+ return {"type": "array", "items": _json_type(args[0]) if args else {"type": "string"}}
+ if origin in (dict, typing.Dict):
+ return {"type": "object"}
+ if origin is typing.Union:
+ rest = [a for a in typing.get_args(annotation) if a is not type(None)]
+ if rest:
+ return _json_type(rest[0])
+ return {"type": "string"}
+
+
+def _field_of(annotation, default):
+ if isinstance(default, Field):
+ return default
+ if typing.get_origin(annotation) is typing.Annotated:
+ for meta in typing.get_args(annotation)[1:]:
+ if isinstance(meta, Field):
+ return meta
+ return None
+
+
+def _parse_doc(doc):
+ if not doc:
+ return "", {}
+ lines = doc.strip("\n").splitlines()
+ heads = ("args:", "arguments:", "parameters:", "params:")
+ desc, args, i = [], {}, 0
+ while i < len(lines) and lines[i].strip().lower() not in heads:
+ desc.append(lines[i].strip())
+ i += 1
+ for line in lines[i + 1:]:
+ m = re.match(r"\s+(\w+)\s*(?:\([^)]*\))?\s*:\s*(.+)", line)
+ if m:
+ args[m.group(1)] = m.group(2).strip()
+ return " ".join(w for w in desc if w).strip(), args
+
+
+def build_schema(fn):
+ signature = inspect.signature(fn)
+ try:
+ hints = typing.get_type_hints(fn, include_extras=True)
+ except Exception:
+ hints = {}
+ description, arg_docs = _parse_doc(fn.__doc__)
+ properties, required = {}, []
+ for name, param in signature.parameters.items():
+ if name in ("self", "cls") or param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
+ continue
+ annotation = hints.get(name, param.annotation)
+ schema = _json_type(annotation)
+ if name in arg_docs and "description" not in schema:
+ schema["description"] = arg_docs[name]
+ field = _field_of(annotation, param.default)
+ if field:
+ field.apply(schema)
+ properties[name] = schema
+ has_default = param.default is not param.empty and not isinstance(param.default, Field)
+ if field and field.has_default():
+ has_default = True
+ if not has_default and not _is_optional(annotation):
+ required.append(name)
+ parameters = {"type": "object", "properties": properties}
+ if required:
+ parameters["required"] = required
+ out = {"name": fn.__name__, "parameters": parameters}
+ if description:
+ out["description"] = description
+ return out
+
+
+def _is_pydantic_model(obj):
+ return isinstance(obj, type) and any(
+ base.__module__.startswith("pydantic") and base.__name__ == "BaseModel"
+ for base in obj.__mro__)
+
+
+def pydantic_schema(model):
+ raw = model.model_json_schema() if hasattr(model, "model_json_schema") else model.schema()
+ parameters = {"type": "object", "properties": raw.get("properties", {})}
+ for key in ("required", "$defs", "definitions"):
+ if key in raw:
+ parameters[key] = raw[key]
+ out = {"name": raw.get("title", model.__name__), "parameters": parameters}
+ doc = (model.__doc__ or "").strip()
+ if doc:
+ out["description"] = doc
+ return out
+
+
+def tool(fn):
+ fn._needle_tool = build_schema(fn)
+ return fn
diff --git a/needle/cli.py b/needle/cli.py
index 705ba92..2ba520e 100644
--- a/needle/cli.py
+++ b/needle/cli.py
@@ -47,12 +47,8 @@ def _install_xla_log_filter():
sys.stderr = os.fdopen(py_stderr_fd, "w", encoding="utf-8",
errors="replace", buffering=1)
- # 2. Dup another copy for the filter thread to write filtered C-level
- # output to.
- out_fd = os.dup(2) # before dup2 below, fd 2 is still the real terminal
+ out_fd = os.dup(2)
- # 3. Redirect fd 2 to the write end of a new pipe. All C-level writes
- # from XLA now land in the pipe.
r_fd, w_fd = os.pipe()
os.dup2(w_fd, 2)
os.close(w_fd)
@@ -98,7 +94,6 @@ os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2")
os.environ.setdefault("GRPC_VERBOSITY", "ERROR")
_install_xla_log_filter()
-from .dataset.tokenizer import DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN
def main():
@@ -108,88 +103,6 @@ def main():
parser = argparse.ArgumentParser(prog="needle", add_help=False)
sub = parser.add_subparsers(dest="command")
-
- p = sub.add_parser("train", add_help=False)
- p.add_argument("--name", type=str, default="baseline",
- help="Experiment name for checkpoints and wandb (default: baseline)")
- p.add_argument("--checkpoint", type=str, default=None,
- help="Full resume: adopts checkpoint's config and step counter")
- p.add_argument("--init-from", type=str, default=None,
- help="Initialize params from a pretrained base on HF "
- "(e.g. needle_base.pkl). Uses CLI config; partial-loads "
- "matching params, random-inits the rest.")
- p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=64)
- p.add_argument("--lr", type=float, default=3e-5)
- p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=512)
- p.add_argument("--num-heads", type=int, default=8)
- p.add_argument("--num-kv-heads", type=int, default=4)
- p.add_argument("--num-layers", type=int, default=12)
- p.add_argument("--num-dec-layers", type=int, default=8)
- p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
- p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
- p.add_argument("--max-samples", type=int, default=None)
- p.add_argument("--warmup-ratio", type=float, default=0.05)
- p.add_argument("--decay-ratio", type=float, default=0.05)
- p.add_argument("--wandb", action="store_true")
- p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
- p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
- p.add_argument("--seed", type=int, default=42)
- p.add_argument("--eval-every", type=int, default=1000)
- p.add_argument("--max-eval-samples", type=int, default=5000)
- p.add_argument("--contrastive-weight", type=float, default=0.1,
- help="Weight for CLIP-style contrastive loss (default: 0.1)")
- p.add_argument("--contrastive-dim", type=int, default=128,
- help="Dimension of contrastive projection head (default: 128)")
- p.add_argument("--w-name", type=float, default=2.0,
- help="Loss weight for tool name tokens (default: 2.0)")
- p.add_argument("--w-value", type=float, default=4.0,
- help="Loss weight for argument value tokens (default: 4.0)")
- p.add_argument("--w-key", type=float, default=1.5,
- help="Loss weight for argument key tokens (default: 1.5)")
-
- p = sub.add_parser("pretrain", add_help=False)
- p.add_argument("--name", type=str, default="pretrain",
- help="Experiment name for wandb (default: pretrain)")
- p.add_argument("--checkpoint", type=str, default=None,
- help="Resume from checkpoint (e.g. checkpoints/needle_base.pkl)")
- p.add_argument("--resume-step", type=int, default=None,
- help="Override resume step (skip this many batches)")
- p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=128)
- p.add_argument("--lr", type=float, default=3e-4)
- p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=512)
- p.add_argument("--num-heads", type=int, default=8)
- p.add_argument("--num-kv-heads", type=int, default=4)
- p.add_argument("--num-layers", type=int, default=12)
- p.add_argument("--num-dec-layers", type=int, default=8)
- p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
- p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
- p.add_argument("--warmup-ratio", type=float, default=0.05)
- p.add_argument("--decay-ratio", type=float, default=0.05)
- p.add_argument("--wandb", action="store_true")
- p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
- p.add_argument("--seed", type=int, default=42)
- p.add_argument("--max-steps", type=int, default=None,
- help="Stop after N steps (default: full epoch)")
- p.add_argument("--save-every", type=int, default=1000,
- help="Save and upload checkpoint every N steps (default: 1000)")
- p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
-
- p = sub.add_parser("tokenize", add_help=False)
- p.add_argument("--max-samples", type=int, default=None,
- help="Limit samples per split (for dev/test)")
- p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN,
- help=f"Max encoder sequence length (default: {DEFAULT_MAX_ENC_LEN})")
- p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN,
- help=f"Max decoder sequence length (default: {DEFAULT_MAX_DEC_LEN})")
- p.add_argument("--shuffle-tools", action=argparse.BooleanOptionalAction, default=True,
- help="Shuffle tool order in encoder input (default: True)")
- p.add_argument("--max-tool-len", type=int, default=256,
- help="Max token length for individual tool descriptions (default: 256)")
-
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
@@ -199,143 +112,61 @@ def main():
p.add_argument("--no-constrained", action="store_true",
help="Disable grammar-constrained decoding for tool names/arg keys")
- p = sub.add_parser("eval", add_help=False)
- p.add_argument("--checkpoint", type=str, required=True)
- p.add_argument("--batch-size", type=int, default=32)
- p.add_argument("--max-eval-samples", type=int, default=5000)
- p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
- p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
- p.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)
- p.add_argument("--tool-call-samples", type=int, default=200,
- help="Samples for tool-call accuracy eval (default: 200)")
- p.add_argument("--throughput-runs", type=int, default=10)
- p.add_argument("--no-constrained", action="store_true",
- help="Disable grammar-constrained decoding for tool names/arg keys")
-
- p = sub.add_parser("generate-data", add_help=False)
- p.add_argument("--num-samples", type=int, default=500, help="Number of samples to generate")
- p.add_argument("--batch-size", type=int, default=25, help="Examples per Gemini call")
- p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")
- p.add_argument("--model", type=str, default=None, help="Gemini model override")
- p.add_argument("--dry-run", action="store_true", help="Generate only, skip save and upload")
- p.add_argument("--output-jsonl", type=str, default=None, help="Also save raw generations to JSONL")
- p.add_argument("--upload-every", type=int, default=None, help="Merge+upload every N samples")
-
- p = sub.add_parser("evaluate", add_help=False)
- p.add_argument("--checkpoint", type=str, required=True)
- p.add_argument("--benchmarks", type=str, nargs="*",
- choices=["wikitext2", "lambada", "hellaswag", "arc_easy"])
- p.add_argument("--max-samples", type=int, default=500)
-
p = sub.add_parser("finetune", add_help=False)
p.add_argument("jsonl_path", type=str, help="Path to JSONL training data")
p.add_argument("--checkpoint", type=str, default=None,
help="Base model checkpoint (auto-downloads from HuggingFace if omitted)")
- p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=64)
+ p.add_argument("--epochs", type=int, default=3)
+ p.add_argument("--batch-size", type=int, default=16)
+ p.add_argument("--lr", type=float, default=1e-4)
+ p.add_argument("--lora-rank", type=int, default=16, help="LoRA adapter rank (default: 16)")
+ p.add_argument("--lora-alpha", type=float, default=32.0, help="LoRA scaling alpha (default: 32)")
+ p.add_argument("--max-len", type=int, default=1024, help="Max training sequence length")
+ p.add_argument("--generate", type=int, default=0,
+ help="Generate N extra examples via OpenRouter before training (0 = off)")
+ p.add_argument("--model", type=str, default="deepseek/deepseek-v4-flash",
+ help="OpenRouter model for --generate")
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
- p.add_argument("--cache-dir", type=str, default=None)
- p.add_argument("--max-enc-len", type=int, default=None)
- p.add_argument("--max-dec-len", type=int, default=None)
+ p.add_argument("--out", type=str, default=None, help="Output adapter path (.pkl)")
+
+ p = sub.add_parser("generate-data", add_help=False)
+ p.add_argument("--tools", type=str, default=None, help="Tool schemas JSON to seed generation")
+ p.add_argument("--augment", type=str, default=None, help="Existing JSONL to expand")
+ p.add_argument("--num-samples", type=int, default=200)
+ p.add_argument("--batch-size", type=int, default=25)
+ p.add_argument("--model", type=str, default="deepseek/deepseek-v4-flash")
+ p.add_argument("--output", type=str, default=None)
+
+ p = sub.add_parser("build", add_help=False)
+ p.add_argument("checkpoint", type=str, help="Base checkpoint (.pkl) to export")
+ p.add_argument("--lora", type=str, default=None, help="LoRA adapter to merge before export")
+ p.add_argument("--out", type=str, default=None, help="Output .cact path")
+ p.add_argument("--upload", action="store_true", help="Push the .cact to $NEEDLE_HF_REPO")
+ p.add_argument("--bits", type=str, default=None, choices=["2", "4"])
p = sub.add_parser("playground", add_help=False)
p.add_argument("--checkpoint", type=str, default=None)
p.add_argument("--port", type=int, default=7860)
p.add_argument("--host", type=str, default="127.0.0.1")
- p = sub.add_parser("tpu", add_help=False)
- tpu_sub = p.add_subparsers(dest="tpu_action")
-
- tp = tpu_sub.add_parser("create", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--type", dest="accel_type", type=str, default="v6e-8")
- tp.add_argument("--version", type=str, default=None,
- help="Software version (auto-detected from --type if omitted)")
- tp.add_argument("--preemptible", action="store_true", default=False,
- help="Create a preemptible (spot) TPU VM")
-
- tp = tpu_sub.add_parser("connect", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
-
- tp = tpu_sub.add_parser("setup", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
-
- tp = tpu_sub.add_parser("sync", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
-
- tp = tpu_sub.add_parser("train", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
- tp.add_argument("train_args", nargs=argparse.REMAINDER,
- help="Extra args passed to needle train")
-
- tp = tpu_sub.add_parser("pretrain", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
- tp.add_argument("train_args", nargs=argparse.REMAINDER,
- help="Extra args passed to needle pretrain")
-
- tp = tpu_sub.add_parser("claude", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
-
- tp = tpu_sub.add_parser("stop", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
-
- tp = tpu_sub.add_parser("start", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
-
- tp = tpu_sub.add_parser("delete", add_help=False)
- tp.add_argument("name", type=str)
- tp.add_argument("--zone", type=str, default=None)
-
- tpu_sub.add_parser("list", add_help=False)
-
args = parser.parse_args()
if not args.command:
print(HELP)
sys.exit(0)
- if args.command == "tokenize":
- from .dataset.tokenize import tokenize
- tokenize(args)
- elif args.command == "pretrain":
- import jax
- if os.path.exists("/dev/accel0"):
- jax.distributed.initialize()
- from .training.pretrain import pretrain
- pretrain(args)
- elif args.command == "train":
- import jax
- if os.path.exists("/dev/accel0"):
- jax.distributed.initialize()
- from .training.train import train
- train(args)
- elif args.command == "run":
+ if args.command == "run":
from .model.run import main as run_main
run_main(args)
- elif args.command == "eval":
- from .training.eval import main as eval_main_fn
- eval_main_fn(args)
- elif args.command == "generate-data":
- from .dataset.generate import main as gendata_main, MODEL as _MODEL, UPLOAD_EVERY as _UE
- if args.model is None:
- args.model = _MODEL
- if args.upload_every is None:
- args.upload_every = _UE
- gendata_main(args)
elif args.command == "finetune":
- from .training.finetune import finetune_local
+ from .model.finetune import finetune_local
finetune_local(args)
+ elif args.command == "generate-data":
+ from .model.finetune import generate_main
+ generate_main(args)
+ elif args.command == "build":
+ from .model.finetune import build_main
+ build_main(args)
elif args.command == "playground":
- from .ui.server import main as ui_main
+ from .agent.server import main as ui_main
ui_main(args)
- elif args.command == "tpu":
- from .utils.tpu import tpu_dispatch
- tpu_dispatch(args)
diff --git a/needle/dataset/__init__.py b/needle/dataset/__init__.py
deleted file mode 100644
index 022e8de..0000000
--- a/needle/dataset/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .tokenizer import * # noqa: F401,F403
-from .dataset import * # noqa: F401,F403
diff --git a/needle/dataset/dataset.py b/needle/dataset/dataset.py
deleted file mode 100644
index 3d4a1ea..0000000
--- a/needle/dataset/dataset.py
+++ /dev/null
@@ -1,992 +0,0 @@
-import hashlib
-import json as _json
-import multiprocessing as mp
-import os
-import queue
-import threading
-os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
-os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
-
-import logging
-logging.getLogger("transformers").setLevel(logging.ERROR)
-logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
-
-import numpy as np
-from datasets import Dataset, load_from_disk
-from tqdm import tqdm
-import sentencepiece as spm
-
-from .tokenizer import (
- PAD_ID, EOS_ID, BOS_ID, UNK_ID, TOOL_CALL_ID, TOOLS_ID,
- DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN,
- to_snake_case, NeedleTokenizer, train_tokenizer, get_tokenizer,
- _download_tokenizer_from_hf, TOKENIZER_DIR, TOKENIZER_PREFIX,
-)
-
-import re as _re
-
-HF_DATASET_REPO = "Cactus-Compute/tool-calls"
-
-
-def download_hf_split(split="train", repo_id=HF_DATASET_REPO):
- """Download a dataset split from HuggingFace by reading parquet directly.
-
- Bypasses load_dataset() which can fail when the repo README schema
- doesn't match the actual parquet columns (e.g. after adding 'language').
- """
- from huggingface_hub import HfApi, hf_hub_download
- import pyarrow.parquet as pq
-
- api = HfApi()
- files = api.list_repo_files(repo_id, repo_type="dataset", token=True)
- prefix = f"data/{split}-"
- parquet_files = sorted(f for f in files if f.startswith(prefix) and f.endswith(".parquet"))
-
- if not parquet_files:
- raise FileNotFoundError(
- f"No parquet files found for split '{split}' in {repo_id} "
- f"(looked for {prefix}*.parquet)"
- )
-
- tables = []
- for pf in parquet_files:
- local = hf_hub_download(repo_id=repo_id, filename=pf,
- repo_type="dataset", token=True)
- tables.append(pq.read_table(local))
-
- if len(tables) == 1:
- table = tables[0]
- else:
- import pyarrow as pa
- table = pa.concat_tables(tables)
-
- return Dataset(table)
-
-_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
-_DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")
-LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
-CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")
-
-_split_dataset_cache = {}
-
-
-def _mark_json_value(s, char_w, key, value_str, weight):
- """Find '"key": "value_str"' or '"key": value_str' in s, mark value chars."""
- pattern_str = f'"{_re.escape(key)}"\\s*:\\s*"{_re.escape(value_str)}"'
- for m in _re.finditer(pattern_str, s):
- tail = s[m.start() + len(f'"{key}"'):m.end()]
- val_offset = tail.index(f'"{value_str}"') + 1
- val_start = m.start() + len(f'"{key}"') + val_offset
- val_end = val_start + len(value_str)
- char_w[val_start:val_end] = np.maximum(char_w[val_start:val_end], weight)
- return
-
- pattern_ns = f'"{_re.escape(key)}"\\s*:\\s*{_re.escape(value_str)}'
- for m in _re.finditer(pattern_ns, s):
- colon_offset = s[m.start():m.end()].index(':')
- val_start = m.start() + colon_offset + 1
- while val_start < m.end() and s[val_start] == ' ':
- val_start += 1
- val_end = m.end()
- char_w[val_start:val_end] = np.maximum(char_w[val_start:val_end], weight)
- return
-
-
-def _mark_json_key_in_args(s, char_w, key, weight):
- """Mark the argument key string (inside quotes) in the JSON."""
- for m in _re.finditer(f'"{_re.escape(key)}"\\s*:', s):
- char_w[m.start() + 1:m.start() + 1 + len(key)] = np.maximum(
- char_w[m.start() + 1:m.start() + 1 + len(key)], weight)
-
-
-def _count_tool_calls(answers_str):
- """Count the number of tool calls in an answers JSON string."""
- try:
- calls = _json.loads(answers_str)
- except (ValueError, TypeError):
- return 0
- return len(calls) if isinstance(calls, list) else 0
-
-
-def _shuffle_tools_json(tools_str, seed):
- """Parse tools JSON array, shuffle tool order and parameter order deterministically.
-
- Shuffles both the order of tools in the list and the order of parameters
- within each tool's parameter dict, preventing the model from memorizing
- positional patterns in the encoder input.
- """
- try:
- tools = _json.loads(tools_str)
- except (ValueError, TypeError):
- return tools_str
- if not isinstance(tools, list):
- return tools_str
- rng = np.random.RandomState(seed)
- # Shuffle tool order
- if len(tools) > 1:
- rng.shuffle(tools)
- # Shuffle parameter order within each tool
- for tool in tools:
- if not isinstance(tool, dict):
- continue
- params = tool.get("parameters")
- if isinstance(params, dict) and len(params) > 1:
- keys = list(params.keys())
- rng.shuffle(keys)
- tool["parameters"] = {k: params[k] for k in keys}
- # Also shuffle top-level tool keys (name, description, parameters)
- top_keys = list(tool.keys())
- if len(top_keys) > 1:
- rng.shuffle(top_keys)
- shuffled = {k: tool[k] for k in top_keys}
- tool.clear()
- tool.update(shuffled)
- return _json.dumps(tools, separators=(",", ":"))
-
-
-
-TOKEN_CLASS_BASE = 0
-TOKEN_CLASS_NAME = 1
-TOKEN_CLASS_VALUE = 2
-TOKEN_CLASS_KEY = 3
-
-
-def _token_classes_for_answer(answer_str, token_ids, sp_model):
- """Compute per-token class labels based on JSON structure.
-
- Returns int8 array of class labels (TOKEN_CLASS_*).
- Weight scalars are applied at train time, so tokenization is weight-agnostic.
- """
- n = len(token_ids)
- classes = np.zeros(n, dtype=np.int8)
-
- try:
- calls = _json.loads(answer_str)
- except (ValueError, TypeError):
- return classes
-
- if not isinstance(calls, list):
- return classes
-
- char_cls = np.zeros(len(answer_str), dtype=np.int8)
-
- for call in calls:
- if not isinstance(call, dict):
- continue
- name = call.get("name", "")
- if name:
- _mark_json_value(answer_str, char_cls, "name", name, TOKEN_CLASS_NAME)
-
- args = call.get("arguments", {})
- if isinstance(args, dict):
- for k, v in args.items():
- _mark_json_key_in_args(answer_str, char_cls, k, TOKEN_CLASS_KEY)
- v_str = _json.dumps(v) if not isinstance(v, str) else v
- _mark_json_value(answer_str, char_cls, k, v_str, TOKEN_CLASS_VALUE)
-
- pieces = sp_model.Encode(answer_str, out_type=str)
- pos = 0
- for i, piece in enumerate(pieces):
- if i >= n:
- break
- raw = piece.replace('\u2581', ' ')
- plen = len(raw)
- if plen > 0 and pos + plen <= len(answer_str):
- classes[i] = char_cls[pos:pos + plen].max()
- pos += plen
- else:
- pos += max(plen, 1)
-
- return classes
-
-
-def _token_weights_for_answer(answer_str, token_ids, sp_model,
- w_name=3.0, w_value=2.0, w_key=1.5):
- """Compute per-token loss weights from class labels and weight scalars.
-
- Convenience wrapper for backward compatibility.
- """
- classes = _token_classes_for_answer(answer_str, token_ids, sp_model)
- weight_map = np.array([1.0, w_name, w_value, w_key], dtype=np.float32)
- return weight_map[classes]
-
-
-def _load_split_dataset(split="train"):
- """Load a dataset split from disk or HuggingFace.
-
- Caches each split in memory after first load.
- Uses soundfile as the audio decoding backend.
- """
- hf_split = "validation" if split in ("validation", "val", "test") else "train"
-
- if hf_split in _split_dataset_cache:
- return _split_dataset_cache[hf_split]
-
- local_dir = os.path.join(LOCAL_UNIFIED_DIR, hf_split)
- if os.path.exists(local_dir) and any(
- f.endswith(".arrow") for f in os.listdir(local_dir)
- ):
- try:
- ds = load_from_disk(local_dir)
- print(f"Loaded {hf_split} split from {local_dir} ({len(ds)} rows)")
- _split_dataset_cache[hf_split] = _set_audio_backend(ds)
- return _split_dataset_cache[hf_split]
- except Exception as e:
- print(f"Warning: failed to load from {local_dir}: {e}")
-
- try:
- print(f"Downloading {hf_split} split from HuggingFace ({HF_DATASET_REPO})...")
- ds = download_hf_split(hf_split)
- os.makedirs(local_dir, exist_ok=True)
- ds.save_to_disk(local_dir)
- print(f"Loaded from HuggingFace -> {local_dir} ({len(ds)} rows)")
- _split_dataset_cache[hf_split] = _set_audio_backend(ds)
- return _split_dataset_cache[hf_split]
- except Exception as e:
- print(f"Warning: HuggingFace download failed: {e}")
-
- raise FileNotFoundError(
- f"Dataset split '{hf_split}' not found at {local_dir}. "
- f"Run 'needle tokenize' first."
- )
-
-
-def _set_audio_backend(ds):
- """No-op kept for compatibility."""
- return ds
-
-
-_worker_sp = None
-_worker_max_len = None
-
-
-def _init_worker(model_path, max_length):
- """Initializer for multiprocessing pool — loads SP model once per worker."""
- global _worker_sp, _worker_max_len
- _worker_sp = spm.SentencePieceProcessor()
- _worker_sp.Load(model_path)
- _worker_max_len = max_length
-
-
-def _tokenize_chunk(texts):
- """Encode a chunk of texts in a worker process."""
- return [_worker_sp.Encode(t, out_type=int)[:_worker_max_len] for t in texts]
-
-
-def _compute_classes_chunk(chunk):
- """Compute token classes for a chunk of (answer_str, token_ids) pairs."""
- results = []
- for answer_str, token_ids in chunk:
- results.append(_token_classes_for_answer(answer_str, token_ids, _worker_sp))
- return results
-
-
-_HF_TOKENIZED_REPO = "Cactus-Compute/tokenized-tool-calls"
-
-
-def _download_tokenized_from_hf():
- """Download tokenized .npy files from HuggingFace Hub into CACHE_DIR."""
- from huggingface_hub import snapshot_download
-
- local = snapshot_download(
- _HF_TOKENIZED_REPO,
- repo_type="dataset",
- token=True,
- )
- os.makedirs(CACHE_DIR, exist_ok=True)
- for fname in os.listdir(local):
- if fname.startswith("."):
- continue
- dst = os.path.join(CACHE_DIR, fname)
- # Metadata JSONs: always overwrite so HF matches downloaded data
- if fname.endswith(".json") and os.path.exists(dst):
- os.remove(dst)
- if not os.path.exists(dst):
- os.symlink(os.path.join(local, fname), dst)
-
-
-
-
-def load_tool_calls(split="train", max_samples=None, return_global_indices=False):
- """Load tool-calling dataset split directly from HuggingFace.
-
- The dataset has official train/validation splits on HuggingFace.
- Train is shuffled with a fixed seed; validation is returned as-is.
-
- If return_global_indices is True, also return a numpy array of row indices
- (sequential within the split).
- """
- ds = _load_split_dataset(split)
- n = len(ds)
- global_indices = np.arange(n, dtype=np.int64)
-
- if split == "train":
- rng = np.random.RandomState(42)
- perm = rng.permutation(n).astype(np.int64)
- ds = ds.select(perm.tolist())
- global_indices = perm
-
- if max_samples:
- limit = min(max_samples, n)
- ds = ds.select(range(limit))
- global_indices = global_indices[:limit]
-
- if return_global_indices:
- return ds, global_indices
- return ds
-
-
-def _tokenizer_hash():
- """Hash the tokenizer model file to detect retraining."""
- model_path = TOKENIZER_PREFIX + ".model"
- if os.path.exists(model_path):
- with open(model_path, "rb") as f:
- return hashlib.md5(f.read()).hexdigest()[:8]
- return "none"
-
-
-def _cache_key(prefix, n_samples, max_enc_len, max_dec_len,
- w_name=3.0, w_value=2.0, w_key=1.5, shuffle_tools=True):
- tok_hash = _tokenizer_hash()
- key = f"{prefix}_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}_{w_name}_{w_value}_{w_key}_{shuffle_tools}"
- return hashlib.md5(key.encode()).hexdigest()[:12]
-
-
-def _save_cache_metadata(split, text_cache_id, n_samples, max_enc_len, max_dec_len, max_tool_len=256):
- """Save metadata JSON for a split locally."""
- os.makedirs(CACHE_DIR, exist_ok=True)
- meta = {
- "split": split,
- "text_cache_id": text_cache_id,
- "n_samples": n_samples,
- "max_enc_len": max_enc_len,
- "max_dec_len": max_dec_len,
- "max_tool_len": max_tool_len,
- }
- meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")
- with open(meta_path, "w") as f:
- _json.dump(meta, f)
-
-
-def _load_cache_metadata(split):
- """Load metadata JSON from local cache. Returns dict or None."""
- meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")
- if os.path.exists(meta_path):
- with open(meta_path) as f:
- return _json.load(f)
- return None
-
-
-def _compact_json(s):
- """Compact a JSON string (remove whitespace). Picklable for multiprocessing."""
- try:
- return _json.dumps(_json.loads(s), separators=(",", ":"))
- except (ValueError, TypeError):
- return s
-
-
-def _shuffle_tools_worker(args):
- """Shuffle tools JSON. Picklable for multiprocessing."""
- seed, tools_str = args
- return _shuffle_tools_json(tools_str, seed=seed)
-
-
-def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_dec_len=DEFAULT_MAX_DEC_LEN,
- w_name=3.0, w_value=2.0, w_key=1.5, shuffle_tools=True):
- """Prepare tool-call encoder-decoder pairs with <tool_call> task token.
-
- Builds variable-length sequences in memory as VarLenArray objects.
- The loss array stores per-token class labels (int8: 0=base, 1=name,
- 2=value, 3=key) rather than final weights. Weight scalars are applied
- at train time so tokenization doesn't need to be re-run when weights change.
-
- Returns (enc_vl, dec_in_vl, dec_tgt_vl, loss_vl, kept_indices, tool_counts).
- """
-
- cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
- 1.0, 1.0, 1.0, shuffle_tools) # weights no longer affect cache
- cache_path = os.path.join(CACHE_DIR, cache_id)
-
- eos_id = tokenizer.eos_token_id
- tool_call_id = tokenizer.tool_call_token_id
- tools_sep_id = tokenizer.tools_token_id
-
- enc_texts = [ex["query"] for ex in ds]
- tools_texts = [ex["tools"] for ex in ds]
- ans_texts = [ex["answers"] for ex in ds]
-
- n = len(ds)
- num_workers = min(128, max(1, (os.cpu_count() or 1) * 3 // 4))
- model_path = TOKENIZER_PREFIX + ".model"
- chunk_size = max(1, n // (num_workers * 4))
-
- print(" Compacting JSON and preparing texts...")
- with mp.Pool(num_workers) as pool:
- ans_texts = list(tqdm(pool.imap(_compact_json, ans_texts, chunksize=chunk_size),
- total=n, desc=" Compacting answers"))
- tools_texts = list(tqdm(pool.imap(_compact_json, tools_texts, chunksize=chunk_size),
- total=n, desc=" Compacting tools"))
- tool_counts = np.array([_count_tool_calls(a) for a in ans_texts], dtype=np.int32)
-
- if shuffle_tools:
- shuffle_args = list(enumerate(tools_texts))
- with mp.Pool(num_workers) as pool:
- tools_texts = list(tqdm(pool.imap(_shuffle_tools_worker, shuffle_args, chunksize=chunk_size),
- total=n, desc=" Shuffling tools"))
-
- # Tokenize all three fields in a single pool
- all_texts = enc_texts + tools_texts + ans_texts
- max_tok_len = max(max_enc_len, max_dec_len)
- all_chunks = [all_texts[i:i + chunk_size] for i in range(0, len(all_texts), chunk_size)]
- with mp.Pool(num_workers, initializer=_init_worker,
- initargs=(model_path, max_tok_len)) as pool:
- all_results = list(tqdm(pool.imap(_tokenize_chunk, all_chunks),
- total=len(all_chunks), desc="Tokenizing"))
- all_tokens_flat = [tok for chunk in all_results for tok in chunk]
- all_enc_tokens = all_tokens_flat[:n]
- all_tools_tokens = all_tokens_flat[n:2*n]
- all_ans_tokens = all_tokens_flat[2*n:]
-
- # Build pairs (no SentencePiece needed — pure list ops)
- enc_seqs = []
- dec_in_seqs = []
- dec_tgt_seqs = []
- kept_indices = []
- kept_tool_counts = []
- kept_ans_texts = [] # for parallel class computation
- kept_ans_tokens = []
-
- for i in range(n):
- e_tok = all_enc_tokens[i]
- t_tok = all_tools_tokens[i]
- a_tok = all_ans_tokens[i]
-
- max_query = max_enc_len - 2
- if len(e_tok) > max_query:
- e_tok = e_tok[:max_query]
- remaining = max_enc_len - len(e_tok) - 1
- enc_seq = e_tok + [tools_sep_id] + t_tok[:remaining]
-
- al = len(a_tok)
- if 2 + al + 1 > max_dec_len:
- continue
-
- enc_seqs.append(np.array(enc_seq, dtype=np.int16))
- dec_in_seqs.append(np.array([eos_id, tool_call_id] + list(a_tok), dtype=np.int16))
- dec_tgt_seqs.append(np.array([tool_call_id] + list(a_tok) + [eos_id], dtype=np.int16))
- kept_indices.append(i)
- kept_tool_counts.append(tool_counts[i])
- kept_ans_texts.append(ans_texts[i])
- kept_ans_tokens.append(a_tok)
-
- skipped = n - len(kept_indices)
- if skipped > 0:
- print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")
-
- # Compute token classes in parallel (requires SentencePiece re-encoding)
- class_inputs = list(zip(kept_ans_texts, kept_ans_tokens))
- class_chunks = [class_inputs[i:i + chunk_size] for i in range(0, len(class_inputs), chunk_size)]
- with mp.Pool(num_workers, initializer=_init_worker,
- initargs=(model_path, max_dec_len)) as pool:
- class_results = list(tqdm(pool.imap(_compute_classes_chunk, class_chunks),
- total=len(class_chunks), desc="Computing token classes"))
- all_classes = [cls for chunk in class_results for cls in chunk]
-
- loss_seqs = []
- for j, token_classes in enumerate(all_classes):
- classes = np.zeros(len(dec_tgt_seqs[j]), dtype=np.int8)
- classes[1:1 + len(token_classes)] = token_classes
- loss_seqs.append(classes)
-
- kept_indices = np.array(kept_indices, dtype=np.int64)
- kept_tool_counts = np.array(kept_tool_counts, dtype=np.int32)
-
- enc_vl = VarLenArray.from_sequences(enc_seqs, max_enc_len, pad_value=PAD_ID)
- dec_in_vl = VarLenArray.from_sequences(dec_in_seqs, max_dec_len, pad_value=PAD_ID)
- dec_tgt_vl = VarLenArray.from_sequences(dec_tgt_seqs, max_dec_len, pad_value=PAD_ID)
- loss_vl = VarLenArray.from_sequences(loss_seqs, max_dec_len, pad_value=0)
- print(f"Prepared {len(enc_seqs):,} tool-call pairs")
-
- # Save auxiliary arrays needed for training (kept indices, tool counts, contrastive)
- os.makedirs(CACHE_DIR, exist_ok=True)
- np.save(cache_path + "_kept_idx.npy", kept_indices)
- np.save(cache_path + "_tool_count.npy", kept_tool_counts)
-
- max_tool_len = getattr(prepare_tool_call_pairs, '_max_tool_len', 256)
- _build_contrastive_arrays(ds, enc_texts, tools_texts, all_enc_tokens, cache_path,
- model_path, max_enc_len, max_tool_len,
- num_workers, chunk_size, kept_indices)
-
- return enc_vl, dec_in_vl, dec_tgt_vl, loss_vl, kept_indices, kept_tool_counts
-
-
-def _build_contrastive_arrays(ds, enc_texts, tools_texts, all_enc_tokens, cache_path,
- model_path, max_enc_len, max_tool_len,
- num_workers, chunk_size, kept_indices):
- """Build and save contrastive arrays: query-only + individual tool tokens.
-
- Both are stored as variable-length int16 arrays (VarLen format).
- Query tokens are reused from the main tokenization pass (all_enc_tokens).
- """
- kept_set = set(kept_indices.tolist()) if kept_indices is not None else set(range(len(enc_texts)))
- n_kept = len(kept_set)
-
- tool_texts_individual = []
- tool_ex_idx = []
- tool_is_pos = []
-
- kept_list = sorted(kept_set)
- kept_to_local = {g: i for i, g in enumerate(kept_list)}
-
- for global_i in kept_list:
- local_i = kept_to_local[global_i]
- tools_str = tools_texts[global_i]
- ans_str = ds[global_i]["answers"] if global_i < len(ds) else "[]"
-
- try:
- tools = _json.loads(tools_str)
- except (ValueError, TypeError):
- tools = []
- if not isinstance(tools, list):
- tools = []
-
- try:
- calls = _json.loads(ans_str)
- except (ValueError, TypeError):
- calls = []
- pos_names = set()
- if isinstance(calls, list):
- for c in calls:
- if isinstance(c, dict) and "name" in c:
- pos_names.add(c["name"])
-
- for tool in tools:
- if not isinstance(tool, dict):
- continue
- tool_str = _json.dumps(tool, separators=(",", ":"))
- tool_texts_individual.append(tool_str)
- tool_ex_idx.append(local_i)
- tool_is_pos.append(tool.get("name", "") in pos_names)
-
- if tool_texts_individual:
- t_chunks = [tool_texts_individual[i:i + chunk_size]
- for i in range(0, len(tool_texts_individual), chunk_size)]
- with mp.Pool(num_workers, initializer=_init_worker,
- initargs=(model_path, max_tool_len)) as pool:
- t_results = list(tqdm(pool.imap(_tokenize_chunk, t_chunks),
- total=len(t_chunks), desc="Tokenizing tools (contrastive)"))
- tool_seqs = [np.array(tok[:max_tool_len], dtype=np.int16)
- for chunk in t_results for tok in chunk]
- else:
- tool_seqs = []
-
- # Reuse already-tokenized query tokens (no re-tokenization needed)
- query_seqs = [np.array(all_enc_tokens[i], dtype=np.int16) for i in kept_list]
- _save_varlen(cache_path + "_query_only", query_seqs)
-
- _save_varlen(cache_path + "_tool_ind", tool_seqs)
- np.save(cache_path + "_tool_ex_idx.npy", np.array(tool_ex_idx, dtype=np.int32))
- np.save(cache_path + "_tool_is_pos.npy", np.array(tool_is_pos, dtype=np.bool_))
- print(f" Contrastive: {len(query_seqs)} queries, {len(tool_ex_idx)} individual tools")
-
-
-def get_contrastive_batches(query_tokens, tool_tokens, tool_ex_idx, tool_is_pos, batch_size):
- """Yield (query_batch, tool_batch) for CLIP-style contrastive training.
-
- Each batch: B queries paired with 1 randomly-chosen positive tool each.
- In-batch negatives provide the contrastive signal.
- """
- n_queries = len(query_tokens)
- pos_map = {}
- for t_idx in range(len(tool_ex_idx)):
- if tool_is_pos[t_idx]:
- pos_map.setdefault(int(tool_ex_idx[t_idx]), []).append(t_idx)
-
- valid_queries = [i for i in range(n_queries) if i in pos_map]
- if not valid_queries:
- return
-
- indices = np.array(valid_queries)
- np.random.shuffle(indices)
-
- for start in range(0, len(indices) - batch_size + 1, batch_size):
- batch_q_idx = indices[start:start + batch_size]
- q_batch = np.array(query_tokens[batch_q_idx])
- t_indices = np.array([
- pos_map[int(qi)][np.random.randint(len(pos_map[int(qi)]))]
- for qi in batch_q_idx
- ])
- t_batch = np.array(tool_tokens[t_indices])
- yield q_batch, t_batch
-
-
-def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True,
- loss_mask=None, tool_counts=None, enc_seg_ids=None, dec_seg_ids=None):
- n = len(enc_inputs)
- if tool_counts is not None:
- order = np.argsort(tool_counts, kind='stable')
- for c in np.unique(tool_counts):
- group = np.where(tool_counts[order] == c)[0]
- shuffled = order[group].copy()
- np.random.shuffle(shuffled)
- order[group] = shuffled
- indices = order
- elif shuffle:
- indices = np.random.permutation(n)
- else:
- indices = np.arange(n)
- for i in range(0, n - batch_size + 1, batch_size):
- idx = indices[i : i + batch_size]
- batch = (np.array(enc_inputs[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))
- if loss_mask is not None:
- batch = batch + (np.array(loss_mask[idx]),)
- if enc_seg_ids is not None:
- batch = batch + (np.array(enc_seg_ids[idx]), np.array(dec_seg_ids[idx]))
- yield batch
-
-
-def _seq_lens(arr):
- """Get per-sequence lengths from a VarLenArray via its offsets."""
- return np.diff(arr._offsets).astype(np.int32)
-
-
-def pack_sequences(cache_path, enc_vl, dec_in_vl, dec_tgt_vl, loss_vl):
- """Pre-pack variable-length sequences into dense bins with segment IDs.
-
- Uses first-fit-decreasing bin packing with vectorized bin search.
- Saves packed int16 arrays + segment ID arrays.
-
- Returns the number of packed bins.
- """
- n = len(enc_vl)
- max_enc = enc_vl._max_len
- max_dec = dec_in_vl._max_len
-
- enc_lens = _seq_lens(enc_vl)
- dec_lens = _seq_lens(dec_in_vl)
-
- order = np.argsort(-enc_lens)
- bin_enc_rem = np.empty(n, dtype=np.int32)
- bin_dec_rem = np.empty(n, dtype=np.int32)
- bin_contents = [[] for _ in range(n)]
- n_bins = 0
-
- for idx in tqdm(order, desc="Bin packing"):
- el = int(enc_lens[idx])
- dl = int(dec_lens[idx])
-
- if n_bins > 0:
- fits = (bin_enc_rem[:n_bins] >= el) & (bin_dec_rem[:n_bins] >= dl)
- candidates = np.flatnonzero(fits)
- else:
- candidates = np.array([], dtype=np.intp)
-
- if len(candidates) > 0:
- b = candidates[0]
- bin_enc_rem[b] -= el
- bin_dec_rem[b] -= dl
- bin_contents[b].append(int(idx))
- else:
- bin_enc_rem[n_bins] = max_enc - el
- bin_dec_rem[n_bins] = max_dec - dl
- bin_contents[n_bins].append(int(idx))
- n_bins += 1
-
- bin_enc_rem = bin_enc_rem[:n_bins]
- bin_dec_rem = bin_dec_rem[:n_bins]
- bin_contents = bin_contents[:n_bins]
-
- packed_enc = np.zeros((n_bins, max_enc), dtype=np.int16)
- packed_dec_in = np.zeros((n_bins, max_dec), dtype=np.int16)
- packed_dec_tgt = np.zeros((n_bins, max_dec), dtype=np.int16)
- packed_loss = np.zeros((n_bins, max_dec), dtype=np.int8)
- packed_enc_seg = np.zeros((n_bins, max_enc), dtype=np.int16)
- packed_dec_seg = np.zeros((n_bins, max_dec), dtype=np.int16)
-
- for row in tqdm(range(n_bins), desc="Writing packed bins"):
- enc_pos = 0
- dec_pos = 0
- for seg_id, idx in enumerate(bin_contents[row], start=1):
- el = int(enc_lens[idx])
- dl = int(dec_lens[idx])
-
- e_start = int(enc_vl._offsets[idx])
- packed_enc[row, enc_pos:enc_pos + el] = enc_vl._data[e_start:e_start + el]
- packed_enc_seg[row, enc_pos:enc_pos + el] = seg_id
-
- di_start = int(dec_in_vl._offsets[idx])
- packed_dec_in[row, dec_pos:dec_pos + dl] = dec_in_vl._data[di_start:di_start + dl]
-
- dt_start = int(dec_tgt_vl._offsets[idx])
- packed_dec_tgt[row, dec_pos:dec_pos + dl] = dec_tgt_vl._data[dt_start:dt_start + dl]
-
- lm_start = int(loss_vl._offsets[idx])
- packed_loss[row, dec_pos:dec_pos + dl] = loss_vl._data[lm_start:lm_start + dl]
-
- packed_dec_seg[row, dec_pos:dec_pos + dl] = seg_id
-
- enc_pos += el
- dec_pos += dl
-
- np.save(cache_path + "_packed_enc.npy", packed_enc)
- np.save(cache_path + "_packed_dec_in.npy", packed_dec_in)
- np.save(cache_path + "_packed_dec_tgt.npy", packed_dec_tgt)
- np.save(cache_path + "_packed_loss.npy", packed_loss)
- np.save(cache_path + "_packed_enc_seg.npy", packed_enc_seg)
- np.save(cache_path + "_packed_dec_seg.npy", packed_dec_seg)
-
- total_cap = n_bins * (max_enc + max_dec)
- used = int(np.sum(max_enc - bin_enc_rem)) + int(np.sum(max_dec - bin_dec_rem))
- print(f" Packed {n} sequences into {n_bins} bins ({used / total_cap:.0%} utilization)")
- return n_bins
-
-
-
-class VarLenArray:
- """Variable-length storage with on-demand padding to fixed max_len.
-
- Stores sequences as a flat 1D array (int16 or float16) plus an offsets array.
- When indexed, returns padded arrays cast to int32/float32 for JAX compatibility.
- """
-
- def __init__(self, data, offsets, max_len, pad_value=0):
- self._data = data
- self._offsets = offsets
- self._max_len = max_len
- self._pad_value = pad_value
- self._n = len(offsets) - 1
- self._out_dtype = np.float32 if data.dtype == np.float16 else np.int32
-
- @classmethod
- def from_sequences(cls, sequences, max_len, pad_value=0):
- """Build a VarLenArray in memory from a list of 1D numpy arrays."""
- offsets = np.zeros(len(sequences) + 1, dtype=np.int64)
- for i, seq in enumerate(sequences):
- offsets[i + 1] = offsets[i] + len(seq)
- data = np.concatenate(sequences) if sequences else np.array([], dtype=np.int16)
- return cls(data, offsets, max_len, pad_value)
-
- @classmethod
- def load(cls, prefix, max_len, pad_value=0, mmap_mode=None):
- data = np.load(prefix + "_data.npy", mmap_mode=mmap_mode)
- offsets = np.load(prefix + "_offsets.npy", mmap_mode=mmap_mode)
- return cls(data, offsets, max_len, pad_value)
-
- def __len__(self):
- return self._n
-
- @property
- def shape(self):
- return (self._n, self._max_len)
-
- @property
- def dtype(self):
- return self._out_dtype
-
- def __getitem__(self, idx):
- if isinstance(idx, (int, np.integer)):
- if idx < 0:
- idx += self._n
- start, end = int(self._offsets[idx]), int(self._offsets[idx + 1])
- result = np.full(self._max_len, self._pad_value, dtype=self._out_dtype)
- seq = self._data[start:end]
- result[:len(seq)] = seq.astype(self._out_dtype)
- return result
-
- if isinstance(idx, np.ndarray):
- batch_size = len(idx)
- result = np.full((batch_size, self._max_len), self._pad_value, dtype=self._out_dtype)
- for j, i in enumerate(idx):
- start, end = int(self._offsets[i]), int(self._offsets[i + 1])
- seq = self._data[start:end]
- result[j, :len(seq)] = seq.astype(self._out_dtype)
- return result
-
- if isinstance(idx, slice):
- indices = np.arange(*idx.indices(self._n))
- return self[indices]
-
- raise TypeError(f"Unsupported index type: {type(idx)}")
-
-
-
-def _save_varlen(prefix, sequences):
- """Save variable-length sequences as flat data + offsets .npy files."""
- offsets = np.zeros(len(sequences) + 1, dtype=np.int64)
- for i, seq in enumerate(sequences):
- offsets[i + 1] = offsets[i] + len(seq)
- if sequences:
- data = np.concatenate(sequences)
- else:
- data = np.array([], dtype=np.int16)
- np.save(prefix + "_data.npy", data)
- np.save(prefix + "_offsets.npy", offsets)
-
-
-class ShardedMmapArray:
- """Array-like wrapper over multiple .npy shard files with mmap support.
-
- Provides __len__ and __getitem__ (int, slice, numpy array) to transparently
- index across shards. Each shard is memory-mapped individually.
- """
-
- def __init__(self, paths, mmap_mode="r"):
- self._shards = [np.load(p, mmap_mode=mmap_mode) for p in paths]
- self._lengths = [len(s) for s in self._shards]
- self._cumulative = np.cumsum(self._lengths)
- self._total = int(self._cumulative[-1]) if len(self._cumulative) else 0
-
- def __len__(self):
- return self._total
-
- @property
- def shape(self):
- if not self._shards:
- return (0,)
- return (self._total,) + self._shards[0].shape[1:]
-
- @property
- def dtype(self):
- return self._shards[0].dtype if self._shards else np.float32
-
- def __getitem__(self, idx):
- if isinstance(idx, (int, np.integer)):
- if idx < 0:
- idx += self._total
- si = int(np.searchsorted(self._cumulative, idx, side="right"))
- offset = int(self._cumulative[si - 1]) if si > 0 else 0
- return self._shards[si][idx - offset]
-
- if isinstance(idx, np.ndarray):
- result = np.empty((len(idx),) + self._shards[0].shape[1:],
- dtype=self._shards[0].dtype)
- shard_ids = np.searchsorted(self._cumulative, idx, side="right")
- for si in range(len(self._shards)):
- mask = shard_ids == si
- if not mask.any():
- continue
- offset = int(self._cumulative[si - 1]) if si > 0 else 0
- local = idx[mask] - offset
- result[mask] = self._shards[si][local]
- return result
-
- if isinstance(idx, slice):
- start, stop, step = idx.indices(self._total)
- indices = np.arange(start, stop, step)
- return self[indices]
-
- raise TypeError(f"Unsupported index type: {type(idx)}")
-
-
-def load_prepared_data(split, mmap=False):
- """Load pre-packed tokenized data from disk.
-
- If mmap=True, underlying arrays are memory-mapped.
-
- Returns dict with keys: packed_enc, packed_dec_in, packed_dec_tgt, packed_loss,
- packed_enc_seg, packed_dec_seg, kept_indices, tool_counts, query_only,
- tool_individual, tool_ex_idx, tool_is_pos.
- """
- meta = _load_cache_metadata(split)
- if meta is None:
- try:
- _download_tokenized_from_hf()
- meta = _load_cache_metadata(split)
- except Exception:
- pass
- if meta is None:
- raise FileNotFoundError(
- f"No prepared data found for split '{split}'. Run 'needle tokenize' first."
- )
-
- text_cache_id = meta["text_cache_id"]
- cache_path = os.path.join(CACHE_DIR, text_cache_id)
- max_enc_len = meta.get("max_enc_len", DEFAULT_MAX_ENC_LEN)
- max_dec_len = meta.get("max_dec_len", DEFAULT_MAX_DEC_LEN)
- max_tool_len = meta.get("max_tool_len", 256)
-
- if not os.path.exists(cache_path + "_packed_enc.npy"):
- raise FileNotFoundError(
- f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."
- )
-
- mmap_mode = "r" if mmap else None
-
- def _load_optional(suffix):
- p = cache_path + suffix
- return np.load(p, mmap_mode=mmap_mode) if os.path.exists(p) else None
-
- # Contrastive arrays
- query_only = None
- if os.path.exists(cache_path + "_query_only_data.npy"):
- query_only = VarLenArray.load(cache_path + "_query_only", max_enc_len,
- pad_value=PAD_ID, mmap_mode=mmap_mode)
-
- tool_individual = None
- if os.path.exists(cache_path + "_tool_ind_data.npy"):
- tool_individual = VarLenArray.load(cache_path + "_tool_ind", max_tool_len,
- pad_value=PAD_ID, mmap_mode=mmap_mode)
-
- return {
- "packed_enc": np.load(cache_path + "_packed_enc.npy", mmap_mode=mmap_mode),
- "packed_dec_in": np.load(cache_path + "_packed_dec_in.npy", mmap_mode=mmap_mode),
- "packed_dec_tgt": np.load(cache_path + "_packed_dec_tgt.npy", mmap_mode=mmap_mode),
- "packed_loss": np.load(cache_path + "_packed_loss.npy", mmap_mode=mmap_mode),
- "packed_enc_seg": np.load(cache_path + "_packed_enc_seg.npy", mmap_mode=mmap_mode),
- "packed_dec_seg": np.load(cache_path + "_packed_dec_seg.npy", mmap_mode=mmap_mode),
- "kept_indices": _load_optional("_kept_idx.npy"),
- "tool_counts": _load_optional("_tool_count.npy"),
- "query_only": query_only,
- "tool_individual": tool_individual,
- "tool_ex_idx": _load_optional("_tool_ex_idx.npy"),
- "tool_is_pos": _load_optional("_tool_is_pos.npy"),
- }
-
-
-class PrefetchIterator:
- """Generic prefetch wrapper: runs any batch-generating callable in a background thread."""
-
- def __init__(self, generator_fn, prefetch=4):
- """generator_fn: callable that returns an iterable of batches."""
- self._queue = queue.Queue(maxsize=prefetch)
- self._stop = threading.Event()
- self._generator_fn = generator_fn
- self._thread = threading.Thread(target=self._produce, daemon=True)
- self._thread.start()
-
- def _produce(self):
- try:
- for batch in self._generator_fn():
- if self._stop.is_set():
- return
- self._queue.put(batch)
- self._queue.put(None) # sentinel
- except Exception as e:
- self._queue.put(e)
-
- def __iter__(self):
- return self
-
- def __next__(self):
- item = self._queue.get()
- if item is None:
- raise StopIteration
- if isinstance(item, Exception):
- raise item
- return item
-
- def close(self):
- self._stop.set()
- while not self._queue.empty():
- try:
- self._queue.get_nowait()
- except queue.Empty:
- break
- self._thread.join(timeout=5)
-
-
-def count_batches(n_samples, batch_size):
- """Return the number of full batches for a dataset of n_samples."""
- return n_samples // batch_size
-
-
diff --git a/needle/dataset/generate.py b/needle/dataset/generate.py
deleted file mode 100644
index cc358b0..0000000
--- a/needle/dataset/generate.py
+++ /dev/null
@@ -1,1936 +0,0 @@
-#!/usr/bin/env python3
-"""Generate on-device assistant tool-calling training data with Gemini and merge
-into the existing unified dataset.
-
-Usage:
- GEMINI_API_KEY=... python scripts/generate_data.py --num-samples 5000
- GEMINI_API_KEY=... python scripts/generate_data.py --num-samples 100 --dry-run
- GEMINI_API_KEY=... python scripts/generate_data.py --num-samples 5000 --workers 32
-"""
-
-import argparse
-import concurrent.futures
-import json
-import os
-import random
-import re
-import sys
-from concurrent.futures import ThreadPoolExecutor
-
-from google import genai
-from tqdm import tqdm
-
-POOL_TIME_PRODUCTIVITY = [
- {"name": "set_timer", "description": "Set a timer for the specified duration or end time.", "parameters": {"time_human": {"type": "string", "description": "The duration or target end time in human readable format e.g. '1 hour and 30 minutes', '45 minutes', 'for 1:50pm', 'at 13:30'.", "required": True}}},
- {"name": "set_alarm", "description": "Set an alarm for a specified time.", "parameters": {"time_hours": {"type": "number", "description": "The hour component of the alarm time (24 hour time).", "required": True}, "time_minutes": {"type": "number", "description": "The minute component of the alarm time (0-59).", "required": True}, "label": {"type": "string", "description": "An optional label or title for the alarm.", "required": False}}},
- {"name": "create_reminder", "description": "Create a reminder for the user at a specific time.", "parameters": {"message": {"type": "string", "description": "The reminder message e.g. 'Buy more milk'.", "required": True}, "date_time_human": {"type": "string", "description": "The date and/or time for the reminder in human readable format e.g. 'tomorrow at 13:00', 'next Monday at 9am'.", "required": False}}},
- {"name": "create_calendar_event", "description": "Create a new calendar event with a title, start time, and optional end time.", "parameters": {"title": {"type": "string", "description": "The title of the calendar event.", "required": True}, "start_time_human": {"type": "string", "description": "The start date/time in human readable format e.g. 'tomorrow at 2pm', 'March 15 at 10:00'.", "required": True}, "end_time_human": {"type": "string", "description": "The end date/time in human readable format.", "required": False}, "location": {"type": "string", "description": "The location of the event.", "required": False}}},
- {"name": "stop_timer", "description": "Stop the currently running timer.", "parameters": {}},
- {"name": "snooze_alarm", "description": "Snooze the currently ringing alarm for a specified duration.", "parameters": {"minutes": {"type": "number", "description": "Number of minutes to snooze (default 5).", "required": False}}},
- {"name": "delete_alarm", "description": "Delete an existing alarm.", "parameters": {"label": {"type": "string", "description": "Label of the alarm to delete.", "required": False}, "time_hours": {"type": "number", "description": "Hour of the alarm to delete.", "required": False}}},
- {"name": "get_next_alarm", "description": "Get the time of the next scheduled alarm.", "parameters": {}},
- {"name": "delete_reminder", "description": "Delete a reminder by matching its message.", "parameters": {"search_text": {"type": "string", "description": "Text to match the reminder to delete.", "required": True}}},
- {"name": "get_upcoming_events", "description": "Get upcoming calendar events for today or a specified date.", "parameters": {"date_human": {"type": "string", "description": "Date in human readable format e.g. 'today', 'tomorrow', 'next Friday'. Defaults to today.", "required": False}}},
- {"name": "delete_calendar_event", "description": "Delete a calendar event by title.", "parameters": {"title": {"type": "string", "description": "Title of the event to delete.", "required": True}}},
- {"name": "start_stopwatch", "description": "Start a stopwatch.", "parameters": {}},
- {"name": "stop_stopwatch", "description": "Stop the running stopwatch and return elapsed time.", "parameters": {}},
- {"name": "get_current_time", "description": "Get the current date and time.", "parameters": {"timezone": {"type": "string", "description": "Optional timezone e.g. 'America/New_York', 'Europe/London'.", "required": False}}},
- {"name": "start_focus_mode", "description": "Start a focus/concentration session blocking notifications.", "parameters": {"duration_minutes": {"type": "number", "description": "Duration of focus session in minutes.", "required": True}, "allow_calls": {"type": "boolean", "description": "Whether to allow phone calls during focus.", "required": False}}},
-]
-
-POOL_LISTS_NOTES = [
- {"name": "create_list_item", "description": "Add a new item to a user's list (shopping, todo, etc.) with an optional reminder.", "parameters": {"list_name": {"type": "string", "description": "Short name of the list e.g. 'shopping', 'todo', 'groceries'.", "required": True}, "message": {"type": "string", "description": "The text of the list item.", "required": True}, "reminder_date_time_human": {"type": "string", "description": "Optional reminder date/time in human readable format.", "required": False}}},
- {"name": "create_note", "description": "Create a new note with the given text.", "parameters": {"text": {"type": "string", "description": "The text of the note.", "required": True}, "title": {"type": "string", "description": "Optional title for the note.", "required": False}}},
- {"name": "delete_list_item", "description": "Remove an item from a list by searching for it.", "parameters": {"list_name": {"type": "string", "description": "The name of the list.", "required": True}, "search_text": {"type": "string", "description": "Text to match the item to delete.", "required": True}}},
- {"name": "get_list_items", "description": "Retrieve all items from a specified list.", "parameters": {"list_name": {"type": "string", "description": "The name of the list to retrieve.", "required": True}}},
- {"name": "create_list", "description": "Create a new empty list.", "parameters": {"list_name": {"type": "string", "description": "Name for the new list.", "required": True}}},
- {"name": "delete_list", "description": "Delete an entire list.", "parameters": {"list_name": {"type": "string", "description": "Name of the list to delete.", "required": True}}},
- {"name": "search_notes", "description": "Search through existing notes by keyword.", "parameters": {"query": {"type": "string", "description": "Search keyword or phrase.", "required": True}}},
- {"name": "delete_note", "description": "Delete a note by matching its title or content.", "parameters": {"search_text": {"type": "string", "description": "Text to match the note to delete.", "required": True}}},
- {"name": "mark_list_item_done", "description": "Mark a list item as completed.", "parameters": {"list_name": {"type": "string", "description": "The list name.", "required": True}, "search_text": {"type": "string", "description": "Text to match the item to mark done.", "required": True}}},
- {"name": "share_list", "description": "Share a list with another contact.", "parameters": {"list_name": {"type": "string", "description": "The list to share.", "required": True}, "contact_id": {"type": "string", "description": "The contact to share with.", "required": True}}},
- {"name": "edit_note", "description": "Edit an existing note by appending or replacing its content.", "parameters": {"search_text": {"type": "string", "description": "Text to match the note to edit.", "required": True}, "new_text": {"type": "string", "description": "New text to append or replace with.", "required": True}, "mode": {"type": "string", "description": "'append' to add to existing or 'replace' to overwrite.", "required": False}}},
- {"name": "pin_note", "description": "Pin or unpin a note so it stays at the top.", "parameters": {"search_text": {"type": "string", "description": "Text to match the note.", "required": True}, "pinned": {"type": "boolean", "description": "True to pin, false to unpin.", "required": True}}},
- {"name": "tag_note", "description": "Add or remove a tag/label on a note.", "parameters": {"search_text": {"type": "string", "description": "Text to match the note.", "required": True}, "tag": {"type": "string", "description": "Tag name e.g. 'work', 'personal', 'urgent', 'ideas'.", "required": True}, "action": {"type": "string", "description": "'add' or 'remove'.", "required": False}}},
- {"name": "archive_note", "description": "Archive or unarchive a note.", "parameters": {"search_text": {"type": "string", "description": "Text to match the note.", "required": True}, "archive": {"type": "boolean", "description": "True to archive, false to unarchive.", "required": True}}},
- {"name": "create_voice_note", "description": "Record a voice note with optional transcription.", "parameters": {"title": {"type": "string", "description": "Optional title for the voice note.", "required": False}, "transcribe": {"type": "boolean", "description": "Whether to automatically transcribe the recording.", "required": False}}},
- {"name": "create_checklist", "description": "Create a note with a checklist of items.", "parameters": {"title": {"type": "string", "description": "Title for the checklist.", "required": True}, "items": {"type": "string", "description": "Comma-separated list of checklist items.", "required": True}}},
- {"name": "rename_list", "description": "Rename an existing list.", "parameters": {"list_name": {"type": "string", "description": "Current name of the list.", "required": True}, "new_name": {"type": "string", "description": "New name for the list.", "required": True}}},
- {"name": "sort_list", "description": "Sort list items by a given criteria.", "parameters": {"list_name": {"type": "string", "description": "The list to sort.", "required": True}, "sort_by": {"type": "string", "description": "'alphabetical', 'date_added', 'priority', or 'completed'.", "required": False}}},
- {"name": "duplicate_list", "description": "Create a copy of an existing list.", "parameters": {"list_name": {"type": "string", "description": "The list to duplicate.", "required": True}, "new_name": {"type": "string", "description": "Name for the duplicate.", "required": False}}},
- {"name": "export_note", "description": "Export a note as a text file or PDF.", "parameters": {"search_text": {"type": "string", "description": "Text to match the note to export.", "required": True}, "format": {"type": "string", "description": "'text' or 'pdf'.", "required": False}}},
- {"name": "get_all_notes", "description": "List all notes, optionally filtered by tag.", "parameters": {"tag": {"type": "string", "description": "Optional tag to filter by.", "required": False}}},
-]
-
-POOL_MESSAGING = [
- {"name": "send_instant_message", "description": "Send an instant message to a contact.", "parameters": {"contact_id": {"type": "string", "description": "The unique identifier of the recipient contact.", "required": True}, "text": {"type": "string", "description": "The message text to send.", "required": True}}},
- {"name": "search_for_contact", "description": "Search for a contact by name to get their contact ID.", "parameters": {"name": {"type": "string", "description": "The name of the contact to search for.", "required": True}}},
- {"name": "make_phone_call", "description": "Initiate a phone call to a contact.", "parameters": {"contact_id": {"type": "string", "description": "The unique identifier of the contact to call.", "required": True}}},
- {"name": "send_email", "description": "Send an email to a recipient.", "parameters": {"to": {"type": "string", "description": "The recipient's email address.", "required": True}, "subject": {"type": "string", "description": "The email subject line.", "required": True}, "body": {"type": "string", "description": "The email body text.", "required": True}}},
- {"name": "send_sms", "description": "Send an SMS text message to a phone number.", "parameters": {"phone_number": {"type": "string", "description": "The recipient phone number.", "required": True}, "text": {"type": "string", "description": "The message text.", "required": True}}},
- {"name": "get_recent_messages", "description": "Get recent messages from a contact or all contacts.", "parameters": {"contact_id": {"type": "string", "description": "Optional contact ID to filter messages.", "required": False}, "count": {"type": "number", "description": "Number of recent messages to retrieve (default 10).", "required": False}}},
- {"name": "get_call_history", "description": "Get recent call history.", "parameters": {"count": {"type": "number", "description": "Number of recent calls to show (default 10).", "required": False}}},
- {"name": "create_contact", "description": "Create a new contact entry.", "parameters": {"name": {"type": "string", "description": "Full name of the contact.", "required": True}, "phone": {"type": "string", "description": "Phone number.", "required": False}, "email": {"type": "string", "description": "Email address.", "required": False}}},
- {"name": "block_contact", "description": "Block or unblock a contact.", "parameters": {"contact_id": {"type": "string", "description": "The contact to block/unblock.", "required": True}, "action": {"type": "string", "description": "'block' or 'unblock'.", "required": True}}},
- {"name": "start_video_call", "description": "Start a video call with a contact.", "parameters": {"contact_id": {"type": "string", "description": "The contact to video call.", "required": True}}},
-]
-
-POOL_DEVICE_CONTROL = [
- {"name": "set_brightness", "description": "Set the screen brightness level.", "parameters": {"level": {"type": "number", "description": "Brightness level from 0 to 100.", "required": True}}},
- {"name": "set_volume", "description": "Set the device volume level.", "parameters": {"level": {"type": "number", "description": "Volume level from 0 to 100.", "required": True}, "stream": {"type": "string", "description": "Which audio stream: 'media', 'ringtone', 'alarm', 'notification'.", "required": False}}},
- {"name": "toggle_wifi", "description": "Turn Wi-Fi on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "toggle_bluetooth", "description": "Turn Bluetooth on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "toggle_flashlight", "description": "Turn the flashlight on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to turn on, false to turn off.", "required": True}}},
- {"name": "toggle_do_not_disturb", "description": "Enable or disable Do Not Disturb mode.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable DND, false to disable.", "required": True}, "duration_minutes": {"type": "number", "description": "Optional duration in minutes before auto-disabling.", "required": False}}},
- {"name": "set_screen_timeout", "description": "Set the screen auto-lock timeout duration.", "parameters": {"seconds": {"type": "number", "description": "Timeout in seconds (e.g. 30, 60, 120, 300).", "required": True}}},
- {"name": "toggle_auto_brightness", "description": "Enable or disable automatic brightness adjustment.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "toggle_auto_rotate", "description": "Enable or disable auto screen rotation.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "toggle_hotspot", "description": "Enable or disable the mobile hotspot.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}, "password": {"type": "string", "description": "Optional hotspot password.", "required": False}}},
- {"name": "toggle_power_saving", "description": "Enable or disable battery power saving mode.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "connect_bluetooth_device", "description": "Connect to a specific Bluetooth device by name.", "parameters": {"device_name": {"type": "string", "description": "Name of the Bluetooth device e.g. 'AirPods Pro', 'JBL Speaker'.", "required": True}}},
- {"name": "connect_wifi_network", "description": "Connect to a specific Wi-Fi network.", "parameters": {"ssid": {"type": "string", "description": "Name of the Wi-Fi network.", "required": True}, "password": {"type": "string", "description": "Wi-Fi password if required.", "required": False}}},
- {"name": "toggle_night_mode", "description": "Enable or disable night/blue light filter.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
-]
-
-POOL_MEDIA = [
- {"name": "play_music", "description": "Play music by song name, artist, album, or genre.", "parameters": {"query": {"type": "string", "description": "Search query: song title, artist name, album, or genre.", "required": True}}},
- {"name": "pause_media", "description": "Pause the currently playing media.", "parameters": {}},
- {"name": "resume_media", "description": "Resume the paused media playback.", "parameters": {}},
- {"name": "skip_track", "description": "Skip to the next track in the current playlist.", "parameters": {}},
- {"name": "previous_track", "description": "Go back to the previous track.", "parameters": {}},
- {"name": "play_podcast", "description": "Play a podcast by name or topic.", "parameters": {"query": {"type": "string", "description": "Podcast name or topic to search for.", "required": True}}},
- {"name": "play_radio", "description": "Play a radio station by name or frequency.", "parameters": {"station": {"type": "string", "description": "Station name or frequency e.g. 'NPR', '101.1 FM'.", "required": True}}},
- {"name": "play_audiobook", "description": "Play an audiobook by title or author.", "parameters": {"query": {"type": "string", "description": "Audiobook title or author name.", "required": True}}},
- {"name": "set_repeat_mode", "description": "Set the media repeat mode.", "parameters": {"mode": {"type": "string", "description": "'off', 'one', or 'all'.", "required": True}}},
- {"name": "set_shuffle", "description": "Enable or disable shuffle playback.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable shuffle, false to disable.", "required": True}}},
- {"name": "get_now_playing", "description": "Get info about the currently playing media.", "parameters": {}},
- {"name": "add_to_playlist", "description": "Add the current song to a playlist.", "parameters": {"playlist_name": {"type": "string", "description": "Name of the playlist.", "required": True}}},
- {"name": "play_playlist", "description": "Play a specific playlist by name.", "parameters": {"playlist_name": {"type": "string", "description": "Name of the playlist to play.", "required": True}}},
- {"name": "cast_media", "description": "Cast current media to a speaker or TV.", "parameters": {"device_name": {"type": "string", "description": "Name of the cast target device e.g. 'Living Room TV', 'Kitchen Speaker'.", "required": True}}},
-]
-
-POOL_NAVIGATION = [
- {"name": "get_directions", "description": "Get directions to a destination.", "parameters": {"destination": {"type": "string", "description": "The destination address or place name.", "required": True}, "mode": {"type": "string", "description": "Travel mode: 'driving', 'walking', 'transit', 'cycling'.", "required": False}}},
- {"name": "share_location", "description": "Share current location with a contact.", "parameters": {"contact_id": {"type": "string", "description": "The contact to share location with.", "required": True}, "duration_minutes": {"type": "number", "description": "How long to share location in minutes.", "required": False}}},
- {"name": "find_nearby", "description": "Search for nearby places by category.", "parameters": {"category": {"type": "string", "description": "Type of place e.g. 'gas station', 'restaurant', 'pharmacy', 'coffee shop'.", "required": True}}},
- {"name": "get_eta", "description": "Get estimated time of arrival to a destination.", "parameters": {"destination": {"type": "string", "description": "The destination.", "required": True}, "mode": {"type": "string", "description": "Travel mode.", "required": False}}},
- {"name": "save_location", "description": "Save a location as a favorite or bookmark.", "parameters": {"name": {"type": "string", "description": "Name for the saved location e.g. 'Home', 'Work', 'Gym'.", "required": True}, "address": {"type": "string", "description": "The address to save.", "required": True}}},
- {"name": "get_traffic", "description": "Get current traffic conditions for a route.", "parameters": {"destination": {"type": "string", "description": "The destination to check traffic for.", "required": True}}},
- {"name": "start_navigation", "description": "Start turn-by-turn navigation to a destination.", "parameters": {"destination": {"type": "string", "description": "The destination.", "required": True}, "mode": {"type": "string", "description": "Travel mode.", "required": False}, "avoid_tolls": {"type": "boolean", "description": "Whether to avoid toll roads.", "required": False}}},
- {"name": "stop_navigation", "description": "Stop the current navigation.", "parameters": {}},
- {"name": "report_traffic_incident", "description": "Report a traffic incident at current location.", "parameters": {"type": {"type": "string", "description": "Type of incident: 'accident', 'construction', 'hazard', 'police'.", "required": True}}},
-]
-
-POOL_SMART_HOME = [
- {"name": "set_thermostat", "description": "Set the home thermostat to a target temperature.", "parameters": {"temperature": {"type": "number", "description": "Target temperature in degrees.", "required": True}, "unit": {"type": "string", "description": "'fahrenheit' or 'celsius'.", "required": False}}},
- {"name": "control_lights", "description": "Turn lights on or off in a specific room, optionally setting brightness or color.", "parameters": {"room": {"type": "string", "description": "The room name e.g. 'bedroom', 'kitchen', 'living room'.", "required": True}, "action": {"type": "string", "description": "'on', 'off', or 'dim'.", "required": True}, "brightness": {"type": "number", "description": "Brightness 0-100 (for 'on' or 'dim').", "required": False}, "color": {"type": "string", "description": "Optional light color e.g. 'warm white', 'cool white', 'red', 'blue'.", "required": False}}},
- {"name": "lock_door", "description": "Lock or unlock a smart door lock.", "parameters": {"door": {"type": "string", "description": "Which door e.g. 'front door', 'back door', 'garage'.", "required": True}, "action": {"type": "string", "description": "'lock' or 'unlock'.", "required": True}}},
- {"name": "start_robot_vacuum", "description": "Start or stop the robot vacuum cleaner.", "parameters": {"action": {"type": "string", "description": "'start', 'stop', or 'dock'.", "required": True}, "room": {"type": "string", "description": "Optional specific room to clean.", "required": False}}},
- {"name": "control_fan", "description": "Control a smart fan.", "parameters": {"room": {"type": "string", "description": "Room where the fan is.", "required": True}, "action": {"type": "string", "description": "'on', 'off'.", "required": True}, "speed": {"type": "string", "description": "'low', 'medium', 'high'.", "required": False}}},
- {"name": "control_blinds", "description": "Open or close smart blinds/curtains.", "parameters": {"room": {"type": "string", "description": "Room name.", "required": True}, "action": {"type": "string", "description": "'open', 'close', or a percentage (0-100).", "required": True}}},
- {"name": "arm_security_system", "description": "Arm or disarm the home security system.", "parameters": {"action": {"type": "string", "description": "'arm_home', 'arm_away', or 'disarm'.", "required": True}}},
- {"name": "get_security_camera_feed", "description": "View a security camera feed.", "parameters": {"camera_name": {"type": "string", "description": "Name of the camera e.g. 'front porch', 'backyard', 'driveway'.", "required": True}}},
- {"name": "control_sprinklers", "description": "Start or stop the garden sprinkler system.", "parameters": {"action": {"type": "string", "description": "'start' or 'stop'.", "required": True}, "zone": {"type": "string", "description": "Optional zone name e.g. 'front yard', 'backyard'.", "required": False}, "duration_minutes": {"type": "number", "description": "How long to run in minutes.", "required": False}}},
- {"name": "set_light_scene", "description": "Activate a predefined lighting scene.", "parameters": {"scene_name": {"type": "string", "description": "Scene name e.g. 'movie', 'dinner', 'bedtime', 'morning', 'party'.", "required": True}}},
- {"name": "control_garage_door", "description": "Open or close the garage door.", "parameters": {"action": {"type": "string", "description": "'open' or 'close'.", "required": True}}},
- {"name": "get_indoor_temperature", "description": "Get the current indoor temperature reading.", "parameters": {"room": {"type": "string", "description": "Optional room name for a specific sensor.", "required": False}}},
-]
-
-POOL_UTILITY = [
- {"name": "evaluate_js", "description": "Evaluate a JavaScript expression for calculations, math, date, or string manipulation. Use console.log() to output.", "parameters": {"js": {"type": "string", "description": "The JavaScript code to evaluate.", "required": True}}},
- {"name": "web_search", "description": "Search the web for information.", "parameters": {"query": {"type": "string", "description": "The search query.", "required": True}}},
- {"name": "take_screenshot", "description": "Take a screenshot of the current screen.", "parameters": {}},
- {"name": "open_app", "description": "Open an application by name.", "parameters": {"app_name": {"type": "string", "description": "The name of the app to open.", "required": True}}},
- {"name": "close_app", "description": "Close a running application.", "parameters": {"app_name": {"type": "string", "description": "The name of the app to close.", "required": True}}},
- {"name": "set_wallpaper", "description": "Set the device wallpaper from a URL or built-in option.", "parameters": {"source": {"type": "string", "description": "URL or built-in wallpaper name.", "required": True}}},
- {"name": "translate_text", "description": "Translate text from one language to another.", "parameters": {"text": {"type": "string", "description": "The text to translate.", "required": True}, "target_language": {"type": "string", "description": "Target language code e.g. 'es', 'fr', 'de', 'ja'.", "required": True}, "source_language": {"type": "string", "description": "Source language code (auto-detected if omitted).", "required": False}}},
- {"name": "get_weather", "description": "Get current weather or forecast for a location.", "parameters": {"location": {"type": "string", "description": "City name or location.", "required": True}}},
- {"name": "get_weather_forecast", "description": "Get a multi-day weather forecast.", "parameters": {"location": {"type": "string", "description": "City name or location.", "required": True}, "days": {"type": "number", "description": "Number of forecast days (1-7).", "required": False}}},
- {"name": "set_clipboard", "description": "Copy text to the clipboard.", "parameters": {"text": {"type": "string", "description": "Text to copy.", "required": True}}},
- {"name": "get_clipboard", "description": "Get the current clipboard contents.", "parameters": {}},
- {"name": "define_word", "description": "Look up the definition of a word.", "parameters": {"word": {"type": "string", "description": "The word to define.", "required": True}}},
- {"name": "unit_convert", "description": "Convert between units of measurement.", "parameters": {"value": {"type": "number", "description": "The value to convert.", "required": True}, "from_unit": {"type": "string", "description": "Source unit e.g. 'miles', 'kg', 'fahrenheit'.", "required": True}, "to_unit": {"type": "string", "description": "Target unit e.g. 'km', 'lbs', 'celsius'.", "required": True}}},
- {"name": "scan_qr_code", "description": "Open the camera to scan a QR code.", "parameters": {}},
- {"name": "create_qr_code", "description": "Generate a QR code for given text or URL.", "parameters": {"content": {"type": "string", "description": "Text or URL to encode.", "required": True}}},
-]
-
-POOL_CAMERA_PHOTOS = [
- {"name": "take_photo", "description": "Take a photo using the device camera.", "parameters": {"camera": {"type": "string", "description": "'front' or 'back' camera.", "required": False}, "timer_seconds": {"type": "number", "description": "Optional countdown timer in seconds before taking the photo.", "required": False}}},
- {"name": "record_video", "description": "Start or stop recording a video.", "parameters": {"action": {"type": "string", "description": "'start' or 'stop'.", "required": True}, "camera": {"type": "string", "description": "'front' or 'back' camera.", "required": False}}},
- {"name": "open_gallery", "description": "Open the photo gallery or a specific album.", "parameters": {"album": {"type": "string", "description": "Optional album name to open.", "required": False}}},
- {"name": "share_photo", "description": "Share the most recent photo or a specified photo with a contact.", "parameters": {"contact_id": {"type": "string", "description": "The contact to share with.", "required": True}, "photo_description": {"type": "string", "description": "Description to identify which photo, e.g. 'last photo', 'screenshot'.", "required": False}}},
- {"name": "take_panorama", "description": "Take a panoramic photo.", "parameters": {"direction": {"type": "string", "description": "Optional sweep direction e.g. 'left', 'right'.", "required": False}}},
- {"name": "take_burst_photos", "description": "Take a burst of photos in quick succession.", "parameters": {"count": {"type": "number", "description": "Optional number of photos to take in the burst.", "required": False}}},
- {"name": "edit_photo", "description": "Edit a photo with a specified action.", "parameters": {"action": {"type": "string", "description": "'crop', 'rotate', or 'filter'.", "required": True}, "filter_name": {"type": "string", "description": "Optional filter name when action is 'filter' e.g. 'sepia', 'noir', 'vivid'.", "required": False}}},
- {"name": "delete_photo", "description": "Delete a photo by description.", "parameters": {"photo_description": {"type": "string", "description": "Description to identify the photo to delete.", "required": True}}},
- {"name": "create_photo_album", "description": "Create a new photo album.", "parameters": {"album_name": {"type": "string", "description": "Name for the new album.", "required": True}}},
- {"name": "search_photos", "description": "Search photos by keyword or description.", "parameters": {"query": {"type": "string", "description": "Search query e.g. 'beach', 'sunset', 'birthday'.", "required": True}}},
-]
-
-POOL_FITNESS_HEALTH = [
- {"name": "start_workout", "description": "Start tracking a workout activity.", "parameters": {"workout_type": {"type": "string", "description": "Type of workout e.g. 'running', 'cycling', 'walking', 'strength', 'yoga'.", "required": True}}},
- {"name": "stop_workout", "description": "Stop the currently active workout tracking.", "parameters": {}},
- {"name": "log_water_intake", "description": "Log water consumption.", "parameters": {"amount_ml": {"type": "number", "description": "Amount of water in milliliters.", "required": True}}},
- {"name": "get_step_count", "description": "Get the current step count for today.", "parameters": {}},
- {"name": "start_sleep_tracking", "description": "Start or stop sleep tracking.", "parameters": {"action": {"type": "string", "description": "'start' or 'stop'.", "required": True}}},
- {"name": "log_meal", "description": "Log a meal or food item for nutrition tracking.", "parameters": {"description": {"type": "string", "description": "Description of the food or meal.", "required": True}, "meal_type": {"type": "string", "description": "'breakfast', 'lunch', 'dinner', or 'snack'.", "required": False}}},
- {"name": "get_heart_rate_history", "description": "Get heart rate history over a period.", "parameters": {"period": {"type": "string", "description": "Optional period e.g. 'today', 'this_week', 'this_month'.", "required": False}}},
- {"name": "set_step_goal", "description": "Set a daily step goal.", "parameters": {"steps": {"type": "number", "description": "The target number of steps per day.", "required": True}}},
- {"name": "get_sleep_summary", "description": "Get a summary of sleep data for a given date.", "parameters": {"date": {"type": "string", "description": "Optional date in human readable format e.g. 'last night', 'yesterday'. Defaults to last night.", "required": False}}},
- {"name": "log_blood_pressure", "description": "Log a blood pressure reading.", "parameters": {"systolic": {"type": "number", "description": "The systolic pressure value.", "required": True}, "diastolic": {"type": "number", "description": "The diastolic pressure value.", "required": True}}},
-]
-
-POOL_SYSTEM = [
- {"name": "toggle_airplane_mode", "description": "Turn airplane mode on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "clear_notifications", "description": "Clear all or specific app notifications.", "parameters": {"app_name": {"type": "string", "description": "Optional app name to clear notifications for. Clears all if omitted.", "required": False}}},
- {"name": "open_settings", "description": "Open device settings or a specific settings page.", "parameters": {"page": {"type": "string", "description": "Optional settings page e.g. 'wifi', 'bluetooth', 'display', 'battery', 'storage'.", "required": False}}},
- {"name": "check_battery", "description": "Check the current battery level and charging status.", "parameters": {}},
- {"name": "toggle_dark_mode", "description": "Enable or disable dark mode.", "parameters": {"enabled": {"type": "boolean", "description": "True for dark mode, false for light mode.", "required": True}}},
- {"name": "toggle_location_services", "description": "Turn location services on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "restart_device", "description": "Restart or shut down the device.", "parameters": {"action": {"type": "string", "description": "'restart' or 'shutdown'.", "required": True}}},
-]
-
-POOL_FINANCE = [
- {"name": "send_payment", "description": "Send a payment to a contact.", "parameters": {"contact_id": {"type": "string", "description": "The recipient contact ID.", "required": True}, "amount": {"type": "number", "description": "The amount to send.", "required": True}, "currency": {"type": "string", "description": "Currency code e.g. 'USD', 'EUR', 'GBP'.", "required": False}, "note": {"type": "string", "description": "Optional payment note e.g. 'for dinner'.", "required": False}}},
- {"name": "check_balance", "description": "Check the current account balance.", "parameters": {"account": {"type": "string", "description": "Account name e.g. 'checking', 'savings', 'credit card'.", "required": False}}},
- {"name": "convert_currency", "description": "Convert an amount between currencies.", "parameters": {"amount": {"type": "number", "description": "The amount to convert.", "required": True}, "from_currency": {"type": "string", "description": "Source currency code e.g. 'USD'.", "required": True}, "to_currency": {"type": "string", "description": "Target currency code e.g. 'EUR'.", "required": True}}},
- {"name": "request_payment", "description": "Request a payment from a contact.", "parameters": {"contact_id": {"type": "string", "description": "The contact to request payment from.", "required": True}, "amount": {"type": "number", "description": "The amount to request.", "required": True}, "note": {"type": "string", "description": "Optional note for the request.", "required": False}}},
- {"name": "get_recent_transactions", "description": "Get a list of recent transactions.", "parameters": {"count": {"type": "number", "description": "Number of transactions to retrieve.", "required": False}, "account": {"type": "string", "description": "Optional account name to filter by.", "required": False}}},
- {"name": "set_budget", "description": "Set a spending budget for a category.", "parameters": {"category": {"type": "string", "description": "Budget category e.g. 'food', 'entertainment', 'transport'.", "required": True}, "amount": {"type": "number", "description": "The budget amount.", "required": True}, "period": {"type": "string", "description": "Optional budget period e.g. 'weekly', 'monthly'.", "required": False}}},
- {"name": "get_spending_summary", "description": "Get a summary of spending for a period.", "parameters": {"period": {"type": "string", "description": "Optional period e.g. 'today', 'this_week', 'this_month'.", "required": False}}},
- {"name": "pay_bill", "description": "Pay a bill to a biller.", "parameters": {"biller_name": {"type": "string", "description": "Name of the biller e.g. 'electric company', 'internet provider'.", "required": True}, "amount": {"type": "number", "description": "Optional amount to pay. Uses default if omitted.", "required": False}}},
- {"name": "split_bill", "description": "Split a bill among contacts.", "parameters": {"contact_ids": {"type": "string", "description": "Comma-separated contact IDs to split with.", "required": True}, "total_amount": {"type": "number", "description": "The total bill amount to split.", "required": True}, "note": {"type": "string", "description": "Optional note for the split.", "required": False}}},
- {"name": "get_stock_price", "description": "Get the current price of a stock.", "parameters": {"symbol": {"type": "string", "description": "The stock ticker symbol e.g. 'AAPL', 'GOOGL'.", "required": True}}},
-]
-
-POOL_READING_NEWS = [
- {"name": "get_news_headlines", "description": "Get the latest news headlines, optionally filtered by topic.", "parameters": {"topic": {"type": "string", "description": "Optional topic filter e.g. 'technology', 'sports', 'politics', 'business'.", "required": False}}},
- {"name": "read_aloud", "description": "Read text content aloud using text-to-speech.", "parameters": {"text": {"type": "string", "description": "The text to read aloud.", "required": True}, "speed": {"type": "number", "description": "Speech rate multiplier (0.5 = slow, 1.0 = normal, 2.0 = fast).", "required": False}}},
- {"name": "summarize_page", "description": "Summarize the content of the currently open web page or article.", "parameters": {}},
- {"name": "save_article", "description": "Save the current article for later reading.", "parameters": {"title": {"type": "string", "description": "Optional title to save the article under.", "required": False}}},
- {"name": "get_saved_articles", "description": "Retrieve the list of saved articles.", "parameters": {}},
- {"name": "subscribe_newsletter", "description": "Subscribe to a newsletter by topic.", "parameters": {"topic": {"type": "string", "description": "The newsletter topic to subscribe to.", "required": True}}},
- {"name": "open_ebook", "description": "Open an ebook by title.", "parameters": {"title": {"type": "string", "description": "The title of the ebook to open.", "required": True}}},
- {"name": "set_reading_goal", "description": "Set a daily reading goal in pages.", "parameters": {"pages_per_day": {"type": "number", "description": "Number of pages to read per day.", "required": True}}},
- {"name": "get_reading_progress", "description": "Get current reading progress toward the daily goal.", "parameters": {}},
- {"name": "listen_to_article", "description": "Listen to an article using text-to-speech.", "parameters": {"speed": {"type": "number", "description": "Optional playback speed multiplier (0.5 to 2.0).", "required": False}}},
-]
-
-POOL_ACCESSIBILITY = [
- {"name": "set_font_size", "description": "Set the system font size.", "parameters": {"size": {"type": "string", "description": "'small', 'medium', 'large', or 'extra_large'.", "required": True}}},
- {"name": "toggle_voice_assistant", "description": "Enable or disable the voice assistant listener.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "toggle_magnifier", "description": "Enable or disable the screen magnifier.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "toggle_screen_reader", "description": "Enable or disable the screen reader for accessibility.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
-]
-
-POOL_SHOPPING = [
- {"name": "search_product", "description": "Search for a product to buy online.", "parameters": {"query": {"type": "string", "description": "Product search query.", "required": True}}},
- {"name": "add_to_cart", "description": "Add a product to the shopping cart.", "parameters": {"product_name": {"type": "string", "description": "Name or description of the product.", "required": True}, "quantity": {"type": "number", "description": "Number of items to add.", "required": False}}},
- {"name": "check_order_status", "description": "Check the status of a recent order.", "parameters": {"order_id": {"type": "string", "description": "The order ID to check. If omitted, checks the most recent order.", "required": False}}},
-]
-
-POOL_SOCIAL = [
- {"name": "post_status_update", "description": "Post a status update or message to social media.", "parameters": {"text": {"type": "string", "description": "The status text to post.", "required": True}, "platform": {"type": "string", "description": "Social platform e.g. 'twitter', 'facebook', 'instagram'.", "required": False}}},
- {"name": "check_social_notifications", "description": "Check recent social media notifications.", "parameters": {"platform": {"type": "string", "description": "Optional platform filter.", "required": False}}},
- {"name": "like_post", "description": "Like a post on social media.", "parameters": {"post_id": {"type": "string", "description": "The ID of the post to like.", "required": True}}},
- {"name": "comment_on_post", "description": "Leave a comment on a social media post.", "parameters": {"post_id": {"type": "string", "description": "The ID of the post to comment on.", "required": True}, "text": {"type": "string", "description": "The comment text.", "required": True}}},
- {"name": "share_post", "description": "Share a post to another platform or feed.", "parameters": {"post_id": {"type": "string", "description": "The ID of the post to share.", "required": True}, "platform": {"type": "string", "description": "Optional target platform e.g. 'twitter', 'facebook'.", "required": False}}},
- {"name": "follow_user", "description": "Follow a user on social media.", "parameters": {"username": {"type": "string", "description": "The username to follow.", "required": True}}},
- {"name": "unfollow_user", "description": "Unfollow a user on social media.", "parameters": {"username": {"type": "string", "description": "The username to unfollow.", "required": True}}},
- {"name": "get_trending_topics", "description": "Get currently trending topics on social media.", "parameters": {"platform": {"type": "string", "description": "Optional platform filter.", "required": False}}},
- {"name": "send_direct_message", "description": "Send a direct message to a user on social media.", "parameters": {"username": {"type": "string", "description": "The recipient username.", "required": True}, "text": {"type": "string", "description": "The message text.", "required": True}}},
- {"name": "check_direct_messages", "description": "Check recent direct messages on social media.", "parameters": {"platform": {"type": "string", "description": "Optional platform filter.", "required": False}}},
-]
-
-POOL_RIDE_DELIVERY = [
- {"name": "request_ride", "description": "Request a ride to a destination.", "parameters": {"destination": {"type": "string", "description": "The destination address or place name.", "required": True}, "ride_type": {"type": "string", "description": "'standard', 'xl', or 'shared'.", "required": False}}},
- {"name": "cancel_ride", "description": "Cancel the current ride request.", "parameters": {}},
- {"name": "track_ride_eta", "description": "Track the ETA of the current ride.", "parameters": {}},
- {"name": "rate_ride", "description": "Rate a completed ride.", "parameters": {"rating": {"type": "number", "description": "Rating from 1 to 5.", "required": True}, "feedback": {"type": "string", "description": "Optional feedback text.", "required": False}}},
- {"name": "order_food", "description": "Order food delivery from a restaurant.", "parameters": {"restaurant": {"type": "string", "description": "Name of the restaurant.", "required": True}, "items": {"type": "string", "description": "Description of what to order.", "required": True}}},
- {"name": "track_food_delivery", "description": "Track the status of a food delivery order.", "parameters": {}},
-]
-
-POOL_FILE_MANAGEMENT = [
- {"name": "open_file", "description": "Open a file by name.", "parameters": {"file_name": {"type": "string", "description": "Name of the file to open.", "required": True}}},
- {"name": "share_file", "description": "Share a file with a contact.", "parameters": {"file_name": {"type": "string", "description": "Name of the file to share.", "required": True}, "contact_id": {"type": "string", "description": "The contact to share with.", "required": True}}},
- {"name": "download_file", "description": "Download a file from a URL.", "parameters": {"url": {"type": "string", "description": "The URL to download from.", "required": True}}},
- {"name": "move_file", "description": "Move a file to a different folder.", "parameters": {"file_name": {"type": "string", "description": "Name of the file to move.", "required": True}, "destination_folder": {"type": "string", "description": "The destination folder path.", "required": True}}},
- {"name": "compress_files", "description": "Compress files into an archive.", "parameters": {"file_names": {"type": "string", "description": "Description of files to compress.", "required": True}, "archive_name": {"type": "string", "description": "Name for the resulting archive.", "required": True}}},
- {"name": "create_folder", "description": "Create a new folder.", "parameters": {"folder_name": {"type": "string", "description": "Name of the folder to create.", "required": True}}},
- {"name": "delete_file", "description": "Delete a file.", "parameters": {"file_name": {"type": "string", "description": "Name of the file to delete.", "required": True}}},
-]
-
-POOL_WEARABLE = [
- {"name": "check_heart_rate", "description": "Check the current heart rate from the wearable sensor.", "parameters": {}},
- {"name": "check_blood_oxygen", "description": "Check the current blood oxygen level.", "parameters": {}},
- {"name": "start_breathing_exercise", "description": "Start a guided breathing exercise.", "parameters": {"duration_minutes": {"type": "number", "description": "Duration of the exercise in minutes.", "required": False}}},
- {"name": "find_my_phone", "description": "Trigger the phone to ring so you can find it.", "parameters": {}},
- {"name": "toggle_always_on_display", "description": "Enable or disable the always-on display.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "change_watch_face", "description": "Change the watch face.", "parameters": {"face_name": {"type": "string", "description": "Name of the watch face to switch to.", "required": True}}},
- {"name": "set_activity_goal", "description": "Set a daily activity goal.", "parameters": {"goal_type": {"type": "string", "description": "'steps', 'calories', or 'distance'.", "required": True}, "value": {"type": "number", "description": "The target value for the goal.", "required": True}}},
- {"name": "check_activity_progress", "description": "Check progress toward the current activity goal.", "parameters": {}},
- {"name": "toggle_fall_detection", "description": "Enable or disable fall detection.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "trigger_emergency_sos", "description": "Trigger an emergency SOS alert.", "parameters": {}},
-]
-
-POOL_DESKTOP = [
- {"name": "minimize_window", "description": "Minimize the current window.", "parameters": {}},
- {"name": "maximize_window", "description": "Maximize the current window.", "parameters": {}},
- {"name": "split_screen", "description": "Snap the current window to a screen position.", "parameters": {"position": {"type": "string", "description": "'left' or 'right'.", "required": True}}},
- {"name": "switch_virtual_desktop", "description": "Switch to another virtual desktop.", "parameters": {"direction": {"type": "string", "description": "'next', 'previous', or a desktop number.", "required": True}}},
- {"name": "toggle_vpn", "description": "Turn VPN on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}, "server": {"type": "string", "description": "Optional VPN server name or location.", "required": False}}},
- {"name": "switch_audio_output", "description": "Switch the audio output device.", "parameters": {"device_name": {"type": "string", "description": "Name of the output device e.g. 'External Speakers', 'HDMI'.", "required": True}}},
- {"name": "switch_audio_input", "description": "Switch the audio input device.", "parameters": {"device_name": {"type": "string", "description": "Name of the input device e.g. 'External Mic', 'Webcam'.", "required": True}}},
- {"name": "print_document", "description": "Print the current document.", "parameters": {"printer_name": {"type": "string", "description": "Optional printer name.", "required": False}, "copies": {"type": "number", "description": "Number of copies to print.", "required": False}}},
- {"name": "check_system_updates", "description": "Check for available system updates.", "parameters": {}},
- {"name": "install_system_updates", "description": "Install available system updates.", "parameters": {}},
- {"name": "take_screenshot_region", "description": "Take a screenshot of a specific region.", "parameters": {"region": {"type": "string", "description": "'full', 'window', or 'selection'.", "required": False}}},
- {"name": "kill_process", "description": "Force quit an application by name.", "parameters": {"app_name": {"type": "string", "description": "Name of the application to kill.", "required": True}}},
- {"name": "check_system_resources", "description": "Check current CPU, memory, and disk usage.", "parameters": {}},
-]
-
-POOL_BROWSER = [
- {"name": "open_tab", "description": "Open a new browser tab.", "parameters": {"url": {"type": "string", "description": "Optional URL to open.", "required": False}, "query": {"type": "string", "description": "Optional search query to open.", "required": False}}},
- {"name": "close_tab", "description": "Close the current browser tab.", "parameters": {}},
- {"name": "bookmark_page", "description": "Bookmark the current page.", "parameters": {}},
- {"name": "clear_browsing_data", "description": "Clear browsing data.", "parameters": {"data_type": {"type": "string", "description": "'history', 'cookies', 'cache', or 'all'.", "required": True}}},
- {"name": "open_private_window", "description": "Open a new private/incognito browser window.", "parameters": {}},
- {"name": "find_in_page", "description": "Find text on the current page.", "parameters": {"text": {"type": "string", "description": "The text to search for.", "required": True}}},
-]
-
-POOL_EMERGENCY_SAFETY = [
- {"name": "call_emergency_services", "description": "Call emergency services.", "parameters": {"service_type": {"type": "string", "description": "'police', 'fire', or 'ambulance'.", "required": False}}},
- {"name": "share_emergency_location", "description": "Share current location with an emergency contact.", "parameters": {"contact_id": {"type": "string", "description": "The emergency contact to share location with.", "required": True}}},
- {"name": "set_emergency_contact", "description": "Set up an emergency contact.", "parameters": {"name": {"type": "string", "description": "Full name of the emergency contact.", "required": True}, "phone": {"type": "string", "description": "Phone number of the emergency contact.", "required": True}}},
- {"name": "toggle_medical_id", "description": "Enable or disable the medical ID display.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "send_safety_check", "description": "Send a safety check-in to a contact.", "parameters": {"contact_id": {"type": "string", "description": "The contact to send the safety check to.", "required": True}, "message": {"type": "string", "description": "Optional message to include.", "required": False}}},
- {"name": "get_emergency_contacts", "description": "Get the list of emergency contacts.", "parameters": {}},
- {"name": "report_safety_issue", "description": "Report a safety issue or hazard.", "parameters": {"description": {"type": "string", "description": "Description of the safety issue.", "required": True}, "location": {"type": "string", "description": "Optional location of the issue.", "required": False}}},
- {"name": "enable_crash_detection", "description": "Enable or disable automatic crash detection.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
-]
-
-POOL_DIGITAL_WELLBEING = [
- {"name": "check_screen_time", "description": "Check screen time usage.", "parameters": {"period": {"type": "string", "description": "'today' or 'week'.", "required": False}}},
- {"name": "set_app_timer", "description": "Set a usage time limit for an app.", "parameters": {"app_name": {"type": "string", "description": "Name of the app to limit.", "required": True}, "duration_minutes": {"type": "number", "description": "Maximum usage time in minutes.", "required": True}}},
- {"name": "toggle_bedtime_mode", "description": "Enable or disable bedtime mode.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "set_content_restriction", "description": "Set content restriction level.", "parameters": {"restriction_level": {"type": "string", "description": "'off', 'moderate', or 'strict'.", "required": True}}},
- {"name": "check_app_usage", "description": "Check usage statistics for an app.", "parameters": {"app_name": {"type": "string", "description": "Optional app name to check. Shows all apps if omitted.", "required": False}}},
- {"name": "pause_all_notifications", "description": "Temporarily pause all notifications.", "parameters": {"duration_minutes": {"type": "number", "description": "Optional duration in minutes before resuming notifications.", "required": False}}},
- {"name": "get_daily_summary", "description": "Get a daily summary of device usage and wellbeing stats.", "parameters": {}},
- {"name": "set_downtime_schedule", "description": "Set a recurring downtime schedule to limit device use.", "parameters": {"start_time": {"type": "string", "description": "Start time for downtime e.g. '22:00'.", "required": True}, "end_time": {"type": "string", "description": "End time for downtime e.g. '07:00'.", "required": True}}},
-]
-
-POOL_TV_STREAMING = [
- {"name": "play_show", "description": "Play a TV show or movie by title on a streaming service.", "parameters": {"title": {"type": "string", "description": "Title of the show or movie.", "required": True}, "service": {"type": "string", "description": "Streaming service e.g. 'netflix', 'hulu', 'disney+', 'prime'.", "required": False}}},
- {"name": "switch_tv_input", "description": "Switch the TV input source.", "parameters": {"input": {"type": "string", "description": "Input source e.g. 'HDMI 1', 'HDMI 2', 'USB', 'antenna'.", "required": True}}},
- {"name": "toggle_subtitles", "description": "Turn subtitles on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}, "language": {"type": "string", "description": "Subtitle language e.g. 'english', 'spanish'.", "required": False}}},
- {"name": "set_tv_volume", "description": "Set the TV volume level.", "parameters": {"level": {"type": "number", "description": "Volume level from 0 to 100.", "required": True}}},
- {"name": "tv_power", "description": "Turn the TV on or off.", "parameters": {"action": {"type": "string", "description": "'on' or 'off'.", "required": True}}},
- {"name": "tv_channel", "description": "Switch to a specific TV channel.", "parameters": {"channel": {"type": "string", "description": "Channel number or name.", "required": True}}},
- {"name": "rewind_media", "description": "Rewind the current media by a specified amount.", "parameters": {"seconds": {"type": "number", "description": "Number of seconds to rewind.", "required": False}}},
- {"name": "fast_forward_media", "description": "Fast forward the current media by a specified amount.", "parameters": {"seconds": {"type": "number", "description": "Number of seconds to fast forward.", "required": False}}},
- {"name": "search_streaming", "description": "Search for content across streaming services.", "parameters": {"query": {"type": "string", "description": "Search query for movies, shows, or actors.", "required": True}}},
- {"name": "add_to_watchlist", "description": "Add a show or movie to your watchlist.", "parameters": {"title": {"type": "string", "description": "Title to add.", "required": True}}},
- {"name": "get_continue_watching", "description": "Get a list of shows/movies you haven't finished.", "parameters": {}},
- {"name": "set_sleep_timer_tv", "description": "Set a sleep timer to auto-turn off the TV.", "parameters": {"minutes": {"type": "number", "description": "Minutes until the TV turns off.", "required": True}}},
-]
-
-POOL_VEHICLE = [
- {"name": "lock_car", "description": "Lock or unlock the car remotely.", "parameters": {"action": {"type": "string", "description": "'lock' or 'unlock'.", "required": True}}},
- {"name": "start_car", "description": "Remote start or stop the car engine.", "parameters": {"action": {"type": "string", "description": "'start' or 'stop'.", "required": True}}},
- {"name": "car_climate", "description": "Set the car's climate control remotely.", "parameters": {"temperature": {"type": "number", "description": "Target temperature in degrees.", "required": True}, "unit": {"type": "string", "description": "'fahrenheit' or 'celsius'.", "required": False}}},
- {"name": "find_my_car", "description": "Locate your parked car on a map.", "parameters": {}},
- {"name": "check_car_status", "description": "Check car status: fuel/battery level, tire pressure, doors.", "parameters": {}},
- {"name": "honk_horn", "description": "Honk the car horn remotely to locate it.", "parameters": {}},
- {"name": "flash_lights", "description": "Flash the car lights remotely.", "parameters": {}},
- {"name": "open_trunk", "description": "Open or close the car trunk remotely.", "parameters": {"action": {"type": "string", "description": "'open' or 'close'.", "required": True}}},
- {"name": "get_fuel_level", "description": "Check the current fuel or battery level.", "parameters": {}},
- {"name": "find_ev_charger", "description": "Find nearby EV charging stations.", "parameters": {"connector_type": {"type": "string", "description": "Optional connector type e.g. 'CCS', 'CHAdeMO', 'Tesla'.", "required": False}}},
- {"name": "set_charge_limit", "description": "Set the EV charge limit percentage.", "parameters": {"percent": {"type": "number", "description": "Charge limit 50-100%.", "required": True}}},
- {"name": "get_range", "description": "Get the estimated remaining driving range.", "parameters": {}},
- {"name": "toggle_valet_mode", "description": "Enable or disable valet mode.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
-]
-
-POOL_TRAVEL = [
- {"name": "search_flights", "description": "Search for flights between cities.", "parameters": {"origin": {"type": "string", "description": "Origin city or airport code.", "required": True}, "destination": {"type": "string", "description": "Destination city or airport code.", "required": True}, "date": {"type": "string", "description": "Travel date in human readable format.", "required": True}}},
- {"name": "search_hotels", "description": "Search for hotels in a location.", "parameters": {"location": {"type": "string", "description": "City or area to search.", "required": True}, "check_in": {"type": "string", "description": "Check-in date.", "required": True}, "check_out": {"type": "string", "description": "Check-out date.", "required": True}}},
- {"name": "check_in_flight", "description": "Check in for an upcoming flight.", "parameters": {"confirmation_code": {"type": "string", "description": "Booking confirmation code.", "required": True}}},
- {"name": "show_boarding_pass", "description": "Display the boarding pass for an upcoming flight.", "parameters": {"confirmation_code": {"type": "string", "description": "Optional booking confirmation code. Shows next flight if omitted.", "required": False}}},
- {"name": "get_flight_status", "description": "Check the status of a flight.", "parameters": {"flight_number": {"type": "string", "description": "Flight number e.g. 'UA123', 'DL456'.", "required": True}}},
- {"name": "book_rental_car", "description": "Search for rental cars at a location.", "parameters": {"location": {"type": "string", "description": "Pickup location.", "required": True}, "pickup_date": {"type": "string", "description": "Pickup date.", "required": True}, "return_date": {"type": "string", "description": "Return date.", "required": True}}},
- {"name": "get_trip_itinerary", "description": "Show the itinerary for an upcoming trip.", "parameters": {"trip_name": {"type": "string", "description": "Optional trip name or destination.", "required": False}}},
- {"name": "convert_timezone", "description": "Convert a time between timezones.", "parameters": {"time": {"type": "string", "description": "Time to convert e.g. '3pm'.", "required": True}, "from_tz": {"type": "string", "description": "Source timezone e.g. 'EST', 'America/New_York'.", "required": True}, "to_tz": {"type": "string", "description": "Target timezone.", "required": True}}},
- {"name": "get_exchange_rate", "description": "Get the current exchange rate between currencies.", "parameters": {"from_currency": {"type": "string", "description": "Source currency code.", "required": True}, "to_currency": {"type": "string", "description": "Target currency code.", "required": True}}},
- {"name": "translate_phrase", "description": "Translate a phrase for travel purposes.", "parameters": {"text": {"type": "string", "description": "Text to translate.", "required": True}, "target_language": {"type": "string", "description": "Target language.", "required": True}}},
-]
-
-POOL_COOKING_KITCHEN = [
- {"name": "search_recipe", "description": "Search for a recipe by dish name or ingredients.", "parameters": {"query": {"type": "string", "description": "Dish name or ingredients to search for.", "required": True}}},
- {"name": "set_oven", "description": "Preheat or set the smart oven temperature.", "parameters": {"temperature": {"type": "number", "description": "Temperature in degrees.", "required": True}, "unit": {"type": "string", "description": "'fahrenheit' or 'celsius'.", "required": False}, "mode": {"type": "string", "description": "'bake', 'broil', 'convection', or 'air_fry'.", "required": False}}},
- {"name": "start_dishwasher", "description": "Start or schedule the dishwasher.", "parameters": {"cycle": {"type": "string", "description": "'normal', 'heavy', 'quick', or 'eco'.", "required": False}, "delay_minutes": {"type": "number", "description": "Optional delay before starting.", "required": False}}},
- {"name": "control_coffee_maker", "description": "Control the smart coffee maker.", "parameters": {"action": {"type": "string", "description": "'brew', 'stop', or 'warm'.", "required": True}, "size": {"type": "string", "description": "'small', 'medium', or 'large'.", "required": False}, "strength": {"type": "string", "description": "'mild', 'medium', or 'strong'.", "required": False}}},
- {"name": "cooking_convert", "description": "Convert between cooking measurements.", "parameters": {"value": {"type": "number", "description": "The value to convert.", "required": True}, "from_unit": {"type": "string", "description": "Source unit e.g. 'cups', 'tablespoons', 'grams', 'ounces'.", "required": True}, "to_unit": {"type": "string", "description": "Target unit.", "required": True}}},
- {"name": "set_cooking_timer", "description": "Set a named cooking timer.", "parameters": {"label": {"type": "string", "description": "Label for the timer e.g. 'pasta', 'chicken', 'rice'.", "required": True}, "minutes": {"type": "number", "description": "Timer duration in minutes.", "required": True}}},
- {"name": "get_substitution", "description": "Get a cooking ingredient substitution.", "parameters": {"ingredient": {"type": "string", "description": "The ingredient to substitute e.g. 'buttermilk', 'eggs'.", "required": True}}},
- {"name": "control_slow_cooker", "description": "Control the smart slow cooker.", "parameters": {"action": {"type": "string", "description": "'start', 'stop', or 'warm'.", "required": True}, "setting": {"type": "string", "description": "'low', 'medium', or 'high'.", "required": False}, "hours": {"type": "number", "description": "Cooking duration in hours.", "required": False}}},
- {"name": "check_fridge_inventory", "description": "Check what's in the smart fridge.", "parameters": {"category": {"type": "string", "description": "Optional category e.g. 'dairy', 'vegetables', 'drinks'.", "required": False}}},
-]
-
-POOL_PARENTAL_FAMILY = [
- {"name": "get_family_location", "description": "Check the current location of a family member.", "parameters": {"member": {"type": "string", "description": "Name of the family member.", "required": True}}},
- {"name": "set_child_screen_time", "description": "Set a daily screen time limit for a child's device.", "parameters": {"child_name": {"type": "string", "description": "Name of the child.", "required": True}, "minutes": {"type": "number", "description": "Daily screen time limit in minutes.", "required": True}}},
- {"name": "lock_child_device", "description": "Immediately lock or unlock a child's device.", "parameters": {"child_name": {"type": "string", "description": "Name of the child.", "required": True}, "action": {"type": "string", "description": "'lock' or 'unlock'.", "required": True}}},
- {"name": "get_child_activity", "description": "Get a child's recent device activity and app usage.", "parameters": {"child_name": {"type": "string", "description": "Name of the child.", "required": True}}},
- {"name": "set_content_filter", "description": "Set content filtering level for a child's device.", "parameters": {"child_name": {"type": "string", "description": "Name of the child.", "required": True}, "level": {"type": "string", "description": "'strict', 'moderate', or 'off'.", "required": True}}},
- {"name": "send_family_broadcast", "description": "Send a message to all family members.", "parameters": {"message": {"type": "string", "description": "The message to broadcast.", "required": True}}},
- {"name": "set_family_geofence", "description": "Set an alert when a family member arrives at or leaves a location.", "parameters": {"member": {"type": "string", "description": "Family member name.", "required": True}, "location": {"type": "string", "description": "Location name e.g. 'school', 'home', 'work'.", "required": True}, "trigger": {"type": "string", "description": "'arrive' or 'leave'.", "required": True}}},
- {"name": "request_check_in", "description": "Send a check-in request to a family member.", "parameters": {"member": {"type": "string", "description": "Family member name.", "required": True}}},
-]
-
-POOL_AUTOMATION = [
- {"name": "run_shortcut", "description": "Run a saved automation shortcut by name.", "parameters": {"name": {"type": "string", "description": "Name of the shortcut e.g. 'morning routine', 'bedtime', 'leaving home'.", "required": True}}},
- {"name": "create_routine", "description": "Create a new automation routine with a trigger and actions.", "parameters": {"name": {"type": "string", "description": "Name for the routine.", "required": True}, "trigger": {"type": "string", "description": "When to run e.g. 'every weekday at 7am', 'when I arrive home', 'at sunset'.", "required": True}, "actions": {"type": "string", "description": "Description of actions to perform.", "required": True}}},
- {"name": "list_routines", "description": "List all saved automation routines.", "parameters": {}},
- {"name": "delete_routine", "description": "Delete an automation routine by name.", "parameters": {"name": {"type": "string", "description": "Name of the routine to delete.", "required": True}}},
- {"name": "toggle_routine", "description": "Enable or disable an automation routine.", "parameters": {"name": {"type": "string", "description": "Name of the routine.", "required": True}, "enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "run_scene", "description": "Activate a smart home scene.", "parameters": {"scene_name": {"type": "string", "description": "Scene name e.g. 'movie night', 'dinner party', 'good morning', 'away'.", "required": True}}},
- {"name": "get_routine_history", "description": "Check when a routine last ran.", "parameters": {"name": {"type": "string", "description": "Name of the routine.", "required": True}}},
-]
-
-POOL_CONNECTIVITY = [
- {"name": "toggle_mobile_data", "description": "Turn mobile data on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "toggle_nfc", "description": "Turn NFC on or off.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
- {"name": "share_wifi", "description": "Share the current WiFi network credentials via QR code or nearby share.", "parameters": {}},
- {"name": "get_data_usage", "description": "Check mobile data usage for the current billing period.", "parameters": {}},
- {"name": "set_data_limit", "description": "Set a mobile data usage warning or limit.", "parameters": {"limit_gb": {"type": "number", "description": "Data limit in gigabytes.", "required": True}}},
- {"name": "get_wifi_info", "description": "Get info about the current WiFi connection: network name, signal strength, speed.", "parameters": {}},
- {"name": "forget_wifi_network", "description": "Forget a saved WiFi network.", "parameters": {"ssid": {"type": "string", "description": "Name of the WiFi network to forget.", "required": True}}},
- {"name": "get_connected_devices", "description": "List all Bluetooth or WiFi devices currently connected.", "parameters": {"connection_type": {"type": "string", "description": "'bluetooth' or 'wifi'.", "required": False}}},
- {"name": "disconnect_bluetooth_device", "description": "Disconnect a specific Bluetooth device.", "parameters": {"device_name": {"type": "string", "description": "Name of the device to disconnect.", "required": True}}},
-]
-
-POOL_SCREEN_CAPTURE = [
- {"name": "start_screen_recording", "description": "Start recording the screen.", "parameters": {"audio": {"type": "boolean", "description": "Whether to record microphone audio too.", "required": False}}},
- {"name": "stop_screen_recording", "description": "Stop the current screen recording and save it.", "parameters": {}},
- {"name": "take_scrolling_screenshot", "description": "Take a long scrolling screenshot of the current page.", "parameters": {}},
- {"name": "annotate_screenshot", "description": "Open the most recent screenshot for annotation/markup.", "parameters": {}},
- {"name": "screen_mirror", "description": "Mirror the screen to an external display.", "parameters": {"device_name": {"type": "string", "description": "Name of the display e.g. 'Living Room TV', 'Office Monitor'.", "required": True}}},
- {"name": "stop_screen_mirror", "description": "Stop screen mirroring.", "parameters": {}},
- {"name": "picture_in_picture", "description": "Enable or disable picture-in-picture mode for the current app.", "parameters": {"enabled": {"type": "boolean", "description": "True to enable, false to disable.", "required": True}}},
-]
-
-POOL_EMAIL = [
- {"name": "compose_email", "description": "Compose and send an email.", "parameters": {"to": {"type": "string", "description": "Recipient email address.", "required": True}, "subject": {"type": "string", "description": "Email subject line.", "required": True}, "body": {"type": "string", "description": "Email body text.", "required": True}, "cc": {"type": "string", "description": "Optional CC recipients.", "required": False}}},
- {"name": "reply_email", "description": "Reply to the most recent or specified email.", "parameters": {"body": {"type": "string", "description": "Reply text.", "required": True}, "subject_match": {"type": "string", "description": "Optional subject line to identify which email to reply to.", "required": False}}},
- {"name": "forward_email", "description": "Forward an email to someone.", "parameters": {"to": {"type": "string", "description": "Recipient to forward to.", "required": True}, "subject_match": {"type": "string", "description": "Subject line to identify the email.", "required": False}}},
- {"name": "archive_email", "description": "Archive an email.", "parameters": {"subject_match": {"type": "string", "description": "Subject line to identify the email.", "required": True}}},
- {"name": "snooze_email", "description": "Snooze an email to reappear later.", "parameters": {"subject_match": {"type": "string", "description": "Subject line to identify the email.", "required": True}, "snooze_until": {"type": "string", "description": "When to resurface e.g. 'tomorrow morning', 'next Monday'.", "required": True}}},
- {"name": "label_email", "description": "Apply a label or folder to an email.", "parameters": {"subject_match": {"type": "string", "description": "Subject to identify the email.", "required": True}, "label": {"type": "string", "description": "Label name e.g. 'work', 'personal', 'receipts'.", "required": True}}},
- {"name": "mark_email_spam", "description": "Mark an email as spam.", "parameters": {"subject_match": {"type": "string", "description": "Subject to identify the email.", "required": True}}},
- {"name": "get_unread_emails", "description": "Get unread emails, optionally filtered.", "parameters": {"from_contact": {"type": "string", "description": "Optional sender name to filter by.", "required": False}, "count": {"type": "number", "description": "Number of emails to show.", "required": False}}},
- {"name": "search_email", "description": "Search emails by keyword.", "parameters": {"query": {"type": "string", "description": "Search query.", "required": True}}},
-]
-
-POOL_PASSWORDS = [
- {"name": "get_password", "description": "Retrieve a saved password for a site or app.", "parameters": {"site": {"type": "string", "description": "Website or app name.", "required": True}}},
- {"name": "generate_password", "description": "Generate a secure random password.", "parameters": {"length": {"type": "number", "description": "Password length (default 16).", "required": False}, "include_symbols": {"type": "boolean", "description": "Whether to include symbols.", "required": False}}},
- {"name": "get_totp_code", "description": "Get the current 2FA/TOTP code for a site.", "parameters": {"site": {"type": "string", "description": "Website or app name.", "required": True}}},
- {"name": "save_password", "description": "Save or update a password for a site.", "parameters": {"site": {"type": "string", "description": "Website or app name.", "required": True}, "username": {"type": "string", "description": "Username or email.", "required": True}, "password": {"type": "string", "description": "The password to save.", "required": True}}},
- {"name": "check_password_breach", "description": "Check if a password or account has been in a data breach.", "parameters": {"site": {"type": "string", "description": "Optional site to check. Checks all if omitted.", "required": False}}},
- {"name": "autofill_login", "description": "Autofill login credentials on the current page.", "parameters": {}},
-]
-
-ALL_POOLS = [
- POOL_TIME_PRODUCTIVITY,
- POOL_LISTS_NOTES,
- POOL_MESSAGING,
- POOL_DEVICE_CONTROL,
- POOL_MEDIA,
- POOL_NAVIGATION,
- POOL_SMART_HOME,
- POOL_UTILITY,
- POOL_CAMERA_PHOTOS,
- POOL_FITNESS_HEALTH,
- POOL_SYSTEM,
- POOL_FINANCE,
- POOL_READING_NEWS,
- POOL_ACCESSIBILITY,
- POOL_SHOPPING,
- POOL_SOCIAL,
- POOL_RIDE_DELIVERY,
- POOL_FILE_MANAGEMENT,
- POOL_WEARABLE,
- POOL_DESKTOP,
- POOL_BROWSER,
- POOL_EMERGENCY_SAFETY,
- POOL_DIGITAL_WELLBEING,
- POOL_TV_STREAMING,
- POOL_VEHICLE,
- POOL_TRAVEL,
- POOL_COOKING_KITCHEN,
- POOL_PARENTAL_FAMILY,
- POOL_AUTOMATION,
- POOL_CONNECTIVITY,
- POOL_SCREEN_CAPTURE,
- POOL_EMAIL,
- POOL_PASSWORDS,
-]
-
-
-SCENARIOS = [
- # Time & productivity
- "setting a cooking timer", "morning alarm routine", "pomodoro work timer",
- "reminder to take medication", "reminder to pick up kids from school",
- "scheduling a dentist appointment", "meeting reminder for tomorrow",
- "snoozing an alarm", "stopping a running timer",
- # Lists & notes
- "adding groceries to shopping list", "creating a packing list for vacation",
- "jotting down a quick idea", "adding tasks to a todo list for the week",
- "checking what's on the shopping list", "removing an item from a list",
- "noting down a recipe", "noting a license plate number",
- # Messaging & calls
- "texting a friend about dinner plans", "calling mom",
- "sending a work email about a deadline", "looking up a contact then messaging them",
- "sending a birthday greeting", "replying to a message",
- # Device control
- "turning down brightness at night", "turning off wifi to save battery",
- "enabling do not disturb for a meeting", "turning on the flashlight",
- "turning up the volume", "connecting bluetooth headphones",
- "setting screen timeout", "turning on airplane mode",
- # Media
- "playing a specific song", "pausing music", "skipping a track",
- "playing a jazz playlist", "listening to a true crime podcast",
- "resuming a podcast", "playing white noise for sleep",
- # Navigation
- "getting directions to the airport", "finding nearby coffee shops",
- "sharing location with a friend", "finding the nearest gas station",
- "walking directions to the park", "transit directions to downtown",
- # Smart home
- "turning off bedroom lights", "setting thermostat to 72 degrees",
- "locking the front door before bed", "dimming living room lights",
- "starting the robot vacuum", "turning on kitchen lights",
- # Utility
- "calculating a restaurant tip", "converting miles to kilometers",
- "translating a phrase to Spanish", "checking the weather",
- "opening the camera app", "searching the web for a recipe",
- "calculating days until a date", "taking a screenshot",
- # Multi-tool combos
- "setting an alarm AND a reminder", "messaging someone AND setting a timer",
- "turning off lights AND locking doors before bed",
- "adding to shopping list AND setting a reminder to go shopping",
- "looking up a contact AND calling them",
- "getting directions AND sharing location",
- # Camera & photos
- "taking a selfie", "recording a video", "opening the photo gallery",
- "sharing a photo with a friend", "taking a timed photo",
- # Fitness & health
- "starting a run workout", "logging water intake",
- "checking step count for the day", "starting sleep tracking",
- "logging lunch for nutrition", "stopping a workout",
- # System
- "turning on airplane mode for a flight", "clearing all notifications",
- "checking battery level", "turning on dark mode",
- "restarting the phone", "turning off location services",
- "opening wifi settings",
- # Finance
- "sending money to a friend for dinner", "checking bank balance",
- "converting dollars to euros", "paying someone back",
- # Reading & news
- "getting top news headlines", "reading an article aloud",
- "summarizing the current page", "getting sports news",
- # Accessibility
- "making the font bigger", "turning on the screen reader",
- "enabling the magnifier", "turning off voice assistant",
- # Shopping
- "searching for a product online", "adding something to cart",
- "checking order delivery status", "buying a phone case",
- # Social
- "posting on social media", "checking social notifications",
- "posting a status update",
- # Ride-hailing & delivery
- "ordering a ride to a destination", "tracking a food delivery",
- "canceling a ride", "ordering food from a nearby place",
- # File management
- "opening a downloaded PDF", "sharing a document with a coworker",
- "creating a new folder for photos", "compressing files to send",
- "moving a file to the downloads folder",
- # Wearable
- "checking heart rate during exercise", "finding my phone from my watch",
- "changing the watch face", "starting a breathing exercise",
- "checking if I hit my step goal", "checking blood oxygen before sleep",
- # Desktop/laptop
- "splitting the screen for multitasking", "switching to external speakers",
- "printing a document", "killing a frozen app",
- "connecting to VPN for work", "checking for system updates",
- "switching to the next virtual desktop",
- # Browser
- "opening a new private window", "bookmarking this page",
- "clearing browser history", "finding text on this page",
- # Emergency & safety
- "calling 911", "sending my location to emergency contact",
- "triggering SOS from watch", "setting up emergency contacts",
- # Digital wellbeing
- "checking how much time I spent on my phone today",
- "setting a 30 minute limit on social media",
- "turning on bedtime mode", "checking which app I used most this week",
- # Cross-device & undo
- "cancel the timer I just set", "undo the last action",
- "reply to the last message", "call back the last number",
- "find the nearest pharmacy and navigate there",
- # Meeting & productivity context
- "noting an action item from a meeting", "scheduling a follow-up meeting with the team",
- "reminding me to send the proposal after the call", "texting someone I'm running late to the meeting",
- "setting a timer for a 15 minute break between meetings", "adding meeting notes about the budget discussion",
- "creating a todo for the deliverables we agreed on", "sharing meeting notes with attendees",
- "scheduling a 1-on-1 for next week", "setting do not disturb for an hour-long meeting",
- "recording a voice memo of key takeaways", "emailing the client a summary after the call",
- "checking my calendar for conflicts before accepting", "creating a checklist of action items from standup",
- # Indirect / implicit intent — observations, feelings, complaints (not commands)
- "it's really dark in here", "I can't hear anything on this call",
- "I'm freezing", "it's too bright", "I'm bored",
- "I need to wake up early tomorrow", "I keep forgetting to drink water",
- "my phone is about to die", "I can barely see my screen",
- "this room is stuffy", "it's so loud in here", "I can't sleep",
- "I'm lost and I don't know where I am", "my eyes are hurting from this screen",
- "it's getting dark outside", "I'm starving", "I wonder what the weather's like",
- "ugh I have so much to do today", "I'm late for my meeting",
- "my back hurts from sitting all day", "I haven't moved in hours",
- "it smells like something is burning in the kitchen", "the house feels empty and creepy",
- "I'm so tired", "why is it so hot in here", "the baby is sleeping so everything needs to be quiet",
- "I think I left the front door unlocked", "my hands are full and I need light",
- "this song is terrible", "I have no idea what time it is over there",
- "I promised someone I'd call them back", "the kids have been on their tablets all day",
- "I should probably stretch or something", "I wonder if my package arrived",
- "this article is really long I don't have time to read all of it",
- "I need to remember to buy milk on the way home",
- "my car must be low on gas by now", "I have no idea where I parked",
- # Corrections and follow-ups
- "correcting a time just set", "redirecting a message to a different person",
- "cancel what I just set", "change the timer to 10 minutes",
- "undo that", "never mind, turn it back on",
- # Temporal and conditional
- "every weekday at 8am", "when I get home remind me to call mom",
- "in 2 hours turn off the lights", "tomorrow morning check the weather",
- "after work start the robot vacuum", "at sunset close the blinds",
- # Multi-person and group
- "messaging two people about the same event", "setting up a group call",
- "sharing location with a family group",
- "sending the same email to multiple recipients",
- # Context references
- "call them back", "open that file from earlier",
- "reply saying I'll be there in 10", "send them what I just screenshotted",
- "play that song again", "go back to the last page",
- # Multilingual and code-switched
- "set un timer por 5 minutos", "envoie un message à Jean",
- "spiel etwas Musik", "mets le volume à 50",
- "llama a mamá", "recherche le restaurant le plus proche",
- # Ultra-terse commands
- "timer 5", "alarm 7am", "lights off", "call mom", "weather",
- "screenshot", "wifi off", "volume up", "next song", "lock door",
- # Verbose and conversational
- "hey so I was thinking could you maybe set a reminder for me to pick up the dry cleaning sometime tomorrow afternoon if that's not too much trouble",
- "ok so I need you to do a few things: first turn off the living room lights, then lock the front door, and also set the alarm for 6:30",
- "I'm about to go into a meeting so can you put my phone on do not disturb and also remind me in an hour to check my email",
- # Multi-call with same tool (few tools, different args)
- "sending different messages to two people about different topics",
- "setting two alarms for different times and purposes",
- "adding multiple grocery items to a shopping list separately",
- "creating reminders for several appointments across the week",
- "texting three different people with different status updates",
- "setting multiple cooking timers for different dishes",
- "sending two separate emails about different work topics",
- "adding several distinct notes about different subjects",
- "creating multiple calendar events for different days",
- "logging several meals throughout the day",
- # Multi-call with long argument values
- "sending a detailed work message with a specific reason for a schedule change",
- "creating a note with a full recipe including all ingredients and steps",
- "emailing someone about a maintenance issue with a detailed explanation",
- "setting a reminder that includes a full street address",
- "posting a long enthusiastic status update about an upcoming event",
- # TV & streaming
- "playing a show on a streaming service", "turning on subtitles", "switching TV input",
- "setting a TV sleep timer", "searching for a genre of movies", "rewinding playback",
- "resuming what was playing last", "adding something to a watchlist",
- "adjusting TV volume to a specific level", "playing the next episode",
- # Vehicle & car
- "locking the car remotely", "remote starting the car with climate on", "finding where I parked",
- "checking tire pressure", "honking to locate the car",
- "checking fuel or battery level", "preheating the cabin",
- "opening the trunk remotely", "finding a charging station", "setting EV charge limit",
- "checking if car is locked", "enabling valet mode",
- # Travel & booking
- "searching for flights to a destination", "checking in for a flight",
- "showing a boarding pass", "checking flight status", "checking currency exchange rate",
- "checking time in another timezone", "searching for hotels in a city",
- "viewing a trip itinerary", "renting a car at an airport",
- # Cooking & kitchen
- "preheating the oven to a temperature", "converting cooking measurements",
- "setting a named cooking timer", "finding an ingredient substitution",
- "starting the dishwasher on a specific cycle", "brewing coffee with size and strength",
- "searching for a specific recipe", "checking fridge inventory",
- "starting the slow cooker with settings",
- # Parental & family
- "checking a family member's location", "locking a child's device",
- "checking a child's screen time usage", "setting screen time limits",
- "broadcasting a message to the whole family",
- "geofencing alerts for when someone arrives or leaves", "requesting a check-in",
- "setting content filters on kids' devices",
- # Automation & shortcuts
- "run my morning routine", "trigger the bedtime shortcut",
- "what automations do I have", "turn off the leaving home routine",
- "run movie night scene", "delete the weekend cleanup routine",
- "when did my morning routine last run",
- "create a routine that turns off lights at 11pm every night",
- # Connectivity
- "turn off mobile data", "how much data have I used this month",
- "what wifi am I connected to", "share the wifi password",
- "forgetting a saved wifi network", "listing connected bluetooth devices",
- "disconnecting a specific bluetooth device", "toggling NFC", "setting a data usage limit",
- # Screen capture & mirroring
- "start screen recording", "stop recording the screen",
- "take a scrolling screenshot", "mirror my screen to the living room TV",
- "stop mirroring", "turn on picture in picture",
- "record the screen with my mic on",
- # Email management
- "checking unread emails", "replying to an email from someone",
- "archiving an email about a topic", "snoozing an email until later",
- "labeling an email by category", "marking email as spam",
- "forwarding an email to someone", "searching emails by keyword",
- "checking emails from a specific sender",
- # Passwords & security
- "retrieving a saved password for a service", "generating a new password",
- "getting a 2FA code for a site", "checking for data breaches",
- "autofilling login credentials", "saving a new password",
- # Chained real-world workflows
- "I just parked — save this location and set a 2 hour timer",
- "I'm leaving work — start the car, turn on the AC, and get directions home",
- "going to bed — lock the doors, turn off all lights, and set alarm for 6:30",
- "heading to a meeting — put phone on DND, text my wife I'll be late, and set a 1 hour timer",
- "at the airport — check in for my flight, show boarding pass, and turn on airplane mode",
- "cooking dinner — preheat oven to 400, set a timer for chicken 45 minutes, and play jazz",
- # Device-state queries
- "is my bluetooth on", "what devices are connected", "how much storage do I have left",
- "what's my battery percentage", "is wifi connected", "what network am I on",
- "is do not disturb on", "what's the screen brightness set to",
- # Quantified and precise requests
- "set brightness to exactly 40 percent", "volume to 30", "timer for 7 minutes and 30 seconds",
- "set thermostat to 68.5 degrees", "alarm for 6:15am sharp",
- "rewind exactly 10 seconds", "set charge limit to 90 percent",
- # Negation and cancellation
- "don't set an alarm for tomorrow", "stop reminding me about the dentist",
- "I don't want notifications from Instagram anymore",
- "turn off all my alarms", "remove everything from my shopping list",
- "unsubscribe from the sports newsletter", "disable the morning routine",
- "cancel my ride", "stop the screen recording",
- # Disfluent / natural speech — filler words, false starts, self-corrections
- "um can you like set a timer for uh 10 minutes",
- "hey so I was thinking like maybe turn off the lights or whatever",
- "okay so uh send a message to... wait who was it... oh right, send it to Mike",
- "alright so um I need an alarm for like 7... no wait 7:30 tomorrow morning",
- "hmm can you uh play some music, like jazz or something chill I guess",
- "oh yeah hey turn the volume down a bit it's kinda loud",
- "so like I think I need to uh check my email or something",
- "wait actually no, set it to 20 minutes not 15",
- "could you um like maybe turn on do not disturb if that's a thing",
- "hey uh what's the... what's the weather like out there",
- "rambling about needing to text someone that you're running late",
- "set a — no actually make that a reminder, not a timer, for 3pm",
- "uh brightness, like turn it up, I can't really see",
- "so I think maybe we should lock the front door, right? yeah do that",
- "hey siri — I mean, uh, can you just play the next song",
- "well I guess I should probably set an alarm, like for early, maybe 6?",
- "turn turn off the uh the bluetooth thing",
- "oh um could you check like how much battery I have left or whatever",
- "right so basically I need directions to the uh the airport, yeah",
- "let me think... okay yeah just add milk to the shopping list",
- "so um like my mom asked me to call her so yeah can you call mom",
- "hey wait before I forget can you remind me tomorrow about the dentist thing",
- "I dunno just like play something, anything really, some background music",
- "ugh okay fine just turn on the flashlight I can't see anything",
- # Typos and keyboard errors — themes for Gemini to generate its own misspellings
- "sending a message with typos in the command",
- "setting a timer with misspelled words",
- "playing music with garbled text", "toggling a setting with missing letters",
- "checking something with swapped letters", "calling someone with truncated words",
- "searching nearby with typos", "toggling connectivity with misspelling",
- "checking email with keyboard errors", "locking something with missing vowels",
- "running words together without spaces", "setting something with no spaces",
- "adjusting a setting with a typo", "navigating with misspelled destination",
- "a reminder with text-speak abbreviations", "texting with informal shorthand",
- # Speech-to-text / ASR errors — themes
- "a timer request where a number sounds like a different word",
- "connecting a device where the name gets mangled by speech recognition",
- "playing music where the genre gets misheard",
- "checking something where homophones get confused (weather/whether, to/two, for/four)",
- "casual speech with wanna/gonna/gotta contractions",
- "slang and phonetic spelling like lemme/gimme/hafta/shoulda",
- # Dropped words / telegram style — themes
- "ultra-short command with just a number or single word",
- "device command with just the setting name and value, no verbs",
- "message with just recipient and content, no filler words",
- "directions with just destination and constraint, no articles",
- "reminder with just topic, day, and time — no complete sentence",
- "multiple toggles in telegram style, no connecting words",
- # Ambiguous queries requiring disambiguation between similar tools
- "sending something to someone without specifying the channel",
- "messaging someone about a topic without specifying SMS vs instant message",
- "setting a reminder without specifying reminder vs list item",
- "playing something without specifying song vs playlist vs radio",
- "navigating somewhere without specifying directions vs turn-by-turn",
- "setting a timer without specifying generic vs cooking timer",
- "writing something down without specifying note vs list vs reminder",
- "sharing something without specifying the method or recipient type",
- "looking up a place without specifying search vs navigate",
- "telling someone something without specifying call vs text vs email",
- "booking something without specifying the service type",
- # Argument value edge cases — themes for special characters, diverse content
- "texting someone with emoji in the message",
- "emailing someone at a non-English domain about a work topic",
- "setting a reminder with a full street address including apartment number",
- "noting down a complex password or code with special characters",
- "sending a casual excited message with lots of emoji",
- "emailing a colleague at a foreign company about a deadline",
- "creating a note with a recipe using fractions and degree symbols",
- "texting someone about travel logistics with flight details",
- "sending a professional email with specific numbers, percentages, and a meeting time",
- "adding a shopping list with quantities, brands, and metric measurements",
- # Math, calculations, and conversions
- "what's 15 percent of 89.99", "split 247 dollars three ways",
- "how many days until March 15", "what's 72 fahrenheit in celsius",
- "calculate the area of a circle with radius 5", "what's the square root of 144",
- "how much is 30% off of $65", "if I drive 180 miles at 60mph how long will it take",
- "convert 2.5 hours to minutes and seconds", "what's 19 times 23 plus 7",
- "how many calories if I eat 3 servings of 240 calories each",
- "what's my average if I scored 87, 92, 78, and 95", "tip for $43.50 at 20%",
- "what day of the week is January 1 2025", "BMI for 175 lbs and 5 foot 10",
- "how many cups in 750ml", "convert 3 tablespoons to teaspoons",
- "what's 1000 yen in dollars roughly", "compound interest on 5000 at 4% for 3 years",
- # Edge cases
- "very short terse command like 'timer 5 min'",
- "long polite request with extra context",
- "ambiguous request that maps to one tool",
- "casual voice assistant style like 'hey set my alarm for seven'",
- "request where no available tool can help",
-]
-
-CALL_TYPES = [
- ("single", "exactly 1 tool call"), # 3/16 ≈ 19%
- ("single", "exactly 1 tool call"),
- ("single", "exactly 1 tool call"),
- ("multi", "2-3 tool calls (the user wants multiple things done at once)"), # 3/16 ≈ 19%
- ("multi", "2-3 tool calls (the user wants multiple things done at once)"),
- ("multi", "2-3 tool calls (the user wants multiple things done at once)"),
- ("multi_few_tools", "2-4 tool calls using ONLY 1-3 of the available tools — the user wants multiple actions with the SAME or very few tools, with DIFFERENT detailed argument values each time (e.g. sending messages to 3 different people, setting 2 different alarms, creating multiple list items). Each call MUST have distinct, realistic argument values — vary names, times, locations, messages, etc."), # 4/16 ≈ 25%
- ("multi_few_tools", "2-4 tool calls using ONLY 1-3 of the available tools — the user wants multiple actions with the SAME or very few tools, with DIFFERENT detailed argument values each time (e.g. sending messages to 3 different people, setting 2 different alarms, creating multiple list items). Each call MUST have distinct, realistic argument values — vary names, times, locations, messages, etc."),
- ("multi_few_tools", "2-4 tool calls using ONLY 1-3 of the available tools — the user wants multiple actions with the SAME or very few tools, with DIFFERENT detailed argument values each time (e.g. sending messages to 3 different people, setting 2 different alarms, creating multiple list items). Each call MUST have distinct, realistic argument values — vary names, times, locations, messages, etc."),
- ("multi_few_tools", "2-4 tool calls using ONLY 1-3 of the available tools — the user wants multiple actions with the SAME or very few tools, with DIFFERENT detailed argument values each time (e.g. sending messages to 3 different people, setting 2 different alarms, creating multiple list items). Each call MUST have distinct, realistic argument values — vary names, times, locations, messages, etc."),
- ("multi_long_values", "2-3 tool calls where argument values are LONG and detailed — full sentences for message text, multi-word descriptions, specific addresses, complete email bodies, detailed notes. Each argument value should be at least 5-10 words, not just a single word or number."), # 2/16 ≈ 12%
- ("multi_long_values", "2-3 tool calls where argument values are LONG and detailed — full sentences for message text, multi-word descriptions, specific addresses, complete email bodies, detailed notes. Each argument value should be at least 5-10 words, not just a single word or number."),
- ("none", "NO tool calls — the user asks a question or makes a request that NONE of the available tools can fulfill. "
- "answers must be []. Every query must be specific, detailed, and UNIQUE — no generic one-liners."),
- ("near_miss", "NO tool calls — the query is RELATED to the domain of the available tools but the tools CANNOT actually fulfill the request. "
- "Examples: asking for HISTORY of alarms when only set_alarm exists, asking to EDIT a calendar event when only create/delete exist, "
- "asking for a feature a tool doesn't support (e.g. 'set a recurring timer' when set_timer has no repeat param), "
- "asking about the STATUS of something when only action tools exist, asking to do something to an entity the tools don't cover. "
- "The query should sound natural and plausible — a real user might reasonably expect this to work. answers must be []"),
- ("indirect", "1-2 tool calls — but the query is an INDIRECT or IMPLICIT statement, NOT a direct command. " # 2/18 ≈ 11%
- "The user describes a situation, feeling, observation, or complaint and the assistant must INFER which tool to call. "
- "Examples: 'it's freezing in here' → set_thermostat higher, 'I can barely see my screen' → set_brightness higher, "
- "'it's so loud' → set_volume lower, 'I'm bored' → play_music or get_news, "
- "'my phone is about to die' → toggle_power_saving on, 'I can't sleep' → toggle_bedtime_mode or start_sleep_tracking, "
- "'I'm lost' → get_directions or share_location, 'this room is stuffy' → control_fan on. "
- "The query must NEVER contain the tool name or action verb directly — it should be a natural human observation or complaint. "
- "Vary between statements ('it's dark'), questions ('why is it so hot?'), and complaints ('ugh this is way too bright')."),
- ("indirect", "1-2 tool calls — but the query is an INDIRECT or IMPLICIT statement, NOT a direct command. "
- "The user describes a situation, feeling, observation, or complaint and the assistant must INFER which tool to call. "
- "Examples: 'it's freezing in here' → set_thermostat higher, 'I can barely see my screen' → set_brightness higher, "
- "'it's so loud' → set_volume lower, 'I'm bored' → play_music or get_news, "
- "'my phone is about to die' → toggle_power_saving on, 'I can't sleep' → toggle_bedtime_mode or start_sleep_tracking, "
- "'I'm lost' → get_directions or share_location, 'this room is stuffy' → control_fan on. "
- "The query must NEVER contain the tool name or action verb directly — it should be a natural human observation or complaint. "
- "Vary between statements ('it's dark'), questions ('why is it so hot?'), and complaints ('ugh this is way too bright')."),
- ("disfluent", "1-2 tool calls — but the query must contain NATURAL SPEECH DISFLUENCIES like real voice transcriptions. " # 2/20 ≈ 10%
- "Include filler words (um, uh, hmm, like, so, well, okay, right), false starts ('set a — no wait, make that...'), "
- "self-corrections ('send it to — no wait, the other one'), hedging ('I think maybe', 'if possible', 'or whatever'), "
- "trailing off ('can you like... turn off the...'), repetitions ('turn turn off the lights'), "
- "and casual padding ('oh yeah', 'hey so', 'alright so', 'wait actually'). "
- "The underlying intent should still be clear and map to a valid tool call — the disfluencies are just surface noise. "
- "Mix short disfluent queries ('uh lights off') with longer rambly ones."),
- ("disfluent", "1-2 tool calls — but the query must contain NATURAL SPEECH DISFLUENCIES like real voice transcriptions. "
- "Include filler words (um, uh, hmm, like, so, well, okay, right), false starts ('set a — no wait, make that...'), "
- "self-corrections ('send it to — no wait, the other one'), hedging ('I think maybe', 'if possible', 'or whatever'), "
- "trailing off ('can you like... turn off the...'), repetitions ('turn turn off the lights'), "
- "and casual padding ('oh yeah', 'hey so', 'alright so', 'wait actually'). "
- "The underlying intent should still be clear and map to a valid tool call — the disfluencies are just surface noise. "
- "Mix short disfluent queries ('uh lights off') with longer rambly ones."),
- ("garbled", "1-2 tool calls — but the query must contain REALISTIC TYPOS and SPEECH-TO-TEXT ERRORS, as if transcribed by an imperfect ASR system or typed hastily on a phone keyboard. " # 2/22 ≈ 9%
- "Include: swapped/missing/doubled letters ('trun' for 'turn', 'mesage' for 'message', 'tiemr' for 'timer'), "
- "autocorrect mistakes ('set a duck timer' for 'set a 10 minute timer'), "
- "missing spaces ('turnoff the lights', 'setanalarm'), "
- "homophones from speech ('there' for 'their', 'weather' for 'whether', 'to' for 'two', 'four' for 'for'), "
- "dropped words ('set timer 5' instead of 'set a timer for 5 minutes'), "
- "and phonetic misspellings from voice ('cud you' for 'could you', 'wanna' for 'want to', 'gonna' for 'going to'). "
- "Keep 1-3 errors per query — not every word should be wrong. The intent must still be recoverable. "
- "The tool calls in answers must be CORRECT — garbling is only in the query, not the output."),
- ("garbled", "1-2 tool calls — but the query must contain REALISTIC TYPOS and SPEECH-TO-TEXT ERRORS, as if transcribed by an imperfect ASR system or typed hastily on a phone keyboard. "
- "Include: swapped/missing/doubled letters ('trun' for 'turn', 'mesage' for 'message', 'tiemr' for 'timer'), "
- "autocorrect mistakes ('set a duck timer' for 'set a 10 minute timer'), "
- "missing spaces ('turnoff the lights', 'setanalarm'), "
- "homophones from speech ('there' for 'their', 'weather' for 'whether', 'to' for 'two', 'four' for 'for'), "
- "dropped words ('set timer 5' instead of 'set a timer for 5 minutes'), "
- "and phonetic misspellings from voice ('cud you' for 'could you', 'wanna' for 'want to', 'gonna' for 'going to'). "
- "Keep 1-3 errors per query — not every word should be wrong. The intent must still be recoverable. "
- "The tool calls in answers must be CORRECT — garbling is only in the query, not the output."),
- ("no_tools", "NO tool calls — there are NO tools available at all, answers must be []. "
- "Generate diverse, specific queries as if the user expects tools to exist. Every query must be UNIQUE."),
-]
-
-MODEL = "gemini-3.1-flash-lite-preview"
-
-LANGUAGES = [
- "English",
- "Bulgarian", "Croatian", "Czech", "Danish", "Dutch",
- "Estonian", "Finnish", "French", "German", "Greek", "Hungarian",
- "Italian", "Latvian", "Lithuanian", "Maltese", "Polish",
- "Portuguese", "Romanian", "Slovak", "Slovenian", "Spanish",
- "Swedish", "Russian", "Ukrainian",
-]
-
-MAX_TOOLS = 10
-
-_TOOL_COUNT_WEIGHTS = {
- 1: 8, 2: 10, 3: 14, 4: 14, 5: 14, 6: 12, 7: 10, 8: 8, 9: 5, 10: 5,
-}
-_TOOL_COUNTS = list(_TOOL_COUNT_WEIGHTS.keys())
-_TOOL_WEIGHTS = list(_TOOL_COUNT_WEIGHTS.values())
-
-
-def _pick_tools(rng, force_empty=False, few_tools=False):
- """Pick tools from 1-3 random pools, with explicit control over total count.
-
- If few_tools=True, pick only 1-3 tools from a single pool (for multi-call
- scenarios where the same tools are called multiple times with different args).
- If force_empty=True, return no tools.
- Otherwise, sample a target count from _TOOL_COUNT_WEIGHTS and fill from pools.
- """
- if force_empty:
- return []
- if few_tools:
- pool = rng.choice(ALL_POOLS)
- k = rng.randint(1, min(3, len(pool)))
- return rng.sample(pool, k)
-
- # Pick target tool count from explicit distribution
- target = rng.choices(_TOOL_COUNTS, weights=_TOOL_WEIGHTS, k=1)[0]
-
- # Draw from pools, adding more pools if needed to meet the target count
- pool_order = list(range(len(ALL_POOLS)))
- rng.shuffle(pool_order)
-
- candidates = []
- seen = set()
- for pi in pool_order:
- for t in ALL_POOLS[pi]:
- if t["name"] not in seen:
- seen.add(t["name"])
- candidates.append(t)
- if len(candidates) >= target:
- break
-
- rng.shuffle(candidates)
- return candidates[:target]
-
-
-# Random context seeds to inject diversity into each batch's prompt.
-# Each batch gets one — forces Gemini into a different user/situation space.
-_CONTEXT_SEEDS = [
- "The user is a busy parent managing kids and household",
- "The user is a college student studying for finals",
- "The user is commuting on public transit",
- "The user is cooking dinner while multitasking",
- "The user is an elderly person who isn't very tech-savvy",
- "The user is a teenager using their phone casually",
- "The user is a remote worker in a home office",
- "The user is at the gym between sets",
- "The user is traveling abroad and unfamiliar with the area",
- "The user is a professional in back-to-back meetings all day",
- "The user is getting ready for bed and winding down",
- "The user is driving and using voice commands hands-free",
- "The user is a small business owner managing their shop",
- "The user is hosting a dinner party with guests arriving soon",
- "The user is a freelancer juggling multiple client projects",
- "The user is on a hiking trail with limited connectivity",
- "The user is a nurse on a busy hospital shift",
- "The user is a musician setting up for a live performance",
- "The user is a pet owner managing their dog's routine",
- "The user is moving into a new apartment this weekend",
- "The user is planning a surprise birthday party",
- "The user is a photographer on location at a shoot",
- "The user is a teacher preparing for tomorrow's class",
- "The user is recovering from surgery and resting at home",
- "The user is at an airport waiting for a delayed flight",
- "The user is a gamer streaming on Twitch",
- "The user is gardening outside on a sunny afternoon",
- "The user is a new employee on their first week at work",
- "The user is babysitting their niece and nephew",
- "The user is a rideshare driver between pickups",
-]
-
-
-def build_prompt(batch_size, call_desc, tools, rng, query_length_hint=None, language="English"):
- """Build a prompt asking Gemini to generate a batch of examples."""
- scenarios_sample = rng.sample(SCENARIOS, min(20, len(SCENARIOS)))
- scenarios_str = "\n".join(f" - {s}" for s in scenarios_sample)
- context_seed = rng.choice(_CONTEXT_SEEDS)
-
- length_instruction = ""
- if query_length_hint:
- length_instruction = f"\n- Query length for this batch: {query_length_hint}"
-
- if language != "English":
- language_instruction = (
- f"\n- LANGUAGE: ALL queries MUST be written in {language}. "
- f"Use natural, native-sounding {language} — not literal translations from English. "
- f"Include culturally appropriate names, places, idioms, and references for {language}-speaking users. "
- f"Tool names and parameter keys stay in English (they are API identifiers), but string argument VALUES "
- f"(message text, notes, search queries, labels, destinations, etc.) should be in {language} where a real user would use {language}."
- )
- else:
- language_instruction = ""
-
- if tools:
- tools_section = f"AVAILABLE TOOLS:\n{json.dumps(tools, separators=(',', ':'))}"
-
- # Check if overlapping tools are present for disambiguation instruction
- tool_names = [t["name"] for t in tools]
- has_overlaps = any(
- a["name"] in tool_names and b["name"] in tool_names
- for a, b in _OVERLAP_PAIRS
- )
- disambig_rule = ""
- if has_overlaps:
- disambig_rule = ("\n- IMPORTANT: Some tools in this set have OVERLAPPING functionality. "
- "When generating queries, make sure the user's intent clearly maps to ONE specific tool. "
- "For ambiguous cases where the query could match either tool, pick the MOST specific one "
- "(e.g. 'text mom' → send_text_message, 'message mom on WhatsApp' → send_instant_message).")
-
- tool_rules = f"""- Tool call format: {{"name": "tool_name", "arguments": {{"param": "value"}}}}
-- Arguments must match the parameter schemas exactly — use correct types (string, number, boolean)
-- Do NOT invent tools not in the list — only use the tools shown above
-- For contact_id params, use realistic placeholders like "contact_firstname_NNN" with varied names and numbers — never repeat the same contact_id across examples
-- For evaluate_js, write real working JavaScript with console.log()
-- Number params should be actual numbers not strings (e.g. 7 not "7")
-- Boolean params should be actual booleans not strings (e.g. true not "true")
-- Vary argument values widely — don't repeat the same locations, names, times, or phrases across examples
-- Sometimes include optional parameters, sometimes omit them — mix it up naturally
-- Never produce partial tool calls — every call must have "name" and "arguments" with all required params
-- Include REALISTIC argument values: names from MANY diverse cultures and languages (Arabic, East Asian, South Asian, European, African, Latin American — never repeat the same name across examples in this batch), emoji in messages, decimal numbers, varied time formats ('7am', '19:30', 'quarter past 3')
-- CRITICAL GROUNDING RULE: argument values must come DIRECTLY from the query. Do NOT invent specific names, brands, numbers, or details the user didn't say. If the user says "streaming service", use "streaming service" not "Netflix". If they say "furniture store", use "furniture store" not "IKEA". If they say "set a timer for the pizza", use label "pizza" not "pepperoni pizza". The user's EXACT words should appear in the arguments. Only contact_id params may use placeholders like "contact_alice_123".
-- For album names, labels, titles: use ONLY words the user actually said — do not embellish or add creative flourishes{disambig_rule}"""
- else:
- tools_section = "AVAILABLE TOOLS: NONE — no tools are available."
- tool_rules = "- Since no tools are available, ALL answers must be empty arrays []"
-
- return f"""Generate {batch_size} diverse on-device assistant tool-calling training examples for phones, wearables, and computers.
-
-USER CONTEXT HINT (use as inspiration for ~half the examples, vary the rest freely): {context_seed}
-
-{tools_section}
-
-REQUIREMENTS:
-- Each example: a "query" (user's natural language) and "answers" (JSON tool calls array)
-- This batch: {call_desc}
-- Queries should sound like real users talking to a phone/computer/watch voice assistant or typing a quick command{length_instruction}{language_instruction}
-- Vary style: casual, terse, polite ("Could you please..."), conversational, indirect ("it's dark in here" meaning turn on lights)
-{tool_rules}
-- For empty call examples, answers must be []
-- Every query must be UNIQUE — do not repeat patterns or rephrase the same intent
-- CRITICAL: use DIFFERENT tools across examples in this batch — do NOT generate multiple examples that all call the same tool. Spread usage across ALL available tools. If there are 6 tools, each should appear in at least one example
-- Vary which tools are used and how many are called per example — some examples should use tools from different ends of the list
-
-THEME INSPIRATION — use these as starting points, but generate your OWN unique queries with different names, places, numbers, and details every time. NEVER copy these verbatim:
-{scenarios_str}
-
-OUTPUT FORMAT — return a JSON array, nothing else:
-[
- {{
- "query": "user's request",
- "answers": [{{"name": "tool_name", "arguments": {{"param": "value"}}}}]
- }}
-]
-
-Return ONLY valid JSON. No markdown, no explanation."""
-
-def make_clients():
- """Create Gemini clients from GEMINI_API_KEY (comma-separated for multiple keys)."""
- raw = os.environ.get("GEMINI_API_KEY", "")
- keys = [k.strip() for k in raw.split(",") if k.strip()]
- if not keys:
- print("Error: GEMINI_API_KEY environment variable not set.", file=sys.stderr)
- print("Set one or more comma-separated keys:", file=sys.stderr)
- print(" export GEMINI_API_KEY=key1,key2,key3", file=sys.stderr)
- print("Get keys at https://aistudio.google.com/apikey", file=sys.stderr)
- sys.exit(1)
- clients = [genai.Client(api_key=k) for k in keys]
- print(f"Using {len(clients)} API key(s)")
- return clients
-
-
-class ClientPool:
- """Round-robin pool of Gemini clients for distributing requests across API keys."""
-
- def __init__(self, clients):
- self._clients = clients
- self._idx = 0
- self._lock = __import__("threading").Lock()
-
- def get(self):
- with self._lock:
- client = self._clients[self._idx % len(self._clients)]
- self._idx += 1
- return client
-
-
-
-# Overlapping tool pairs — semantically similar tools that require disambiguation
-_OVERLAP_PAIRS = [
- (
- {"name": "send_text_message", "description": "Send a text message to a contact via SMS.", "parameters": {"contact_id": {"type": "string", "description": "The contact to message.", "required": True}, "text": {"type": "string", "description": "The message text.", "required": True}}},
- {"name": "send_instant_message", "description": "Send an instant message to a contact.", "parameters": {"contact_id": {"type": "string", "description": "The unique identifier of the recipient contact.", "required": True}, "text": {"type": "string", "description": "The message text to send.", "required": True}}},
- ),
- (
- {"name": "send_email", "description": "Send an email to a recipient.", "parameters": {"to": {"type": "string", "description": "The recipient's email address.", "required": True}, "subject": {"type": "string", "description": "The email subject line.", "required": True}, "body": {"type": "string", "description": "The email body text.", "required": True}}},
- {"name": "compose_email", "description": "Compose and send an email.", "parameters": {"to": {"type": "string", "description": "Recipient email address.", "required": True}, "subject": {"type": "string", "description": "Email subject line.", "required": True}, "body": {"type": "string", "description": "Email body text.", "required": True}, "cc": {"type": "string", "description": "Optional CC recipients.", "required": False}}},
- ),
- (
- {"name": "create_reminder", "description": "Create a reminder for the user at a specific time.", "parameters": {"message": {"type": "string", "description": "The reminder message.", "required": True}, "date_time_human": {"type": "string", "description": "The date and/or time for the reminder.", "required": False}}},
- {"name": "create_list_item", "description": "Add a new item to a user's list with an optional reminder.", "parameters": {"list_name": {"type": "string", "description": "Short name of the list.", "required": True}, "message": {"type": "string", "description": "The text of the list item.", "required": True}, "reminder_date_time_human": {"type": "string", "description": "Optional reminder date/time.", "required": False}}},
- ),
- (
- {"name": "get_directions", "description": "Get directions to a destination.", "parameters": {"destination": {"type": "string", "description": "The destination address or place name.", "required": True}, "mode": {"type": "string", "description": "Travel mode: 'driving', 'walking', 'transit', 'cycling'.", "required": False}}},
- {"name": "start_navigation", "description": "Start turn-by-turn navigation to a destination.", "parameters": {"destination": {"type": "string", "description": "The destination.", "required": True}, "mode": {"type": "string", "description": "Travel mode.", "required": False}, "avoid_tolls": {"type": "boolean", "description": "Whether to avoid toll roads.", "required": False}}},
- ),
- (
- {"name": "play_music", "description": "Play music by song name, artist, album, or genre.", "parameters": {"query": {"type": "string", "description": "Search query: song title, artist name, album, or genre.", "required": True}}},
- {"name": "play_playlist", "description": "Play a specific playlist by name.", "parameters": {"playlist_name": {"type": "string", "description": "Name of the playlist to play.", "required": True}}},
- ),
- (
- {"name": "set_timer", "description": "Set a timer for the specified duration or end time.", "parameters": {"time_human": {"type": "string", "description": "The duration or target end time in human readable format.", "required": True}}},
- {"name": "set_cooking_timer", "description": "Set a named cooking timer.", "parameters": {"label": {"type": "string", "description": "Label for the timer.", "required": True}, "minutes": {"type": "number", "description": "Timer duration in minutes.", "required": True}}},
- ),
-]
-
-
-_QUERY_LENGTH_DESCS = [
- "ultra-short (1-5 words, like 'lights off', 'timer 5', 'call mom')",
- "short (5-10 words, like 'set an alarm for 7am tomorrow')",
- "medium (10-20 words, a normal sentence or two)",
- "long conversational (20+ words, with filler, context, or explanation)",
-]
-
-
-# Words that are common filler / can reasonably appear in argument values
-# without being in the query (articles, prepositions, generic action words).
-_GROUNDING_STOPWORDS = frozenset({
- "a", "an", "the", "of", "to", "in", "on", "at", "for", "and", "or",
- "is", "it", "my", "me", "i", "you", "your", "this", "that", "with",
- "from", "by", "as", "be", "do", "not", "no", "so", "up", "out",
- "if", "but", "all", "just", "about", "can", "will", "am", "are",
- "was", "were", "been", "has", "have", "had", "would", "could",
- "should", "may", "might", "shall", "its", "our", "we", "they",
- "them", "their", "he", "she", "him", "her", "his", "who", "what",
- "when", "where", "how", "which", "some", "any", "each", "every",
- "also", "than", "then", "now", "here", "there", "very", "too",
- "more", "much", "many", "most", "other", "new", "old", "get",
- "set", "make", "take", "let", "got", "going", "gonna", "want",
- "need", "like", "please", "thanks", "hey", "hi", "ok", "okay",
- "sure", "yes", "no", "true", "false", "null", "none",
- "don't", "doesn't", "didn't", "won't", "can't", "couldn't",
- "shouldn't", "wouldn't", "isn't", "aren't", "wasn't", "weren't",
-})
-
-# Param names where values are expected to be generated/inferred, not quoted from query
-_FREEFORM_PARAMS = frozenset({
- "contact_id", "password", "js", "url", "source", "confirmation_code",
- "order_id", "post_id",
-})
-
-
-def _grounding_check(pname, pval, pdesc, query, query_lower):
- """Check that significant words in a string argument value are grounded in the query.
-
- Returns False if the value contains specific details not traceable to the query.
- Strict: every specific noun/name/identifier must come from the query.
- """
- # Skip params where values are generated/inferred, not extracted from query
- if pname in _FREEFORM_PARAMS:
- return True
-
- val = pval.strip()
- if not val:
- return True
-
- # Skip if the param description suggests free-form/generated content
- pdesc_lower = pdesc.lower() if pdesc else ""
- freeform_hints = ("url", "link", "password", "code", "javascript", "js ",
- "email address", "phone number")
- if any(hint in pdesc_lower for hint in freeform_hints):
- return True
-
- # For enum-like values that match description options, always allow
- # (e.g., "driving", "bake", "front" — these are valid fixed choices)
- quoted = re.findall(r"'([^']+)'", pdesc_lower)
- if quoted and val.lower() in [q.lower() for q in quoted]:
- return True
-
- # Extract significant words from value and query
- # Covers Latin, Latin Extended, Cyrillic, Greek, and combining diacriticals
- _WORD_RE = r'[\w\u00C0-\u024F\u0370-\u03FF\u0400-\u04FF\u0500-\u052F]{3,}'
- val_words = set()
- for word in re.findall(_WORD_RE, val.lower()):
- if word not in _GROUNDING_STOPWORDS:
- val_words.add(word)
-
- if not val_words:
- return True
-
- query_words = set()
- for word in re.findall(_WORD_RE, query_lower):
- if word not in _GROUNDING_STOPWORDS:
- query_words.add(word)
-
- # Check grounding: every significant value word must trace to the query
- grounded = 0
- for vw in val_words:
- if vw in query_words:
- grounded += 1
- continue
- # Substring/stem match (catches "grocery"/"groceries", "alarm"/"alarms")
- if any(vw in qw or qw in vw for qw in query_words):
- grounded += 1
- continue
- # Raw substring in query (catches compound words, multi-word names)
- if vw in query_lower:
- grounded += 1
- continue
-
- # Strict threshold: 75% of value words must be grounded.
- # For short values (≤3 significant words), require ALL to be grounded —
- # this catches "Netflix" from "streaming service", "IKEA" from "furniture store".
- if len(val_words) <= 3:
- if grounded < len(val_words):
- return False
- else:
- grounding_ratio = grounded / len(val_words)
- if grounding_ratio < 0.75:
- return False
-
- # Additional hard checks for specific fabrication patterns
- # Reject fabricated phone numbers in non-phone params
- phones = re.findall(r'[\+]?[\d\s\-\(\)]{7,}', val)
- if phones and pname not in ("phone", "phone_number"):
- query_digits = set(re.findall(r'\d{3,}', query))
- val_digits = set(re.findall(r'\d{3,}', val))
- if val_digits and not val_digits & query_digits:
- return False
-
- # Reject fabricated email addresses in non-email params
- emails = re.findall(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}', val)
- if emails and pname not in ("to", "email", "cc", "bcc", "from_contact"):
- for email in emails:
- local = email.split("@")[0].replace(".", " ").replace("_", " ")
- if not any(w.lower() in query_lower for w in local.split() if len(w) > 2):
- return False
-
- return True
-
-
-def _semantic_check(tool_name, args, schema, query, call_type="single"):
- """Lightweight rule-based semantic validation of argument values.
-
- Returns False if any argument value is obviously nonsensical.
- Catches the most common Gemini hallucination patterns without needing an LLM judge.
- """
- query_lower = query.lower()
-
- for pname, pval in args.items():
- pinfo = schema.get(pname, {})
- expected_type = pinfo.get("type", "string")
- pdesc = pinfo.get("description", "").lower()
-
- # Number range checks
- if expected_type == "number" and isinstance(pval, (int, float)):
- if any(kw in pname for kw in ("level", "brightness", "percent")):
- if pval < 0 or pval > 100:
- return False
- if pname == "temperature" and "thermostat" in tool_name:
- if pval < 40 or pval > 100:
- return False
- if any(kw in pname for kw in ("minutes", "duration", "hours", "seconds", "count")):
- if pval < 0:
- return False
- if pname == "rating":
- if pval < 1 or pval > 5:
- return False
-
- # String emptiness checks
- if expected_type == "string" and isinstance(pval, str):
- if pinfo.get("required") and not pval.strip():
- return False
- val_lower = pval.strip().lower()
- if val_lower in ("", "null", "none", "undefined", "n/a", "todo",
- "example", "test", "placeholder", "your_",
- "insert", "[", "{", "..."):
- if pinfo.get("required"):
- return False
-
- # Enum-like string checks from description
- if expected_type == "string" and isinstance(pval, str) and pdesc:
- quoted = re.findall(r"'([^']+)'", pdesc)
- if len(quoted) >= 2: # looks like an enum
- if pval.lower() not in [q.lower() for q in quoted]:
- if len(pval) < 30:
- return False
-
- # check alignment with query intent
- if expected_type == "boolean" and isinstance(pval, bool):
- if pname == "enabled":
- disable_words = ("off", "disable", "stop", "don't", "no ", "without")
- enable_words = ("on", "enable", "start", "turn on", "activate")
- query_wants_off = any(w in query_lower for w in disable_words)
- query_wants_on = any(w in query_lower for w in enable_words)
- # Only reject clear contradictions — if ambiguous, allow
- if query_wants_off and not query_wants_on and pval is True:
- return False
- if query_wants_on and not query_wants_off and pval is False:
- return False
-
- # Grounding check: argument values must be traceable to the query.
- # Skip for call types where semantic inference is the point (indirect,
- # disfluent, garbled) — those intentionally have values not literally in the query.
- # For multi_long_values, only skip grounding on free-text content params
- # (message bodies, notes, etc.) — not on identifier params like names/sites.
- _SKIP_GROUNDING_TYPES = ("indirect", "disfluent", "garbled")
- _LONG_CONTENT_PARAMS = frozenset({
- "text", "body", "message", "note", "new_text", "items",
- "description", "feedback", "actions", "js",
- })
- skip_grounding = (
- call_type in _SKIP_GROUNDING_TYPES
- or (call_type == "multi_long_values" and pname in _LONG_CONTENT_PARAMS)
- )
- if expected_type == "string" and isinstance(pval, str) and not skip_grounding:
- ok = _grounding_check(pname, pval, pdesc, query, query_lower)
- if not ok:
- return False
-
- return True
-
-
-# Description rephrase templates — vary how tool descriptions are worded
-# without changing the meaning. Prevents the model from memorizing exact phrasing.
-_DESC_TEMPLATES = [
- lambda d: d, # original
- lambda d: d, # original (weighted higher)
- lambda d: d, # original (weighted higher)
- lambda d: d.rstrip(".") + "." if not d.endswith(".") else d, # normalize period
- lambda d: d[0].upper() + d[1:] if d and d[0].islower() else d, # capitalize
- lambda d: d.replace(".", ",", 1).rstrip(",") + "." if d.count(".") > 1 else d, # comma variant
- lambda d: d.split(".")[0].strip() + "." if "." in d else d, # first sentence only
- lambda d: d.replace(" the ", " a ", 1) if " the " in d else d, # article swap
- lambda d: d.replace("Enable or disable", "Toggle").replace("enable or disable", "toggle") if "nable or disable" in d else d,
- lambda d: d.replace("Turn ", "Switch ").replace("turn ", "switch ") if d.startswith(("Turn", "turn")) else d,
-]
-
-
-# Domain categories for tool synthesis — Gemini invents tools within these bounds
-_SYNTH_DOMAINS = [
- "smart kitchen appliances (oven, fridge, blender, air purifier, water filter)",
- "pet care and pet tech (feeder, tracker, health monitor, activity log)",
- "personal finance and budgeting (subscriptions, savings goals, expense tracking)",
- "study and education tools (flashcards, quiz, study timer, citation generator)",
- "creative tools (drawing, music composition, video editing, photo collage)",
- "gardening and plant care (watering schedule, soil sensor, plant identifier, weather alerts)",
- "meeting and collaboration (screen share, whiteboard, poll, action items, transcription)",
- "home maintenance (HVAC filter, smoke detector, water leak sensor, appliance warranty)",
- "sports and outdoor activities (route planner, gear checklist, weather for trail, pace tracker)",
- "baby and childcare (feeding log, diaper tracker, sleep monitor, milestone tracker)",
- "subscription and account management (cancel subscription, change plan, check renewal date)",
- "custom device peripherals (drone control, 3D printer, smart mirror, e-ink display)",
- "language learning (vocabulary drill, pronunciation check, daily lesson, streak tracker)",
- "car maintenance (oil change reminder, tire rotation schedule, mileage log, service history)",
- "event planning (guest list, RSVP tracker, venue search, seating chart, send invitations)",
-]
-
-_SYNTH_TOOL_PROMPT = """Generate {num_tools} realistic tool definitions for a consumer device assistant in this domain: {domain}
-
-Each tool must have:
-- "name": snake_case function name
-- "description": clear 1-sentence description of what it does
-- "parameters": dict of param_name -> {{"type": "string"|"number"|"boolean", "description": "...", "required": true|false}}
-
-Rules:
-- Tools should feel like real device/app APIs — specific, actionable, not vague
-- Mix of tools with 0-4 parameters
-- Include both action tools (do something) and query tools (get information)
-- Parameter types must be realistic (numbers for quantities, booleans for toggles, strings for names/text)
-- Use snake_case for all names
-
-Return ONLY a valid JSON array of tool objects. No markdown, no explanation."""
-
-
-_synth_stats = {"attempted": 0, "succeeded": 0, "failed_parse": 0, "failed_validate": 0, "failed_exception": 0}
-
-
-def _synthesize_tools(client_pool, rng, model, num_tools):
- """Ask Gemini to invent novel tool definitions within a random domain."""
- _synth_stats["attempted"] += 1
- domain = rng.choice(_SYNTH_DOMAINS)
- prompt = _SYNTH_TOOL_PROMPT.format(num_tools=num_tools, domain=domain)
-
- client = client_pool.get()
- try:
- response = client.models.generate_content(
- model=model, contents=prompt,
- config={"temperature": 0.9, "max_output_tokens": 4096},
- )
- text = response.text.strip()
- if text.startswith("```"):
- text = text.split("\n", 1)[1] if "\n" in text else text[3:]
- if text.endswith("```"):
- text = text[:-3]
- tools = json.loads(text.strip())
- if not isinstance(tools, list):
- _synth_stats["failed_parse"] += 1
- return None
- # Validate and normalize structure to match our pool format
- valid = []
- for t in tools:
- if not isinstance(t, dict):
- continue
- if not t.get("name") or not isinstance(t.get("name"), str):
- continue
- if not t.get("description") or not isinstance(t.get("description"), str):
- continue
- # Ensure snake_case name
- t["name"] = re.sub(r'[^a-z0-9_]', '_', t["name"].lower()).strip('_')
- # Normalize parameters to match our schema format
- raw_params = t.get("parameters", {})
- if not isinstance(raw_params, dict):
- t["parameters"] = {}
- else:
- normalized = {}
- for pname, pinfo in raw_params.items():
- if isinstance(pinfo, dict):
- normalized[pname] = {
- "type": pinfo.get("type", "string"),
- "description": pinfo.get("description", ""),
- "required": pinfo.get("required", False),
- }
- elif isinstance(pinfo, str):
- # Gemini sometimes produces {"param": "description string"}
- normalized[pname] = {
- "type": "string",
- "description": pinfo,
- "required": True,
- }
- t["parameters"] = normalized
- valid.append(t)
- if not valid:
- _synth_stats["failed_validate"] += 1
- return None
- _synth_stats["succeeded"] += 1
- return valid[:num_tools]
- except Exception as e:
- _synth_stats["failed_exception"] += 1
- return None
-
-
-def _rephrase_tool_descriptions(tools, rng):
- """Apply random description rephrasing to tool definitions."""
- result = []
- for t in tools:
- t = dict(t) # shallow copy
- template = rng.choice(_DESC_TEMPLATES)
- t["description"] = template(t["description"])
- # Also occasionally rephrase parameter descriptions
- if isinstance(t.get("parameters"), dict):
- params = {}
- for pname, pinfo in t["parameters"].items():
- pinfo = dict(pinfo)
- if rng.random() < 0.3 and "description" in pinfo:
- pinfo["description"] = rng.choice(_DESC_TEMPLATES)(pinfo["description"])
- params[pname] = pinfo
- t["parameters"] = params
- result.append(t)
- return result
-
-
-def generate_batch(client_pool, batch_size, rng, model, language="English"):
- """Generate one batch of examples. Returns list of dicts."""
- call_type, call_desc = rng.choice(CALL_TYPES)
-
- # ~15% of batches use synthesized novel tools for generalization
- # Only for call types that benefit from multiple tools (not few_tools types)
- use_synth = (
- call_type not in ("no_tools", "multi_few_tools")
- and rng.random() < 0.50
- )
-
- if use_synth:
- # Synthesize at least 3 tools so we get variety, up to 8
- target = max(3, rng.choices(_TOOL_COUNTS, weights=_TOOL_WEIGHTS, k=1)[0])
- target = min(target, 8)
- tools = _synthesize_tools(client_pool, rng, model, target)
- if tools is None:
- use_synth = False
- tools = _pick_tools(rng, force_empty=False,
- few_tools=(call_type in ("multi_few_tools", "multi_long_values")))
- else:
- tools = _pick_tools(
- rng,
- force_empty=(call_type == "no_tools"),
- few_tools=(call_type in ("multi_few_tools", "multi_long_values")),
- )
-
- # Inject overlapping tools ~20% of the time (only for pool tools)
- if tools and not use_synth and len(tools) >= 2 and rng.random() < 0.20:
- pair = rng.choice(_OVERLAP_PAIRS)
- idx = rng.randint(0, len(tools) - 1)
- tools[idx] = pair[0]
- if len(tools) < MAX_TOOLS:
- tools.insert(idx + 1, pair[1])
- else:
- other = (idx + 1) % len(tools)
- tools[other] = pair[1]
-
- # Rephrase tool descriptions ~30% of the time for wording diversity
- if tools and rng.random() < 0.30:
- tools = _rephrase_tool_descriptions(tools, rng)
-
- # Pick a query length bucket for this batch
- query_length_hint = rng.choice(_QUERY_LENGTH_DESCS)
-
- prompt = build_prompt(batch_size, call_desc, tools, rng, query_length_hint=query_length_hint, language=language)
-
- # Vary temperature per batch for diversity: mostly 0.9-1.1 with occasional
- # low (more precise args) and high (more creative queries) extremes.
- temperature = rng.choice([0.7, 0.8, 0.9, 1.0, 1.0, 1.0, 1.1, 1.1, 1.2, 1.3])
-
- client = client_pool.get()
- response = client.models.generate_content(
- model=model,
- contents=prompt,
- config={
- "temperature": temperature,
- "max_output_tokens": 16384,
- },
- )
-
- text = response.text.strip()
- if text.startswith("```"):
- text = text.split("\n", 1)[1] if "\n" in text else text[3:]
- if text.endswith("```"):
- text = text[:-3]
- text = text.strip()
-
- try:
- examples = json.loads(text)
- except json.JSONDecodeError:
- return []
-
- if not isinstance(examples, list):
- return []
-
- valid = []
- tools_str = json.dumps(tools, separators=(",", ":"), ensure_ascii=False)
- tool_name_set = {t["name"] for t in tools}
-
- # Build schema map for parameter validation: {tool_name: {param_name: {type, required}}}
- tool_schema = {}
- for t in tools:
- params = t.get("parameters", {})
- tool_schema[t["name"]] = {
- k: {"type": v.get("type", "string"), "required": v.get("required", False)}
- for k, v in params.items()
- } if isinstance(params, dict) else {}
-
- # Build keyword set for rejecting bad empty-answer examples
- # Maps tool descriptions to simple action words for fuzzy matching
- _tool_keywords = set()
- for t in tools:
- _tool_keywords.add(t["name"].replace("_", " "))
- for word in t.get("description", "").lower().split():
- if len(word) > 4 and word not in ("the", "from", "with", "that", "this", "about", "which", "their", "optional"):
- _tool_keywords.add(word)
-
- for ex in examples:
- if not isinstance(ex, dict):
- continue
- query = ex.get("query", "").strip()
- answers = ex.get("answers")
- if not query or answers is None:
- continue
- if not isinstance(answers, list):
- continue
-
- ok = True
- for call in answers:
- if not isinstance(call, dict):
- ok = False
- break
- cname = call.get("name")
- if cname not in tool_name_set:
- ok = False
- break
- args = call.get("arguments", {})
- if not isinstance(args, dict):
- ok = False
- break
-
- # Validate required params are present
- schema = tool_schema.get(cname, {})
- for pname, pinfo in schema.items():
- if pinfo["required"] and pname not in args:
- ok = False
- break
-
- # Validate and coerce argument types
- if ok:
- for pname, pval in list(args.items()):
- if pname not in schema:
- continue # extra params — tolerate
- expected = schema[pname]["type"]
- if expected == "number" and isinstance(pval, str):
- try:
- args[pname] = float(pval) if "." in pval else int(pval)
- except ValueError:
- ok = False
- break
- elif expected == "boolean" and isinstance(pval, str):
- if pval.lower() in ("true", "false"):
- args[pname] = pval.lower() == "true"
- else:
- ok = False
- break
-
- # Lightweight semantic checks on argument values
- if ok:
- ok = _semantic_check(cname, args, schema, query, call_type)
-
- if not ok:
- break
- if not ok:
- continue
-
- # Reject empty answers that look like they should have tool calls.
- # Skip this filter for "near_miss" — those intentionally have related keywords.
- if len(answers) == 0 and tools and call_type == "none":
- query_lower = query.lower()
- if any(kw in query_lower for kw in _tool_keywords if len(kw) > 5):
- continue # skip — query matches a tool but answer is empty
-
- valid.append({
- "query": query,
- "tools": tools_str,
- "answers": json.dumps(answers, separators=(",", ":"), ensure_ascii=False),
- "source": "synth-gemini-assistant",
- "model": model,
- "call_type": call_type,
- "num_tools": len(tools),
- "synth_tools": use_synth,
- "language": language,
- })
-
- return valid
-
-
-def generate_all(num_samples, workers=8, batch_size=25, model=MODEL, client_pool=None):
- """Generate num_samples examples using parallel Gemini calls."""
- if client_pool is None:
- client_pool = ClientPool(make_clients())
- rng = random.Random(42)
-
- target = int(num_samples * 1.3)
- max_submissions = target // batch_size * 3 # hard cap to prevent infinite loop
- all_examples = []
- failed = 0
-
- pbar = tqdm(total=num_samples, desc="Generating", unit="ex")
-
- with ThreadPoolExecutor(max_workers=workers) as pool:
- pending = set()
- submitted = 0
-
- def _submit_one():
- nonlocal submitted
- batch_rng = random.Random(rng.randint(0, 2**32))
- language = LANGUAGES[submitted % len(LANGUAGES)]
- f = pool.submit(generate_batch, client_pool, batch_size, batch_rng, model, language=language)
- pending.add(f)
- submitted += 1
-
- initial = min(workers, (target + batch_size - 1) // batch_size)
- for _ in range(initial):
- _submit_one()
-
- while pending:
- done, pending = concurrent.futures.wait(
- pending, return_when=concurrent.futures.FIRST_COMPLETED,
- )
- for f in done:
- try:
- results = f.result()
- all_examples.extend(results)
- except Exception as e:
- failed += 1
- pbar.n = min(len(all_examples), num_samples)
- pbar.set_postfix(failed=failed)
- pbar.refresh()
- if len(all_examples) < target and submitted < max_submissions:
- _submit_one()
-
- pbar.close()
-
- def _dedup_key(query):
- """Normalize query for dedup: lowercase, strip punctuation/whitespace."""
- return " ".join(query.lower().split()).strip(".,!?;:'\"")
-
- seen = set()
- deduped = []
- for ex in all_examples:
- key = _dedup_key(ex["query"])
- if key not in seen:
- seen.add(key)
- deduped.append(ex)
-
- if len(deduped) > num_samples:
- deduped = deduped[:num_samples]
-
- # Log distribution stats
- from collections import Counter
- type_counts = Counter(ex.get("call_type", "unknown") for ex in deduped)
- tool_counts = Counter(ex.get("num_tools", -1) for ex in deduped)
- print(f"Generated {len(deduped):,} unique examples ({len(all_examples) - len(deduped)} duplicates removed, {failed} failed batches)")
- print(f" Call type distribution: {dict(type_counts.most_common())}")
- print(f" Tool count distribution: {dict(sorted(tool_counts.items()))}")
- lang_counts = Counter(ex.get("language", "English") for ex in deduped)
- print(f" Language distribution: {dict(lang_counts.most_common())}")
- if _synth_stats["attempted"] > 0:
- print(f" Tool synthesis: {_synth_stats['succeeded']}/{_synth_stats['attempted']} succeeded "
- f"(parse_fail={_synth_stats['failed_parse']}, validate_fail={_synth_stats['failed_validate']}, "
- f"exception={_synth_stats['failed_exception']})")
- return deduped
-
-
-
-LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")
-HF_DATASET_REPO = "Cactus-Compute/tool-calls"
-UPLOAD_EVERY = 100000
-
-
-def _load_existing():
- """Load existing dataset from disk or HuggingFace."""
- from datasets import load_dataset, load_from_disk
-
- local = os.path.abspath(LOCAL_UNIFIED_DIR)
- if os.path.exists(local) and any(f.endswith(".arrow") for f in os.listdir(local)):
- ds = load_from_disk(local)
- print(f"Loaded existing dataset: {len(ds)} rows")
- else:
- print(f"Downloading existing dataset from {HF_DATASET_REPO}...")
- from .dataset import download_hf_split
- ds = download_hf_split("train", HF_DATASET_REPO)
- # Save locally so subsequent chunks don't re-download
- os.makedirs(local, exist_ok=True)
- ds.save_to_disk(local)
- # Clear HF download cache to avoid double storage
- hf_cache = os.path.expanduser("~/.cache/huggingface/datasets")
- if os.path.exists(hf_cache):
- import shutil
- shutil.rmtree(hf_cache)
- print(f"Cleared HF dataset cache ({hf_cache})")
- print(f"Downloaded: {len(ds)} rows")
- return ds
-
-
-def _merge_and_upload(existing, new_examples):
- """Merge new examples into existing dataset, save locally, and upload."""
- import shutil
- from collections import Counter
- from datasets import Dataset, concatenate_datasets
-
- new_ds = Dataset.from_dict({
- "query": [ex["query"] for ex in new_examples],
- "tools": [ex["tools"] for ex in new_examples],
- "answers": [ex["answers"] for ex in new_examples],
- "source": [ex["source"] for ex in new_examples],
- "model": [ex["model"] for ex in new_examples],
- "language": [ex.get("language", "English") for ex in new_examples],
- })
- # Keep only columns that exist in the existing dataset (language may be new)
- overlap_cols = [c for c in new_ds.column_names if c in existing.column_names]
- extra_cols = [c for c in new_ds.column_names if c not in existing.column_names]
- if extra_cols:
- # Add missing columns to existing dataset with defaults
- for col in extra_cols:
- existing = existing.add_column(col, ["English"] * len(existing) if col == "language" else [""] * len(existing))
- new_ds = new_ds.select_columns(existing.column_names)
-
- merged = concatenate_datasets([existing, new_ds])
- print(f"\nMerged: {len(merged)} rows (+{len(new_ds)} new)")
-
- for src, cnt in Counter(merged["source"]).most_common():
- print(f" {src}: {cnt}")
-
- local = os.path.abspath(LOCAL_UNIFIED_DIR)
- tmp_dir = local + "_tmp"
- if os.path.exists(tmp_dir):
- shutil.rmtree(tmp_dir)
- merged.save_to_disk(tmp_dir)
- if os.path.exists(local):
- shutil.rmtree(local)
- os.rename(tmp_dir, local)
- print("Saved locally.")
-
- import tempfile
- from huggingface_hub import HfApi, CommitOperationDelete
-
- api = HfApi()
- api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)
-
- # Export to parquet and upload via HfApi to avoid push_to_hub dataset card bug
- parquet_dir = tempfile.mkdtemp(prefix="needle_upload_")
- print(f"Exporting to parquet...")
- merged.to_parquet(os.path.join(parquet_dir, "train.parquet"))
-
- # Delete old train shards first
- files = api.list_repo_files(HF_DATASET_REPO, repo_type="dataset", token=True)
- old_train = [f for f in files if f.startswith("data/train-")]
- if old_train:
- print(f"Deleting {len(old_train)} old train shards...")
- ops = [CommitOperationDelete(path_in_repo=f) for f in old_train]
- api.create_commit(
- repo_id=HF_DATASET_REPO, repo_type="dataset", operations=ops,
- commit_message="Remove old train shards before re-upload", token=True,
- )
-
- print(f"Uploading to {HF_DATASET_REPO} (train split)...")
- api.upload_file(
- path_or_fileobj=os.path.join(parquet_dir, "train.parquet"),
- path_in_repo="data/train-00000-of-00001.parquet",
- repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,
- commit_message=f"Upload train data ({len(merged)} rows)",
- )
-
- import shutil as _shutil
- _shutil.rmtree(parquet_dir)
- print(f"Upload complete: {HF_DATASET_REPO}")
-
- return merged
-
-
-def main(args):
-
- client_pool = ClientPool(make_clients())
-
- remaining = args.num_samples
- total_generated = 0
- existing = None if args.dry_run else _load_existing()
- def _dedup_key(query):
- return " ".join(query.lower().split()).strip(".,!?;:'\"")
-
- seen_queries = set()
- if existing and not args.dry_run:
- seen_queries = set(_dedup_key(q) for q in existing["query"])
-
- while remaining > 0:
- chunk_size = min(remaining, args.upload_every)
- print(f"\n{'='*50}")
- print(f"Generating chunk: {chunk_size} samples ({total_generated}/{args.num_samples} done)")
- print(f"{'='*50}")
-
- examples = generate_all(chunk_size, args.workers, args.batch_size, args.model, client_pool=client_pool)
-
- if not examples:
- print("No examples generated in this chunk, stopping.")
- break
-
- fresh = []
- for ex in examples:
- key = _dedup_key(ex["query"])
- if key not in seen_queries:
- seen_queries.add(key)
- fresh.append(ex)
- if len(fresh) < len(examples):
- print(f" Removed {len(examples) - len(fresh)} cross-chunk duplicates")
- examples = fresh
-
- if not examples:
- print("All examples were duplicates, stopping.")
- break
-
- total_generated += len(examples)
- remaining -= len(examples)
-
- if args.output_jsonl:
- with open(args.output_jsonl, "a") as f:
- for ex in examples:
- f.write(json.dumps(ex) + "\n")
-
- if total_generated == len(examples):
- # Show synthesized tool examples first (if any)
- synth_examples = [ex for ex in examples if ex.get("synth_tools")]
- if synth_examples:
- print(f"\nSynthesized tool examples ({len(synth_examples)} total):")
- for ex in synth_examples[:10]:
- try:
- tool_names = [t["name"] for t in json.loads(ex["tools"])]
- tools_str = ", ".join(tool_names) if tool_names else "(no tools)"
- except (json.JSONDecodeError, TypeError):
- tools_str = "(no tools)"
- print(f" Tools: [{tools_str}] [SYNTH]")
- print(f" Q: {ex['query']}")
- print(f" A: {ex['answers'][:200]}")
- print()
-
- print(f"\nSample examples:")
- sample_indices = list(range(len(examples)))
- random.shuffle(sample_indices)
- for idx in sample_indices[:100]:
- ex = examples[idx]
- try:
- tool_names = [t["name"] for t in json.loads(ex["tools"])]
- tools_str = ", ".join(tool_names) if tool_names else "(no tools)"
- except (json.JSONDecodeError, TypeError):
- tools_str = "(no tools)"
- synth_tag = " [SYNTH]" if ex.get("synth_tools") else ""
- print(f" Tools: [{tools_str}]{synth_tag}")
- print(f" Q: {ex['query']}")
- print(f" A: {ex['answers'][:200]}")
- print()
-
- if args.dry_run:
- print(f" Chunk: {len(examples)} examples (dry-run, not uploading)")
- continue
-
- existing = _merge_and_upload(existing, examples)
-
- print(f"\nDone. Total generated: {total_generated:,}")
- if args.output_jsonl:
- print(f"Raw JSONL: {args.output_jsonl}")
-
-
-if __name__ == "__main__":
- import argparse as _ap
- _p = _ap.ArgumentParser(description="Generate on-device assistant tool-calling data with Gemini")
- _p.add_argument("--num-samples", type=int, default=500)
- _p.add_argument("--batch-size", type=int, default=10)
- _p.add_argument("--workers", type=int, default=8)
- _p.add_argument("--model", type=str, default=MODEL)
- _p.add_argument("--dry-run", action="store_true")
- _p.add_argument("--output-jsonl", type=str, default=None)
- _p.add_argument("--upload-every", type=int, default=UPLOAD_EVERY)
- main(_p.parse_args())
diff --git a/needle/dataset/tokenize.py b/needle/dataset/tokenize.py
deleted file mode 100644
index cfc446c..0000000
--- a/needle/dataset/tokenize.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.
-
-Downloads the tool-calling dataset from HuggingFace (Cactus-Compute/tool-calls),
-trains the SentencePiece tokenizer, and tokenizes train + val splits,
-caching everything locally.
-
-Usage:
- needle tokenize # full run
- needle tokenize --max-samples 1000 # dev/test
-"""
-
-import os
-import shutil
-
-from .dataset import (
- CACHE_DIR,
- DEFAULT_MAX_DEC_LEN,
- DEFAULT_MAX_ENC_LEN,
- LOCAL_UNIFIED_DIR,
- TOKENIZER_DIR,
- _cache_key,
- _save_cache_metadata,
- get_tokenizer,
- load_tool_calls,
- pack_sequences,
- prepare_tool_call_pairs,
- train_tokenizer,
-)
-
-
-_HF_MODEL_REPO = "Cactus-Compute/needle"
-_HF_TOKENIZED_REPO = "Cactus-Compute/tokenized-tool-calls"
-
-
-def _push_to_hf(cache_dir, tokenizer_dir):
- """Upload tokenizer and tokenized data to HuggingFace."""
- from huggingface_hub import HfApi
-
- api = HfApi()
-
- print(f"Uploading tokenizer to {_HF_MODEL_REPO} ...")
- for fname in os.listdir(tokenizer_dir):
- if fname.endswith((".model", ".vocab")):
- api.upload_file(
- path_or_fileobj=os.path.join(tokenizer_dir, fname),
- path_in_repo=f"tokenizer/{fname}",
- repo_id=_HF_MODEL_REPO,
- repo_type="model",
- )
-
- api.create_repo(_HF_TOKENIZED_REPO, repo_type="dataset", private=True, exist_ok=True)
- print(f"Uploading tokenized data to {_HF_TOKENIZED_REPO} ...")
- api.upload_folder(
- folder_path=cache_dir,
- repo_id=_HF_TOKENIZED_REPO,
- repo_type="dataset",
- allow_patterns=["*.npy", "*.json"],
- delete_patterns=["*.npy", "*.json"],
- )
-
- print("HuggingFace upload complete.")
-
-
-def _clear_local_caches():
- """Remove local .data_cache/, tokenizer/, and unified dataset directories."""
- for d in [CACHE_DIR, TOKENIZER_DIR, LOCAL_UNIFIED_DIR]:
- if os.path.exists(d):
- print(f"Removing {d}/ ...")
- shutil.rmtree(d)
-
-
-def _download_synth_dataset():
- """Download synthesized tool-calling dataset (train + validation) from HuggingFace."""
- from .dataset import download_hf_split
-
- print("Downloading dataset from HuggingFace (Cactus-Compute/tool-calls)...")
- for split in ("train", "validation"):
- try:
- ds = download_hf_split(split)
- except Exception as e:
- raise FileNotFoundError(
- f"Dataset split '{split}' not found on HuggingFace (Cactus-Compute/tool-calls): {e}\n"
- "Run 'python scripts/generate_data.py' first."
- )
- split_dir = os.path.join(LOCAL_UNIFIED_DIR, split)
- os.makedirs(split_dir, exist_ok=True)
- ds.save_to_disk(split_dir)
- print(f"Saved {split}: {len(ds):,} rows to {split_dir}")
-
-
-def tokenize(args):
- print("=== Clearing existing caches ===")
- _clear_local_caches()
-
- print("\n=== Downloading dataset from HuggingFace ===")
- _download_synth_dataset()
-
- print("\n=== Training tokenizer ===")
- train_tokenizer(max_samples=args.max_samples, force=True)
-
- tokenizer = get_tokenizer()
-
- print("\n=== Tokenizing text data ===")
- max_enc_len = getattr(args, "max_enc_len", DEFAULT_MAX_ENC_LEN)
- max_dec_len = getattr(args, "max_dec_len", DEFAULT_MAX_DEC_LEN)
-
- for split in ("train", "val"):
- print(f"\n--- {split} split ---")
- ds, global_indices = load_tool_calls(
- split=split,
- max_samples=args.max_samples,
- return_global_indices=True,
- )
- shuffle_tools = getattr(args, "shuffle_tools", True)
- max_tool_len = getattr(args, "max_tool_len", 256)
- prepare_tool_call_pairs._max_tool_len = max_tool_len
- enc_vl, dec_in_vl, dec_tgt_vl, loss_vl, kept_indices, _ = prepare_tool_call_pairs(
- ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,
- shuffle_tools=shuffle_tools,
- )
- text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
- 1.0, 1.0, 1.0, shuffle_tools)
- cache_path = os.path.join(CACHE_DIR, text_cache_id)
- pack_sequences(cache_path, enc_vl, dec_in_vl, dec_tgt_vl, loss_vl)
-
- _save_cache_metadata(split, text_cache_id, len(kept_indices),
- max_enc_len, max_dec_len, max_tool_len)
-
- _push_to_hf(CACHE_DIR, TOKENIZER_DIR)
-
- if os.path.exists(LOCAL_UNIFIED_DIR):
- print(f"\n=== Removing raw dataset ({LOCAL_UNIFIED_DIR}) ===")
- shutil.rmtree(LOCAL_UNIFIED_DIR)
-
- print("\n=== Tokenization pipeline complete ===")
diff --git a/needle/dataset/tokenizer.py b/needle/dataset/tokenizer.py
deleted file mode 100644
index efa2cc8..0000000
--- a/needle/dataset/tokenizer.py
+++ /dev/null
@@ -1,164 +0,0 @@
-import os
-import re as _re
-
-import sentencepiece as spm
-from tqdm import tqdm
-
-_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
-TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")
-TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
-
-PAD_ID = 0
-EOS_ID = 1
-BOS_ID = 2
-UNK_ID = 3
-TOOL_CALL_ID = 4
-TOOLS_ID = 5
-
-DEFAULT_MAX_ENC_LEN = 1024
-DEFAULT_MAX_DEC_LEN = 512
-DEFAULT_MAX_GEN_LEN = 512
-
-_HF_MODEL_REPO = "Cactus-Compute/needle"
-
-
-def to_snake_case(name):
- """Convert camelCase, PascalCase, or dot.notation name to snake_case."""
- # Replace any non-alphanumeric/underscore characters with underscores
- s = _re.sub(r'[^a-zA-Z0-9_]+', '_', name)
- # Insert underscore before uppercase letters that follow lowercase/digits
- s = _re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s)
- # Insert underscore between consecutive uppercase and uppercase+lowercase
- s = _re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', s)
- # Collapse multiple underscores and strip edges
- s = _re.sub(r'_+', '_', s)
- return s.lower().strip('_')
-
-
-class NeedleTokenizer:
- """Wrapper around SentencePiece providing the interface the codebase expects."""
-
- def __init__(self, model_path):
- self.sp = spm.SentencePieceProcessor()
- self.sp.Load(model_path)
-
- @property
- def pad_token_id(self):
- return PAD_ID
-
- @property
- def eos_token_id(self):
- return EOS_ID
-
- @property
- def bos_token_id(self):
- return BOS_ID
-
- @property
- def tool_call_token_id(self):
- return TOOL_CALL_ID
-
- @property
- def tools_token_id(self):
- return TOOLS_ID
-
- @property
- def vocab_size(self):
- return self.sp.GetPieceSize()
-
- def encode(self, text):
- return self.sp.Encode(text, out_type=int)
-
- def decode(self, ids):
- if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):
- return [self.sp.Decode(seq) for seq in ids]
- return self.sp.Decode(list(ids))
-
- def __call__(self, texts, truncation=True, max_length=None, **kwargs):
- all_ids = []
- for text in texts:
- ids = self.sp.Encode(text, out_type=int)
- if truncation and max_length:
- ids = ids[:max_length]
- all_ids.append(ids)
- return {"input_ids": all_ids}
-
-
-def _download_tokenizer_from_hf():
- """Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""
- from huggingface_hub import hf_hub_download
-
- os.makedirs(TOKENIZER_DIR, exist_ok=True)
- for fname in ["needle.model", "needle.vocab"]:
- hf_hub_download(
- repo_id=_HF_MODEL_REPO,
- filename=f"tokenizer/{fname}",
- repo_type="model",
- local_dir=TOKENIZER_DIR,
- force_download=True,
- )
- # hf_hub_download puts it in tokenizer/tokenizer/needle.model, move up
- nested = os.path.join(TOKENIZER_DIR, "tokenizer", fname)
- dst = os.path.join(TOKENIZER_DIR, fname)
- if os.path.exists(nested) and not os.path.exists(dst):
- os.rename(nested, dst)
-
-
-def get_tokenizer(max_samples=None):
- model_path = TOKENIZER_PREFIX + ".model"
- if not os.path.exists(model_path):
- print("Downloading pretrained tokenizer from HuggingFace...")
- _download_tokenizer_from_hf()
- return NeedleTokenizer(model_path)
-
-
-def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
- """Train a SentencePiece BPE tokenizer on tool-calling corpus."""
- model_path = TOKENIZER_PREFIX + ".model"
- if os.path.exists(model_path) and not force:
- print(f"Tokenizer already exists at {model_path}")
- return model_path
-
- os.makedirs(TOKENIZER_DIR, exist_ok=True)
-
- from datasets import concatenate_datasets
- from .dataset import _load_split_dataset
-
- train_ds = _load_split_dataset("train")
- val_ds = _load_split_dataset("validation")
- ds = concatenate_datasets([train_ds, val_ds])
- if max_samples:
- ds = ds.select(range(min(max_samples, len(ds))))
-
- print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size}, samples={len(ds):,})...")
-
- corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")
- try:
- with open(corpus_path, "w") as f:
- for example in tqdm(ds, desc="Writing corpus"):
- for field in ("query", "tools", "answers"):
- text = example[field].strip()
- if text:
- f.write(text + "\n")
-
- spm.SentencePieceTrainer.Train(
- input=corpus_path,
- model_prefix=TOKENIZER_PREFIX,
- vocab_size=vocab_size,
- model_type="bpe",
- pad_id=PAD_ID,
- eos_id=EOS_ID,
- bos_id=BOS_ID,
- unk_id=UNK_ID,
- user_defined_symbols=["<tool_call>", "<tools>"],
- byte_fallback=True,
- normalization_rule_name="identity",
- num_threads=min(128, max(1, (os.cpu_count() or 1) * 3 // 4)),
- train_extremely_large_corpus=False,
- minloglevel=2,
- )
- finally:
- if os.path.exists(corpus_path):
- os.remove(corpus_path)
- print(f"Tokenizer saved to {model_path}")
- return model_path
diff --git a/needle/model/__init__.py b/needle/model/__init__.py
index 8a306ad..e69de29 100644
--- a/needle/model/__init__.py
+++ b/needle/model/__init__.py
@@ -1,9 +0,0 @@
-from .architecture import * # noqa: F401,F403
-from .quantize import _quantize_params # noqa: F401
-from .export import export_submodel, slice_params # noqa: F401
-from .run import ( # noqa: F401
- generate, generate_batch, load_checkpoint,
- normalize_tools, restore_tool_names,
- encode_for_retrieval, retrieve_tools,
- _get_decode_fn,
-)
diff --git a/needle/model/architecture.py b/needle/model/architecture.py
index c62cc4d..c329843 100644
--- a/needle/model/architecture.py
+++ b/needle/model/architecture.py
@@ -1,11 +1,21 @@
import math
from dataclasses import dataclass
+import numpy as np
import jax
import jax.numpy as jnp
import jax.nn.initializers as jinit
import flax.linen as nn
+from . import quantize as _quantize
+from .quantize import fake_quant_act
+
+
+def _aq(x, quant):
+ if quant is False:
+ return x
+ return jax.lax.cond(quant, fake_quant_act, lambda t: t, x)
+
def default_init():
return jinit.normal(stddev=0.02)
@@ -17,9 +27,18 @@ def residual_init(num_layers):
DTYPE_MAP = {"float32": jnp.float32, "bfloat16": jnp.bfloat16, "float16": jnp.float16}
+PRESETS = {
+ "needle": dict(d_model=768, num_heads=12, num_kv_heads=6, num_layers=27,
+ engram_layers=(2, 15)),
+}
+PRESETS["base"] = dict(d_model=512, num_heads=8, num_kv_heads=4, num_layers=27,
+ engram_layers=(2, 15))
+PRESETS["nano"] = dict(d_model=256, num_heads=4, num_kv_heads=2, num_layers=20,
+ vocab_size=4096, engram_layers=(2, 15), engram_slots=4096,
+ batch_size=256)
+
class ZCRMSNorm(nn.Module):
- """Zero-centred RMSNorm: scale initialized to 0, applied as (1 + γ) * x / RMS(x)."""
epsilon: float = 1e-6
dtype: jnp.dtype = jnp.bfloat16
@@ -30,39 +49,43 @@ class ZCRMSNorm(nn.Module):
return ((1 + scale) * x / rms).astype(self.dtype)
+
@dataclass
class TransformerConfig:
vocab_size: int = 8192
- d_model: int = 128
- num_heads: int = 4
- num_kv_heads: int = 2
- num_encoder_layers: int = 2
- num_decoder_layers: int = 2
- d_ff: int = 512
- max_seq_len: int = 128
+ d_model: int = 512
+ attn_dim: int = 0
+ num_heads: int = 8
+ num_kv_heads: int = 4
+ num_layers: int = 12
+ max_seq_len: int = 2048
pad_token_id: int = 0
- rope_theta: float = 10000.0
- dtype: str = "bfloat16"
- activation: str = "drelu"
- num_memory_slots: int = 64
- dropout_rate: float = 0.1
contrastive_dim: int = 128
- no_feedforward: bool = True
+ rope_theta: float = 100000.0
+ dtype: str = "bfloat16"
+ flash: bool = True
+ engram_orders: tuple = (2, 3)
+ engram_heads: int = 0
+ engram_slots: int = 8192
+ engram_layers: tuple = (2, 15)
+ mhc_lanes: int = 4
+ kv_window: int = 0
+ kv_bits: int = 8
+ act_bits: int = 8
+ weight_bits: str = ""
def __init__(self, **kwargs):
valid = {f.name for f in self.__dataclass_fields__.values()}
for k, v in kwargs.items():
if k in valid:
setattr(self, k, v)
+ self.attn_dim = self.attn_dim or self.d_model
+ self.engram_layers = tuple(self.engram_layers)
@property
def jax_dtype(self):
return DTYPE_MAP[self.dtype]
- @property
- def total_layers(self):
- return self.num_encoder_layers + self.num_decoder_layers
-
def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):
freqs = 1.0 / (theta ** (jnp.arange(0, head_dim, 2).astype(jnp.float32) / head_dim))
@@ -78,7 +101,102 @@ def apply_rope(x, cos, sin):
sin = sin[:T][None, None, :, :]
x1 = x[..., :half]
x2 = x[..., half:]
- return jnp.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1)
+ return jnp.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1).astype(x.dtype)
+
+
+ENGRAM_SUB_DIM = 128
+ENGRAM_CONV_TAPS = 4
+_ENGRAM_SEED = 0x9E3779B9
+_ENGRAM_PRIME = 0x01000193
+
+
+def engram_geometry(config):
+ orders = tuple(config.engram_orders)
+ heads = config.engram_heads or max(1, config.d_model // (len(orders) * ENGRAM_SUB_DIM))
+ sub_dim = config.d_model // (len(orders) * heads)
+ return orders, heads, sub_dim
+
+
+def _shift_right(x, offset):
+ if offset == 0:
+ return x
+ pad = [(0, 0)] * x.ndim
+ pad[1] = (offset, 0)
+ return jnp.pad(x, pad)[:, : x.shape[1]]
+
+
+def _mask_diag(mask, offset):
+ m = mask[:, 0]
+ T = m.shape[-1]
+ if offset >= T:
+ return jnp.zeros(m.shape[:-2] + (T,), m.dtype)
+ d = jnp.diagonal(m, offset=-offset, axis1=-2, axis2=-1)
+ if offset == 0:
+ return d
+ return jnp.pad(d, ((0, 0), (offset, 0)))
+
+
+def engram_indices(tokens, orders, heads, slots):
+ u = tokens.astype(jnp.uint32)
+ idx = []
+ for oi, order in enumerate(orders):
+ for h in range(heads):
+ seed = (_ENGRAM_SEED * (oi * heads + h + 1)) & 0xFFFFFFFF
+ acc = jnp.full_like(u, jnp.uint32(seed))
+ for j in range(order):
+ acc = (acc ^ _shift_right(u, j)) * jnp.uint32(_ENGRAM_PRIME)
+ acc = acc ^ (acc >> jnp.uint32(15))
+ idx.append((acc % jnp.uint32(slots)).astype(jnp.int32))
+ return jnp.stack(idx, axis=-1)
+
+
+def _rms_unit(x, epsilon=1e-6):
+ xf = x.astype(jnp.float32)
+ return xf * jax.lax.rsqrt(jnp.mean(xf ** 2, axis=-1, keepdims=True) + epsilon)
+
+
+def _conv_identity_init(key, shape, dtype=jnp.float32):
+ return jnp.zeros(shape, dtype).at[0].set(1.0)
+
+
+def _sinkhorn(logits, iters=20):
+ log_K = logits
+ for _ in range(iters):
+ log_K = log_K - jax.nn.logsumexp(log_K, axis=-1, keepdims=True)
+ log_K = log_K - jax.nn.logsumexp(log_K, axis=-2, keepdims=True)
+ return jnp.exp(log_K)
+
+
+def _res_identity_init(key, shape, dtype=jnp.float32):
+ return jnp.broadcast_to(4.0 * jnp.eye(shape[-1], dtype=dtype), shape)
+
+
+class Engram(nn.Module):
+ d_model: int
+ num_tables: int
+ slots: int
+ sub_dim: int
+ num_layers: int
+ conv_dilation: int
+ dtype: jnp.dtype = jnp.bfloat16
+
+ @nn.compact
+ def __call__(self, indices, ngram_ok, tap_ok, quant=False):
+ tables = self.param("embedding", default_init(),
+ (self.num_tables, self.slots, self.sub_dim))
+ fetched = tables[jnp.arange(self.num_tables), indices]
+ fetched = fetched * ngram_ok[..., None]
+ e = fetched.reshape(*indices.shape[:2], self.num_tables * self.sub_dim)
+ e = _aq(e.astype(self.dtype), quant)
+ k = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,
+ kernel_init=default_init(), name="key_proj")(e)
+ v = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,
+ kernel_init=residual_init(self.num_layers), name="value_proj")(e)
+ taps = self.param("taps", _conv_identity_init,
+ (ENGRAM_CONV_TAPS, self.d_model)).astype(self.dtype)
+ v = sum(taps[j] * _shift_right(v, j * self.conv_dilation) * tap_ok[j][..., None]
+ for j in range(ENGRAM_CONV_TAPS))
+ return k, v
class MultiHeadAttention(nn.Module):
@@ -87,17 +205,20 @@ class MultiHeadAttention(nn.Module):
d_model: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
- rope_keys_only: bool = False
+ flash: bool = True
+ attn_dim: int = 0
@nn.compact
- def __call__(self, q_input, kv_input, mask=None, rope=None):
- head_dim = self.d_model // self.num_heads
+ def __call__(self, x, mask=None, rope=None, quant=False):
+ attn_dim = self.attn_dim or self.d_model
+ head_dim = attn_dim // self.num_heads
kv_dim = self.num_kv_heads * head_dim
- B = q_input.shape[0]
+ B = x.shape[0]
- q = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="q_proj")(q_input)
- k = nn.Dense(kv_dim, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="k_proj")(kv_input)
- v = nn.Dense(kv_dim, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="v_proj")(kv_input)
+ x = _aq(x, quant)
+ q = nn.Dense(attn_dim, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="q_proj")(x)
+ k = nn.Dense(kv_dim, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="k_proj")(x)
+ v = nn.Dense(kv_dim, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="v_proj")(x)
q = q.reshape(B, -1, self.num_heads, head_dim).transpose(0, 2, 1, 3)
k = k.reshape(B, -1, self.num_kv_heads, head_dim).transpose(0, 2, 1, 3)
@@ -106,350 +227,354 @@ class MultiHeadAttention(nn.Module):
q = ZCRMSNorm(dtype=self.dtype, name="q_norm")(q)
k = ZCRMSNorm(dtype=self.dtype, name="k_norm")(k)
- repeats = self.num_heads // self.num_kv_heads
- if repeats > 1:
- k = jnp.repeat(k, repeats, axis=1)
- v = jnp.repeat(v, repeats, axis=1)
-
if rope is not None:
cos, sin = rope
- if not self.rope_keys_only:
- q = apply_rope(q, cos, sin)
+ q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
- scale = jnp.sqrt(jnp.float32(head_dim))
- attn_weights = jnp.matmul(q, k.transpose(0, 1, 3, 2)) / scale
+ k = _quantize.maybe_quant_kv(k, quant)
+ v = _quantize.maybe_quant_kv(v, quant)
+
+ if self.flash:
+ impl = ("cudnn" if jax.default_backend() == "gpu"
+ and q.dtype in (jnp.bfloat16, jnp.float16) else None)
+ out = jax.nn.dot_product_attention(
+ q.transpose(0, 2, 1, 3),
+ k.transpose(0, 2, 1, 3),
+ v.transpose(0, 2, 1, 3),
+ mask=mask,
+ implementation=impl,
+ )
+ out = out.reshape(B, -1, attn_dim)
+ else:
+ repeats = self.num_heads // self.num_kv_heads
+ if repeats > 1:
+ k = jnp.repeat(k, repeats, axis=1)
+ v = jnp.repeat(v, repeats, axis=1)
+
+ scale = jnp.sqrt(jnp.float32(head_dim))
+ attn_weights = jnp.matmul(q, k.transpose(0, 1, 3, 2)) / scale
- if mask is not None:
- attn_weights = jnp.where(mask, attn_weights, jnp.finfo(attn_weights.dtype).min)
+ if mask is not None:
+ attn_weights = jnp.where(mask, attn_weights, jnp.finfo(attn_weights.dtype).min)
- attn_weights = nn.softmax(attn_weights, axis=-1)
+ attn_weights = nn.softmax(attn_weights, axis=-1)
- out = jnp.matmul(attn_weights, v)
- out = out.transpose(0, 2, 1, 3).reshape(B, -1, self.d_model)
+ out = jnp.matmul(attn_weights, v)
+ out = out.transpose(0, 2, 1, 3).reshape(B, -1, attn_dim)
+
+ out = out * nn.sigmoid(
+ nn.Dense(attn_dim, dtype=self.dtype, use_bias=False,
+ kernel_init=default_init(), name="gate_proj")(x))
+ out = _aq(out, quant)
return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="out_proj")(out)
-class FeedForward(nn.Module):
+def _walsh_matrix(n):
+ H = np.array([[1.0]], dtype=np.float32)
+ while H.shape[0] < n:
+ H = np.block([[H, H], [H, -H]])
+ return jnp.asarray(H / np.sqrt(n))
+
+
+class HadamardMLP(nn.Module):
d_model: int
- d_ff: int
- num_layers: int
dtype: jnp.dtype = jnp.bfloat16
- activation: str = "drelu"
@nn.compact
- def __call__(self, x, ffn_mask=None):
- gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)
- up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)
- if self.activation == "swiglu":
- h = nn.silu(gate) * up
- elif self.activation == "geglu":
- h = nn.gelu(gate) * up
- else: # drelu
- h = nn.relu(gate) * nn.relu(up)
- if ffn_mask is not None:
- h = h * ffn_mask[:, None, :]
- return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(h)
-
-
-class EncoderBlock(nn.Module):
- """Pre-norm self-attention + pre-norm FFN with gated residual connections."""
+ def __call__(self, x):
+ n = 1 << (self.d_model - 1).bit_length()
+ H = _walsh_matrix(n).astype(self.dtype)
+ d1 = self.param("d1", jinit.ones, (n,)).astype(self.dtype)
+ d2 = self.param("d2", jinit.ones, (n,)).astype(self.dtype)
+ d3 = self.param("d3", jinit.constant(0.02), (n,)).astype(self.dtype)
+ pad = n - self.d_model
+ z = jnp.pad(x, ((0, 0), (0, 0), (0, pad))) if pad else x
+ z = (d1 * z) @ H
+ z = nn.silu(d2 * z) @ H
+ return (d3 * z)[..., : self.d_model]
+
+
+class Block(nn.Module):
num_heads: int
num_kv_heads: int
d_model: int
- d_ff: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
- activation: str = "drelu"
- dropout_rate: float = 0.0
- no_feedforward: bool = True
+ flash: bool = True
+ attn_dim: int = 0
+
+ def _gate(self, name):
+ return nn.sigmoid(self.param(name, jinit.zeros, ())).astype(self.dtype)
@nn.compact
- def __call__(self, x, mask=None, rope=None, ffn_mask=None, deterministic=True):
- gate = nn.sigmoid(self.param("attn_gate", jinit.zeros, ())).astype(self.dtype)
- residual = x
+ def __call__(self, x, mask=None, rope=None, quant=False, engram_kv=None, site_flags=None):
+ if engram_kv is not None:
+ ek, ev = engram_kv
+ alpha = nn.sigmoid(jnp.einsum("btd,sbtd->sbt", _rms_unit(x), _rms_unit(ek))
+ / math.sqrt(self.d_model))
+ x = x + jnp.einsum("s,sbt,sbtd->btd", site_flags.astype(jnp.float32),
+ alpha, ev.astype(jnp.float32)).astype(x.dtype)
+
+ skip = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(
- x, x, mask=mask, rope=rope
- )
- x = residual + gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
-
- if not self.no_feedforward:
- ffn_gate = nn.sigmoid(self.param("ffn_gate", jinit.zeros, ())).astype(self.dtype)
- residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x, ffn_mask=ffn_mask)
- x = residual + ffn_gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers,
+ self.dtype, self.flash, attn_dim=self.attn_dim,
+ name="self_attn")(x, mask=mask, rope=rope, quant=quant)
+ x = ZCRMSNorm(dtype=self.dtype, name="post_attn_norm")(x)
+ x = skip + self._gate("attn_gate") * x
- return x
+ skip = x
+ x = ZCRMSNorm(dtype=self.dtype, name="pre_hada_norm")(x)
+ x = HadamardMLP(self.d_model, self.dtype, name="hadamard_mlp")(x)
+ return skip + x
-class _EncoderScanBody(nn.Module):
- """Wraps EncoderBlock for nn.scan: carry = (x, mask, rope, ffn_mask)."""
+class _ScanBody(nn.Module):
num_heads: int
num_kv_heads: int
d_model: int
- d_ff: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
- activation: str = "drelu"
- dropout_rate: float = 0.0
- no_feedforward: bool = True
- deterministic: bool = True
+ flash: bool = True
+ collect_hidden: bool = False
+ attn_dim: int = 0
@nn.compact
- def __call__(self, carry, _):
- x, mask, rope, ffn_mask = carry
- x = EncoderBlock(
- self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,
- self.num_layers, self.dtype, self.activation, self.dropout_rate,
- self.no_feedforward,
- )(x, mask, rope, ffn_mask, self.deterministic)
- return (x, mask, rope, ffn_mask), None
-
-
-class Encoder(nn.Module):
- """Self-attention encoder. Returns (output, mask) tuple."""
+ def __call__(self, x, xs, mask, rope, quant, engram_kv):
+ site_flags, hc = xs
+ block = Block(
+ self.num_heads, self.num_kv_heads, self.d_model, self.num_layers,
+ self.dtype, self.flash, attn_dim=self.attn_dim,
+ name="block",
+ )
+ B, T, n, C = x.shape
+ xf = x.astype(jnp.float32)
+ nx = _rms_unit(x.reshape(B, T, n * C))
+ hpre = nn.sigmoid(hc["a_pre"] * (nx @ hc["phi_pre"].astype(jnp.float32))
+ + hc["b_pre"] + hc["pre_off"])
+ u = jnp.einsum("btn,btnc->btc", hpre, xf).astype(self.dtype)
+ y = block(u, mask=mask, rope=rope, quant=quant,
+ engram_kv=engram_kv, site_flags=site_flags) - u
+ hpost = 2 * nn.sigmoid(hc["a_post"] * (nx @ hc["phi_post"].astype(jnp.float32))
+ + hc["b_post"] + hc["post_off"])
+ res = nx @ hc["phi_res"].astype(jnp.float32)
+ hres = _sinkhorn(hc["a_res"] * res.reshape(B, T, n, n) + hc["b_res"])
+ new_x = (jnp.einsum("btij,btjc->btic", hres, xf)
+ + hpost[..., None] * y.astype(jnp.float32)[:, :, None, :]).astype(self.dtype)
+ out = jnp.mean(new_x, axis=2) if self.collect_hidden else None
+ return new_x, out
+
+
+class Stack(nn.Module):
config: TransformerConfig
@nn.compact
- def __call__(self, x, mask=None, rope=None, ffn_mask=None, deterministic=True):
+ def __call__(self, x, mask=None, rope=None, engram_kv=None, collect_hidden=False,
+ quant=False):
cfg = self.config
dt = cfg.jax_dtype
x = x.astype(dt)
+ site_flags = None
+ if engram_kv is not None:
+ flags = np.zeros((cfg.num_layers, len(cfg.engram_layers)), np.float32)
+ for s, layer in enumerate(cfg.engram_layers):
+ flags[layer, s] = 1.0
+ site_flags = jnp.asarray(flags)
+
+ n, L, nC = cfg.mhc_lanes, cfg.num_layers, cfg.mhc_lanes * cfg.d_model
+ lane = np.eye(n, dtype=np.float32)[np.arange(L) % n]
+ hc = {
+ "phi_pre": self.param("mhc_phi_pre", default_init(), (L, nC, n)),
+ "phi_post": self.param("mhc_phi_post", default_init(), (L, nC, n)),
+ "phi_res": self.param("mhc_phi_res", default_init(), (L, nC, n * n)),
+ "b_pre": self.param("mhc_b_pre", jinit.zeros, (L, n)),
+ "b_post": self.param("mhc_b_post", jinit.zeros, (L, n)),
+ "b_res": self.param("mhc_b_res", _res_identity_init, (L, n, n)),
+ "a_pre": self.param("mhc_a_pre", jinit.constant(0.01), (L,)),
+ "a_post": self.param("mhc_a_post", jinit.constant(0.01), (L,)),
+ "a_res": self.param("mhc_a_res", jinit.constant(0.01), (L,)),
+ "pre_off": jnp.asarray(8 * lane - 4),
+ "post_off": jnp.asarray(-4 * (1 - lane)),
+ }
+ x = jnp.broadcast_to(x[:, :, None, :], (*x.shape[:2], n, x.shape[-1]))
+
ScanBlock = nn.scan(
- nn.remat(_EncoderScanBody),
+ nn.remat(_ScanBody),
variable_axes={"params": 0},
- split_rngs={"params": True, "dropout": True},
- length=cfg.num_encoder_layers,
+ split_rngs={"params": True},
+ length=cfg.num_layers,
+ in_axes=(0, nn.broadcast, nn.broadcast, nn.broadcast, nn.broadcast),
)
- (x, _, _, _), _ = ScanBlock(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
- cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,
- cfg.no_feedforward, deterministic, name="layers",
- )((x, mask, rope, ffn_mask), None)
+ x, hidden = ScanBlock(
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.num_layers, dt,
+ cfg.flash, collect_hidden, attn_dim=cfg.attn_dim,
+ name="layers",
+ )(x, (site_flags, hc), mask, rope, quant, engram_kv)
+ x = jnp.mean(x, axis=2)
x = ZCRMSNorm(dtype=dt, name="final_norm")(x)
- return x, mask
-
-
-class DecoderBlock(nn.Module):
- num_heads: int
- num_kv_heads: int
- d_model: int
- d_ff: int
- num_layers: int
- dtype: jnp.dtype = jnp.bfloat16
- activation: str = "drelu"
- dropout_rate: float = 0.0
- no_feedforward: bool = True
-
- @nn.compact
- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, ffn_mask=None, deterministic=True):
- self_gate = nn.sigmoid(self.param("self_attn_gate", jinit.zeros, ())).astype(self.dtype)
- residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(
- x, x, mask=self_mask, rope=rope
- )
- x = residual + self_gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
-
- cross_gate = nn.sigmoid(self.param("cross_attn_gate", jinit.zeros, ())).astype(self.dtype)
- residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="cross_attn")(
- x, encoder_out, mask=cross_mask
- )
- x = residual + cross_gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
-
- if not self.no_feedforward:
- ffn_gate = nn.sigmoid(self.param("ffn_gate", jinit.zeros, ())).astype(self.dtype)
- residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x, ffn_mask=ffn_mask)
- x = residual + ffn_gate * nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
+ return x, hidden
- return x
+def probe_pool(cells, probes, keep=None, dtype=jnp.bfloat16):
+ b, t, l, d = cells.shape
+ cells = cells.reshape(b, t * l, d)
+ if keep is not None:
+ keep = jnp.repeat(keep, l, axis=1)
+ scores = jnp.einsum("bcd,kd->bkc", cells.astype(jnp.float32),
+ probes.astype(jnp.float32)) / math.sqrt(d)
+ if keep is not None:
+ scores = jnp.where(keep[:, None, :] > 0, scores, -jnp.inf)
+ w = jax.nn.softmax(scores, axis=-1)
+ return jnp.einsum("bkc,bcd->bkd", w, cells.astype(jnp.float32)
+ ).reshape(b, -1).astype(dtype)
-class _DecoderScanBody(nn.Module):
- """Wraps DecoderBlock for nn.scan: carry = (x, encoder_out, self_mask, cross_mask, rope, ffn_mask)."""
- num_heads: int
- num_kv_heads: int
+class ContrastiveHead(nn.Module):
d_model: int
- d_ff: int
- num_layers: int
+ out_dim: int
dtype: jnp.dtype = jnp.bfloat16
- activation: str = "drelu"
- dropout_rate: float = 0.0
- no_feedforward: bool = True
- deterministic: bool = True
-
- @nn.compact
- def __call__(self, carry, _):
- x, encoder_out, self_mask, cross_mask, rope, ffn_mask = carry
- x = DecoderBlock(
- self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,
- self.num_layers, self.dtype, self.activation, self.dropout_rate,
- self.no_feedforward,
- )(x, encoder_out, self_mask, cross_mask, rope, ffn_mask, self.deterministic)
- return (x, encoder_out, self_mask, cross_mask, rope, ffn_mask), None
+ temp_init: float = 0.07
-
-class Decoder(nn.Module):
- config: TransformerConfig
+ PROBES = 4
@nn.compact
- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, ffn_mask=None, deterministic=True):
- cfg = self.config
- dt = cfg.jax_dtype
- x = x.astype(dt)
+ def __call__(self, cells, keep=None):
+ pooled = probe_pool(cells, self.param("probes", default_init(),
+ (self.PROBES, cells.shape[-1])),
+ keep, self.dtype)
+ p = nn.Dense(self.out_dim, dtype=self.dtype, use_bias=False,
+ kernel_init=default_init(), name="proj")(pooled)
+ log_temp = self.param("log_temp", jinit.constant(math.log(self.temp_init)), ())
+ denom = jnp.sqrt(jnp.sum(p.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + 1e-12)
+ return (p / denom.astype(p.dtype)), log_temp
+
+
+class ConfidenceHead(nn.Module):
+ dtype: jnp.dtype = jnp.bfloat16
- ScanBlock = nn.scan(
- nn.remat(_DecoderScanBody),
- variable_axes={"params": 0},
- split_rngs={"params": True, "dropout": True},
- length=cfg.num_decoder_layers,
- )
- (x, _, _, _, _, _), _ = ScanBlock(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
- cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,
- cfg.no_feedforward, deterministic, name="layers",
- )((x, encoder_out, self_mask, cross_mask, rope, ffn_mask), None)
+ PROBES = 8
- x = ZCRMSNorm(dtype=dt)(x)
- return x
+ @nn.compact
+ def __call__(self, cells, keep=None):
+ pooled = probe_pool(cells, self.param("probes", default_init(),
+ (self.PROBES, cells.shape[-1])),
+ keep, self.dtype)
+ logit = nn.Dense(1, dtype=self.dtype, use_bias=True,
+ kernel_init=default_init(), name="proj")(pooled)
+ return logit[..., 0].astype(jnp.float32)
class SimpleAttentionNetwork(nn.Module):
- """Encoder-decoder transformer with shared embeddings, tied output, RoPE, and bfloat16."""
config: TransformerConfig
def setup(self):
- self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
- self.embed_scale = math.sqrt(self.config.d_model)
- self.encoder = Encoder(self.config)
- self.decoder = Decoder(self.config)
- self.contrastive_hidden = nn.Dense(
- self.config.d_model // 4, dtype=self.config.jax_dtype,
- kernel_init=default_init(), name="contrastive_hidden",
- )
- self.contrastive_proj = nn.Dense(
- self.config.contrastive_dim, dtype=self.config.jax_dtype,
- use_bias=False, kernel_init=default_init(), name="contrastive_proj",
- )
- self.log_temp = self.param("log_temp", jinit.zeros, ())
+ cfg = self.config
+ self.embedding = nn.Embed(cfg.vocab_size, cfg.d_model, embedding_init=jinit.normal(stddev=0.02))
+ self.embed_scale = math.sqrt(cfg.d_model)
+ self.stack = Stack(cfg)
+ self.contrastive_head = ContrastiveHead(
+ cfg.d_model, cfg.contrastive_dim, cfg.jax_dtype)
+ self.confidence_head = ConfidenceHead(cfg.jax_dtype)
+ assert all(l < cfg.num_layers for l in cfg.engram_layers)
+ orders, heads, sub_dim = engram_geometry(cfg)
+ self.engrams = [
+ Engram(cfg.d_model, len(orders) * heads, cfg.engram_slots,
+ sub_dim, cfg.num_layers, max(orders), cfg.jax_dtype)
+ for _ in cfg.engram_layers
+ ]
+ self.mtp_combine = nn.Dense(cfg.d_model, dtype=cfg.jax_dtype, use_bias=False,
+ kernel_init=default_init())
+ self.mtp_block = Block(cfg.num_heads, cfg.num_kv_heads, cfg.d_model,
+ cfg.num_layers, cfg.jax_dtype, cfg.flash,
+ attn_dim=cfg.attn_dim)
+ self.mtp_emb_norm = ZCRMSNorm(dtype=cfg.jax_dtype)
+ self.mtp_final_norm = ZCRMSNorm(dtype=cfg.jax_dtype)
def _rope(self, seq_len):
- head_dim = self.config.d_model // self.config.num_heads
- return precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)
-
- def encode_text(self, src, src_mask=None, ffn_mask=None, deterministic=True):
- x = self.embedding(src) * self.embed_scale
- rope = self._rope(src.shape[1])
- return self.encoder(x, mask=src_mask, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)
-
- def encode(self, src, src_mask=None):
- """Backward-compatible alias for encode_text."""
- return self.encode_text(src, src_mask=src_mask)
-
- def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None, deterministic=True):
- """Decode from encoder output with cross_mask for variable-length encoder output."""
- x = self.embedding(tgt) * self.embed_scale
- rope = self._rope(tgt.shape[1])
- x = self.decoder(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)
- logits = x.astype(jnp.float32) @ self.embedding.embedding.T
- return logits
-
- def _mean_pool(self, encoder_out, enc_mask):
- """Mean-pool encoder output over non-padded positions. Returns (B, d_model)."""
- if enc_mask is not None:
- mask_2d = enc_mask[:, 0, 0, :]
- else:
- mask_2d = jnp.ones(encoder_out.shape[:2], dtype=encoder_out.dtype)
- mask_3d = mask_2d[:, :, None].astype(encoder_out.dtype)
- summed = jnp.sum(encoder_out * mask_3d, axis=1)
- counts = jnp.maximum(jnp.sum(mask_2d, axis=1, keepdims=True), 1.0)
- return summed / counts
-
- def encode_contrastive(self, tokens, deterministic=True):
- """Encode tokens and project to L2-normalized contrastive space. Returns (B, contrastive_dim)."""
- src_mask = make_padding_mask(tokens, self.config.pad_token_id)
- encoder_out, enc_mask = self.encode_text(tokens, src_mask=src_mask, deterministic=deterministic)
- pooled = self._mean_pool(encoder_out, enc_mask)
- h = nn.relu(self.contrastive_hidden(pooled))
- projected = self.contrastive_proj(h)
- # Safe L2 normalize: sqrt(sum² + eps²) has smooth gradient at origin,
- # whereas max(norm, eps) NaNs in the backward pass when projected == 0
- # (which happens after pretrain decays contrastive weights to ~0).
- denom = jnp.sqrt(jnp.sum(projected.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + 1e-12)
- return (projected / denom.astype(projected.dtype))
-
- def forward_contrastive(self, query_tokens, tool_tokens, deterministic=True):
- """Encode query and tool tokens, return (q_emb, t_emb, log_temp)."""
- q_emb = self.encode_contrastive(query_tokens, deterministic=deterministic)
- t_emb = self.encode_contrastive(tool_tokens, deterministic=deterministic)
- return q_emb, t_emb, self.log_temp
-
- def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
- encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
- cm = cross_mask if cross_mask is not None else enc_mask
- logits = self.decode(tgt, encoder_out, self_mask=tgt_mask, cross_mask=cm)
- return logits
-
- def _run_decoder(self, encoder_out, tgt, tgt_mask=None, cross_mask=None, ffn_mask=None, deterministic=True):
- """Run decoder and return float32 hidden states."""
- x = self.embedding(tgt) * self.embed_scale
- rope = self._rope(tgt.shape[1])
- x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)
- return x.astype(jnp.float32)
-
- def forward_masked(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, ffn_mask=None, deterministic=True):
- """Single forward with per-batch-item FFN masking. Returns (logits, slot_div)."""
- encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)
- cm = cross_mask if cross_mask is not None else enc_mask
- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=cm, ffn_mask=ffn_mask, deterministic=deterministic)
- logits = x_f32 @ self.embedding.embedding.T
- return logits, 0.0
-
- def _make_eval_ffn_mask(self, ff_width, B, dtype):
- """Default prefix FFN mask for eval: first ff_width neurons active."""
- mask = (jnp.arange(self.config.d_ff) < ff_width).astype(dtype)
- return jnp.broadcast_to(mask[None, :], (B, self.config.d_ff))
-
- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mat_ff_widths=None):
- """Eval-only: separate per-width forwards for reporting per-width PPL.
-
- mat_ff_widths: list of FFN widths to evaluate (e.g. [1024, 512, 256]).
- """
- emb = self.embedding.embedding
- B = src.shape[0]
-
- encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
- cm = cross_mask if cross_mask is not None else enc_mask
- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=cm)
- logits = x_f32 @ emb.T
-
- mat_logits = []
- if mat_ff_widths is not None:
- for ff_w in mat_ff_widths:
- mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)
- enc_m, enc_m_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)
- x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, cross_mask=cm, ffn_mask=mask)
- mat_logits.append(x_m @ emb.T)
-
- return logits, 0.0, mat_logits
-
- def init_all(self, src, tgt):
- """Dummy forward through text pathway to initialize all params."""
- src_mask = make_padding_mask(src, self.config.pad_token_id)
- tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)
- text_out, text_enc_mask = self.encode_text(src, src_mask=src_mask)
- _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask, cross_mask=text_enc_mask)
- _ = self.encode_contrastive(src)
- return jnp.zeros(())
+ cfg = self.config
+ head_dim = (cfg.attn_dim or cfg.d_model) // cfg.num_heads
+ return precompute_rope_freqs(head_dim, seq_len, cfg.rope_theta)
+
+ def _engram_kv(self, tokens, mask, quant):
+ if not self.engrams:
+ return None
+ orders, heads, _ = engram_geometry(self.config)
+ indices = engram_indices(tokens, orders, heads, self.config.engram_slots)
+ ngram_ok = jnp.stack([_mask_diag(mask, o - 1) for o in orders for _ in range(heads)],
+ axis=-1)
+ tap_ok = jnp.stack([_mask_diag(mask, j * max(orders)) for j in range(ENGRAM_CONV_TAPS)])
+ pairs = [e(indices, ngram_ok, tap_ok, quant=quant) for e in self.engrams]
+ return jnp.stack([k for k, _ in pairs]), jnp.stack([v for _, v in pairs])
+
+ def __call__(self, tokens, mask=None, quant=False, return_mtp=False):
+ if mask is None:
+ mask = make_causal_mask(tokens.shape[1])
+ x = self.embedding(tokens) * self.embed_scale
+ rope = self._rope(tokens.shape[1])
+ engram_kv = self._engram_kv(tokens, mask, quant)
+ x, _ = self.stack(x, mask=mask, rope=rope, engram_kv=engram_kv, quant=quant)
+ logits = _aq(x, quant).astype(jnp.float32) @ self.embedding.embedding.T
+ if not (return_mtp or self.is_initializing()):
+ return logits
+
+ nxt = jnp.pad(tokens[:, 1:], ((0, 0), (0, 1)))
+ e2 = self.mtp_emb_norm(self.embedding(nxt) * self.embed_scale)
+ m = self.mtp_combine(_aq(jnp.concatenate([x, e2], axis=-1), quant))
+ m = self.mtp_block(m, mask=mask, rope=rope, quant=quant)
+ m = self.mtp_final_norm(m)
+ mtp_logits = _aq(m, quant).astype(jnp.float32) @ self.embedding.embedding.T
+ if not return_mtp:
+ return logits
+ return logits, mtp_logits
+
+ def hidden_cells(self, tokens, quant=False, window=0, sink=None):
+ cfg = self.config
+ mask = (make_causal_mask(tokens.shape[1])
+ & make_padding_mask(tokens, cfg.pad_token_id))
+ if window:
+ pos = jnp.arange(tokens.shape[1])
+ recent = ((pos[:, None] - pos[None, :]) < window)[None, None, :, :]
+ keep = recent if sink is None else (recent | sink[:, None, None, :])
+ mask = mask & keep
+ x0 = self.embedding(tokens) * self.embed_scale
+ rope = self._rope(tokens.shape[1])
+ engram_kv = self._engram_kv(tokens, mask, quant)
+ _, hidden = self.stack(x0, mask=mask, rope=rope, engram_kv=engram_kv,
+ quant=quant, collect_hidden=True)
+ return jnp.stack([x0, *hidden], axis=2)
+
+ def _encode_contrastive(self, tokens, quant=False, window=0, sink=None):
+ cells = jax.lax.stop_gradient(
+ self.hidden_cells(tokens, quant=quant, window=window, sink=sink))
+ keep = (tokens != self.config.pad_token_id).astype(jnp.float32)
+ return self.contrastive_head(cells, keep=keep)
+
+ def encode_contrastive(self, tokens, quant=False, window=0, sink=None):
+ return self._encode_contrastive(tokens, quant=quant, window=window,
+ sink=sink)[0]
+
+ def forward_contrastive(self, query_tokens, tool_tokens, quant=False):
+ q_emb, log_temp = self._encode_contrastive(query_tokens, quant=quant)
+ t_emb, _ = self._encode_contrastive(tool_tokens, quant=quant)
+ return q_emb, t_emb, log_temp
+
+ def forward_confidence(self, tokens, quant=False, window=0, sink=None):
+ cells = jax.lax.stop_gradient(
+ self.hidden_cells(tokens, quant=quant, window=window, sink=sink))
+ keep = (tokens != self.config.pad_token_id).astype(jnp.float32)
+ return self.confidence_head(cells, keep=keep)
+
+ def hidden_states(self, tokens, mask=None):
+ if mask is None:
+ mask = make_causal_mask(tokens.shape[1])
+ x = self.embedding(tokens) * self.embed_scale
+ rope = self._rope(tokens.shape[1])
+ engram_kv = self._engram_kv(tokens, mask, False)
+ _, hidden = self.stack(x, mask=mask, rope=rope, engram_kv=engram_kv, collect_hidden=True)
+ return hidden.astype(jnp.float32)
def make_causal_mask(seq_len):
@@ -462,38 +587,36 @@ def make_padding_mask(tokens, pad_token_id):
return mask[:, None, None, :]
-def make_packing_mask(seg_ids):
- """Block-diagonal self-attention mask from segment IDs.
+KV_BUDGET_BYTES = 11 * 1024 * 1024 + 512 * 1024
+KV_GROUP = 32
+KV_WINDOW_MIN = 160
- seg_ids: (B, T) int array where 0 = padding, 1+ = segment number.
- Returns: (B, 1, T, T) bool mask — True where attention is allowed.
- Tokens attend to each other only if they share the same non-zero segment ID.
- """
- # (B, T, 1) == (B, 1, T) -> (B, T, T)
- mask = (seg_ids[:, :, None] == seg_ids[:, None, :]) & (seg_ids[:, :, None] > 0)
- return mask[:, None, :, :]
+def kv_budget_window(config):
+ head_dim = (getattr(config, "attn_dim", 0) or config.d_model) // config.num_heads
+ kv = config.num_kv_heads * head_dim
+ d, L = config.d_model, config.num_layers
+ sites = len(tuple(getattr(config, "engram_layers", (2, 15))))
+ per_pos = (L * (2 * kv + 2 * (kv // KV_GROUP) * 4)
+ + sites * (d + (d // KV_GROUP) * 4))
+ window = (KV_BUDGET_BYTES // per_pos) // KV_GROUP * KV_GROUP
+ return max(KV_WINDOW_MIN, min(window, config.max_seq_len))
-def make_causal_packing_mask(seg_ids):
- """Block-diagonal causal mask from segment IDs.
- Same as make_packing_mask but also enforces causal ordering within each segment.
- Returns: (B, 1, T, T) bool mask.
- """
+def effective_kv_window(config):
+ budget = kv_budget_window(config)
+ return min(budget, config.kv_window) if config.kv_window else budget
+
+
+def make_causal_packing_mask(seg_ids, prefix=None, window=0):
T = seg_ids.shape[1]
causal = jnp.tril(jnp.ones((T, T), dtype=jnp.bool_))
block = (seg_ids[:, :, None] == seg_ids[:, None, :]) & (seg_ids[:, :, None] > 0)
- return (block & causal[None, :, :])[:, None, :, :]
-
-
-def make_cross_packing_mask(enc_seg_ids, dec_seg_ids):
- """Cross-attention mask for packed sequences.
-
- enc_seg_ids: (B, T_enc) int segment IDs.
- dec_seg_ids: (B, T_dec) int segment IDs.
- Returns: (B, 1, T_dec, T_enc) bool mask — decoder token d attends to
- encoder token e iff they share the same non-zero segment ID.
- """
- # (B, T_dec, 1) == (B, 1, T_enc) -> (B, T_dec, T_enc)
- mask = (dec_seg_ids[:, :, None] == enc_seg_ids[:, None, :]) & (dec_seg_ids[:, :, None] > 0)
+ mask = block & causal[None, :, :]
+ if window:
+ pos = jnp.arange(T)
+ recent = (pos[:, None] - pos[None, :]) < window
+ sink = (jnp.zeros_like(seg_ids, dtype=jnp.bool_) if prefix is None
+ else prefix > 0)
+ mask = mask & (recent[None, :, :] | sink[:, None, :])
return mask[:, None, :, :]
diff --git a/needle/model/constrained.py b/needle/model/constrained.py
deleted file mode 100644
index b6afbd7..0000000
--- a/needle/model/constrained.py
+++ /dev/null
@@ -1,409 +0,0 @@
-"""Grammar-constrained decoding for tool-call generation.
-
-Constrains the decoder to only produce valid tool names and argument keys
-by tracking position in the output JSON and masking invalid tokens via a
-character-level trie built from the tool definitions.
-
-Needle output format (compact JSON, no spaces):
- [{"name":"tool_name","arguments":{"key1":value1,"key2":value2}}]
-
-Constrained regions:
- - Tool names after "name":" are constrained to known tool names
- - Argument keys after "arguments":{ are constrained to known param names
- - Argument values are unconstrained (strings, numbers, booleans, objects)
-"""
-
-import json
-import logging
-from enum import Enum, auto
-
-import numpy as np
-
-logger = logging.getLogger(__name__)
-
-
-class TrieNode:
- __slots__ = ("children", "is_terminal")
-
- def __init__(self):
- self.children: dict[str, "TrieNode"] = {}
- self.is_terminal: bool = False
-
-
-class Trie:
- """Character-level prefix tree for matching valid names/keys."""
-
- def __init__(self):
- self.root = TrieNode()
-
- def insert(self, word: str):
- node = self.root
- for ch in word:
- if ch not in node.children:
- node.children[ch] = TrieNode()
- node = node.children[ch]
- node.is_terminal = True
-
- def get_node(self, prefix: str) -> TrieNode | None:
- """Walk the trie for *prefix* and return the node, or None."""
- node = self.root
- for ch in prefix:
- if ch not in node.children:
- return None
- node = node.children[ch]
- return node
-
- @property
- def words(self) -> list[str]:
- """Return all words stored in the trie."""
- result = []
- def _dfs(node, path):
- if node.is_terminal:
- result.append("".join(path))
- for ch, child in sorted(node.children.items()):
- _dfs(child, path + [ch])
- _dfs(self.root, [])
- return result
-
-
-class ToolConstraints:
- """Holds one name trie and one param trie per function, built from tool JSON.
-
- Extracts property names from JSON Schema ``parameters.properties``,
- not from the schema-level keys (``type``, ``properties``, ``required``).
- """
-
- def __init__(self, tools_json: str):
- self.name_trie = Trie()
- self.param_tries: dict[str, Trie] = {}
-
- try:
- tools = json.loads(tools_json)
- except (json.JSONDecodeError, TypeError):
- tools = []
-
- if not isinstance(tools, list):
- tools = []
-
- for tool in tools:
- if not isinstance(tool, dict):
- continue
- name = tool.get("name", "")
- if not name:
- continue
- self.name_trie.insert(name)
-
- params = tool.get("parameters", {})
- if isinstance(params, dict):
- param_trie = Trie()
- for key, val in params.items():
- if isinstance(val, dict):
- param_trie.insert(key)
- self.param_tries[name] = param_trie
-
- def get_param_trie(self, function_name: str) -> Trie | None:
- return self.param_tries.get(function_name)
-
-
-class JsonState(Enum):
- FREE = auto()
- IN_NAME = auto()
- IN_ARG_KEY = auto()
-
-
-class JsonStateMachine:
- """Tracks position in needle's compact JSON output to constrain decoding.
-
- Needle format: ``[{"name":"TOOL","arguments":{"key":val,...}},...]``
-
- Constrained spans:
- - After ``"name":"`` → IN_NAME (constrains to valid tool names)
- - After ``"arguments":{"`` or ``,"`` at arguments depth → IN_ARG_KEY
- (constrains to valid parameter names for the current tool)
- - Closing ``"`` in constrained state → FREE
-
- Depth tracking ensures nested objects/arrays in argument VALUES
- do not trigger false IN_ARG_KEY transitions.
- """
-
- def __init__(self):
- self.state = JsonState.FREE
- self.buffer = ""
- self.constrained_buf = ""
- self.current_function = ""
- self.in_arguments = False
- self.arguments_depth = 0
- self.nesting_depth = 0
- self.in_string = False
- self.prev_char_escape = False
-
- def feed(self, text: str):
- """Feed generated text character-by-character to drive transitions."""
- for ch in text:
- self._feed_char(ch)
-
- def _feed_char(self, ch: str):
- if self.state in (JsonState.IN_NAME, JsonState.IN_ARG_KEY):
- if ch == '"':
- if self.state == JsonState.IN_NAME:
- self.current_function = self.constrained_buf
- self.constrained_buf = ""
- self.state = JsonState.FREE
- else:
- self.constrained_buf += ch
- self.buffer += ch
- return
-
- self.buffer += ch
-
- if self.in_string:
- if self.prev_char_escape:
- self.prev_char_escape = False
- return
- if ch == '\\':
- self.prev_char_escape = True
- return
- if ch == '"':
- self.in_string = False
- return
-
- if ch in '{[':
- self.nesting_depth += 1
- elif ch in '}]':
- self.nesting_depth = max(0, self.nesting_depth - 1)
- if ch == '}' and self.in_arguments and self.nesting_depth < self.arguments_depth:
- self.in_arguments = False
- return
-
- if self.buffer.endswith('"name":"') and not self.in_arguments:
- self.state = JsonState.IN_NAME
- self.constrained_buf = ""
- return
-
- if self.buffer.endswith('"arguments":{'):
- self.in_arguments = True
- self.arguments_depth = self.nesting_depth
- return
-
- if (self.in_arguments
- and self.nesting_depth == self.arguments_depth
- and self._at_arg_key_start()):
- self.state = JsonState.IN_ARG_KEY
- self.constrained_buf = ""
- return
-
- if ch == '"' and self._is_value_quote():
- self.in_string = True
-
- def _at_arg_key_start(self) -> bool:
- """True if buffer ends with ``{"`` or ``,"`` — an arg key is opening."""
- if len(self.buffer) < 2:
- return False
- return self.buffer[-2:] in ('{"', ',"')
-
- def _is_value_quote(self) -> bool:
- """True if the current ``"`` opens a JSON string value (preceded by ``:``)."""
- for j in range(len(self.buffer) - 2, -1, -1):
- c = self.buffer[j]
- if c in ' \t\n\r':
- continue
- return c == ':'
- return False
-
-
-def build_token_strings(tokenizer) -> list[str]:
- """Map each vocab ID to the exact characters it contributes to output.
-
- SentencePiece uses ``▁`` (U+2581) as a word-boundary marker that becomes
- a space in decoded text. We convert ``▁`` → ``' '`` so that both the
- state machine and the trie matcher see consistent text. This correctly
- blocks ``▁``-prefixed tokens inside constrained spans (tool names and
- argument keys never contain spaces in needle's compact JSON format).
- """
- vocab_size = tokenizer.vocab_size
- sp = tokenizer.sp
- strings = []
- for i in range(vocab_size):
- if sp.IsControl(i):
- strings.append("")
- elif sp.IsUnknown(i):
- strings.append("")
- elif sp.IsByte(i):
- piece = sp.IdToPiece(i)
- try:
- byte_val = int(piece[1:-1], 16) if piece.startswith("<0x") else ord(piece)
- strings.append(chr(byte_val))
- except (ValueError, IndexError):
- strings.append("")
- else:
- piece = sp.IdToPiece(i)
- strings.append(piece.replace("\u2581", " "))
- return strings
-
-
-class TokenIndex:
- """Maps first character -> list of token IDs for fast candidate filtering."""
-
- def __init__(self, token_strings: list[str]):
- self._index: dict[str, list[int]] = {}
- for tid, s in enumerate(token_strings):
- if not s:
- continue
- first = s[0]
- if first not in self._index:
- self._index[first] = []
- self._index[first].append(tid)
-
- def candidates_for(self, first_char: str) -> list[int]:
- return self._index.get(first_char, [])
-
- @property
- def all_nonempty(self) -> list[int]:
- """All token IDs with non-empty strings."""
- result = []
- for ids in self._index.values():
- result.extend(ids)
- return result
-
-
-def _check_token_valid(token_text: str, trie_node: TrieNode) -> bool:
- """Check if *token_text* is a valid continuation from *trie_node*.
-
- The token text may contain a closing ``"`` which signals end of the
- constrained span — at that point the trie node must be terminal.
- Characters after the closing ``"`` are not checked (they are structural
- JSON that the state machine handles separately).
- """
- node = trie_node
- for i, ch in enumerate(token_text):
- if ch == '"':
- return node.is_terminal
- if ch not in node.children:
- return False
- node = node.children[ch]
- return True
-
-
-def apply_constraints(
- logits: np.ndarray,
- state: JsonState,
- trie_node: TrieNode,
- token_strings: list[str],
- token_index: TokenIndex,
-) -> np.ndarray:
- """Mask logits so only valid tokens survive.
-
- Args:
- logits: shape (vocab_size,) float array
- state: current constrained state (IN_NAME or IN_ARG_KEY)
- trie_node: current position in the relevant trie
- token_strings: mapping from token ID to text
- token_index: first-char index for fast filtering
-
- Returns:
- Modified logits with invalid tokens set to -inf.
- """
- vocab_size = logits.shape[0]
- mask = np.full(vocab_size, False)
-
- valid_first_chars = set(trie_node.children.keys())
- if trie_node.is_terminal:
- valid_first_chars.add('"')
-
- for first_char in valid_first_chars:
- for tid in token_index.candidates_for(first_char):
- if not mask[tid]:
- text = token_strings[tid]
- if _check_token_valid(text, trie_node):
- mask[tid] = True
-
- if not mask.any():
- logger.warning("Constrained decoding: no valid tokens found, falling back to unconstrained")
- return logits
-
- masked_logits = logits.copy()
- masked_logits[~mask] = -np.inf
- return masked_logits
-
-class ConstrainedDecoder:
- """Batch-aware constrained decoder.
-
- Holds per-example ``JsonStateMachine`` instances and shared token
- metadata. Call ``constrain_logits`` between logit computation and
- argmax, then ``update`` after selecting the token.
-
- Both ``constrain_logits`` and ``update`` use the same ``token_strings``
- table (with ``▁`` → space) so the state machine and trie matcher see
- identical text. This correctly blocks ``▁``-prefixed tokens inside
- constrained spans, since spaces are never valid in tool names or
- argument keys.
- """
-
- def __init__(self, tool_constraints_list: list[ToolConstraints],
- token_strings: list[str], token_index: TokenIndex,
- tokenizer=None):
- self.batch_size = len(tool_constraints_list)
- self.tool_constraints = tool_constraints_list
- self.machines = [JsonStateMachine() for _ in range(self.batch_size)]
- self.token_strings = token_strings
- self.token_index = token_index
-
- def is_active(self, batch_idx: int) -> bool:
- """Return True if this batch element is currently in a constrained state."""
- return self.machines[batch_idx].state != JsonState.FREE
-
- def constrain_logits(self, logits: np.ndarray, batch_idx: int) -> np.ndarray:
- """Apply grammar constraints to logits for a single batch element."""
- machine = self.machines[batch_idx]
- tc = self.tool_constraints[batch_idx]
-
- if machine.state == JsonState.FREE:
- return logits
-
- if machine.state == JsonState.IN_NAME:
- trie = tc.name_trie
- elif machine.state == JsonState.IN_ARG_KEY:
- trie = tc.get_param_trie(machine.current_function)
- if trie is None:
- return logits
- else:
- return logits
-
- node = trie.get_node(machine.constrained_buf)
- if node is None:
- logger.warning("Constrained decoding: off-trie at %r, falling back", machine.constrained_buf)
- return logits
-
- return apply_constraints(logits, machine.state, node, self.token_strings, self.token_index)
-
- def update(self, batch_idx: int, token_id: int):
- """Advance the state machine for *batch_idx* with the selected token."""
- text = self.token_strings[token_id]
- self.machines[batch_idx].feed(text)
-
-
-_token_cache: dict[int, tuple[list[str], TokenIndex]] = {}
-
-
-def _get_token_data(tokenizer) -> tuple[list[str], TokenIndex]:
- """Return (token_strings, token_index), cached per tokenizer instance."""
- key = id(tokenizer)
- if key not in _token_cache:
- ts = build_token_strings(tokenizer)
- _token_cache[key] = (ts, TokenIndex(ts))
- return _token_cache[key]
-
-
-def build_constrained_decoder(
- tools_json_list: list[str],
- tokenizer,
-) -> ConstrainedDecoder:
- """Convenience factory: build a ConstrainedDecoder for a batch of examples.
-
- Args:
- tools_json_list: list of tool JSON strings (one per batch element)
- tokenizer: NeedleTokenizer with .sp SentencePiece model
- """
- token_strings, token_index = _get_token_data(tokenizer)
- tc_list = [ToolConstraints(tj) for tj in tools_json_list]
- return ConstrainedDecoder(tc_list, token_strings, token_index)
diff --git a/needle/model/decode.py b/needle/model/decode.py
new file mode 100644
index 0000000..3dbacd2
--- /dev/null
+++ b/needle/model/decode.py
@@ -0,0 +1,405 @@
+import math
+from collections import namedtuple
+
+import jax
+import jax.numpy as jnp
+
+from .architecture import (precompute_rope_freqs, apply_rope, _walsh_matrix,
+ engram_geometry, engram_indices, _rms_unit, _shift_right,
+ _sinkhorn, ENGRAM_CONV_TAPS)
+
+
+def _doc_prefix_len(ids):
+ return 0
+
+DecodeCfg = namedtuple("DecodeCfg", [
+ "d_model", "num_heads", "num_kv_heads", "num_layers", "attn_dim", "mhc_lanes",
+ "engram_layers", "engram_orders", "engram_heads", "engram_slots",
+ "kv_window",
+])
+
+
+def _attn_width(config):
+ return getattr(config, "attn_dim", 0) or config.d_model
+
+
+def decode_cfg(config, kv_window=0):
+ orders, heads, _ = engram_geometry(config)
+ return DecodeCfg(config.d_model, config.num_heads, config.num_kv_heads,
+ config.num_layers, _attn_width(config), config.mhc_lanes,
+ tuple(config.engram_layers), tuple(orders), heads,
+ config.engram_slots, int(kv_window))
+
+
+def init_kv_cache(config, batch, max_len):
+ head_dim = _attn_width(config) // config.num_heads
+ shape = (config.num_layers, batch, config.num_kv_heads, max_len, head_dim)
+ z = jnp.zeros(shape, jnp.float32)
+ return z, z
+
+
+def _zcrms(x, scale, eps=1e-6):
+ rms = jnp.sqrt(jnp.mean(x ** 2, axis=-1, keepdims=True) + eps)
+ return (1.0 + scale) * x / rms
+
+
+def _layer(params, i):
+ b = params["stack"]["layers"]["block"]
+ sa = b["self_attn"]
+ ha = b["hadamard_mlp"]
+ lp = {
+ "pre": b["ZCRMSNorm_0"]["scale"][i],
+ "q": sa["q_proj"]["kernel"][i], "k": sa["k_proj"]["kernel"][i],
+ "v": sa["v_proj"]["kernel"][i], "o": sa["out_proj"]["kernel"][i],
+ "attn_gate_proj": sa["gate_proj"]["kernel"][i],
+ "qn": sa["q_norm"]["scale"][i], "kn": sa["k_norm"]["scale"][i],
+ "post": b["post_attn_norm"]["scale"][i], "attn_gate": b["attn_gate"][i],
+ "pre_hada": b["pre_hada_norm"]["scale"][i],
+ "hd1": ha["d1"][i], "hd2": ha["d2"][i], "hd3": ha["d3"][i],
+ }
+ return lp
+
+
+def _attn_cached(x, lp, k_cache_l, v_cache_l, pos, cos_s, sin_s, cfg, key_valid,
+ flash=False, sink=None):
+ B, S, _ = x.shape
+ H, KV = cfg.num_heads, cfg.num_kv_heads
+ hd = cfg.attn_dim // H
+
+ q = (x @ lp["q"]).reshape(B, S, H, hd).transpose(0, 2, 1, 3)
+ k = (x @ lp["k"]).reshape(B, S, KV, hd).transpose(0, 2, 1, 3)
+ v = (x @ lp["v"]).reshape(B, S, KV, hd).transpose(0, 2, 1, 3)
+ q = _zcrms(q, lp["qn"]); k = _zcrms(k, lp["kn"])
+ q = apply_rope(q, cos_s, sin_s); k = apply_rope(k, cos_s, sin_s)
+
+ k_cache_l = jax.lax.dynamic_update_slice(k_cache_l, k, (0, 0, pos, 0))
+ v_cache_l = jax.lax.dynamic_update_slice(v_cache_l, v, (0, 0, pos, 0))
+ max_len = k_cache_l.shape[2]
+
+ if flash and S > 1:
+ causal = jnp.tril(jnp.ones((S, S), bool))
+ if cfg.kv_window:
+ rows = jnp.arange(S)
+ causal = causal & ((rows[:, None] - rows[None, :]) < cfg.kv_window)
+ if sink is not None:
+ causal = causal | (jnp.tril(jnp.ones((S, S), bool))
+ & sink[:, :S][:, None, :])
+ if key_valid is None:
+ m = causal[None, None, :, :] if causal.ndim == 2 else causal[:, None, :, :]
+ else:
+ c = causal[None] if causal.ndim == 2 else causal
+ m = (c & key_valid[:, :S][:, None, :]) | jnp.eye(S, dtype=bool)[None]
+ m = m[:, None, :, :]
+ impl = ("cudnn" if jax.default_backend() == "gpu"
+ and q.dtype in (jnp.bfloat16, jnp.float16) else None)
+ out = jax.nn.dot_product_attention(
+ q.transpose(0, 2, 1, 3), k.transpose(0, 2, 1, 3), v.transpose(0, 2, 1, 3),
+ mask=m, implementation=impl)
+ out = out.reshape(B, S, cfg.attn_dim)
+ else:
+ reps = H // KV
+ scale = math.sqrt(hd)
+ qg = q.reshape(B, KV, reps, S, hd)
+ aw = jnp.einsum("bgrsd,bgld->bgrsl", qg, k_cache_l) / scale
+ qpos = pos + jnp.arange(S)
+ kpos = jnp.arange(max_len)
+ causal = kpos[None, :] <= qpos[:, None]
+ if cfg.kv_window:
+ recent = (qpos[:, None] - kpos[None, :]) < cfg.kv_window
+ keep = recent[None] if sink is None else (recent[None] | sink[:, None, :])
+ causal = causal[None] & keep
+ else:
+ causal = causal[None]
+ if key_valid is None:
+ mask = causal[:, None, None, :, :]
+ else:
+ mask = (causal & key_valid[:, None, :])[:, None, None, :, :]
+ aw = jnp.where(mask, aw, jnp.finfo(aw.dtype).min)
+ aw = jax.nn.softmax(aw, axis=-1)
+ out = jnp.einsum("bgrsl,bgld->bgrsd", aw, v_cache_l)
+ out = out.reshape(B, H, S, hd).transpose(0, 2, 1, 3).reshape(B, S, cfg.attn_dim)
+
+ out = out * jax.nn.sigmoid(x @ lp["attn_gate_proj"])
+ return out @ lp["o"], k_cache_l, v_cache_l
+
+
+def _hadamard(z, d1, d2, d3, d_model):
+ n = 1 << (d_model - 1).bit_length()
+ H = _walsh_matrix(n).astype(z.dtype)
+ pad = n - d_model
+ if pad:
+ z = jnp.pad(z, ((0, 0), (0, 0), (0, pad)))
+ z = (d1 * z) @ H
+ z = jax.nn.silu(d2 * z) @ H
+ return (d3 * z)[..., :d_model]
+
+
+def _block_cached(x, lp, k_cache_l, v_cache_l, pos, cos_s, sin_s, cfg, key_valid,
+ flash=False, sink=None):
+ skip = x
+ h = _zcrms(x, lp["pre"])
+ attn, k_cache_l, v_cache_l = _attn_cached(h, lp, k_cache_l, v_cache_l, pos,
+ cos_s, sin_s, cfg, key_valid, flash, sink)
+ attn = _zcrms(attn, lp["post"])
+ x = skip + jax.nn.sigmoid(lp["attn_gate"]) * attn
+
+ skip = x
+ h = _zcrms(x, lp["pre_hada"])
+ x = skip + _hadamard(h, lp["hd1"], lp["hd2"], lp["hd3"], cfg.d_model)
+ return x, k_cache_l, v_cache_l
+
+
+def _engram_window(cfg):
+ return ENGRAM_CONV_TAPS * max(cfg.engram_orders) if cfg.engram_layers else 0
+
+
+def _engram_kv(params, cfg, hist, hist_valid, pos, S):
+ W = _engram_window(cfg)
+ B = hist.shape[0]
+ padded = jnp.pad(hist, ((0, 0), (W, 0)))
+ win = jax.lax.dynamic_slice(padded, (0, pos), (B, W + S))
+ pv = jnp.pad(hist_valid, ((0, 0), (W, 0)))
+ win_valid = jax.lax.dynamic_slice(pv, (0, pos), (B, W + S)).astype(jnp.float32)
+
+ orders, heads = cfg.engram_orders, cfg.engram_heads
+ idx = engram_indices(win, orders, heads, cfg.engram_slots)
+ ngram_ok = jnp.stack([_shift_right(win_valid, o - 1)
+ for o in orders for _ in range(heads)], axis=-1)
+ tap_ok = jnp.stack([_shift_right(win_valid, j * max(orders))
+ for j in range(ENGRAM_CONV_TAPS)])
+
+ ks, vs = [], []
+ for site in range(len(cfg.engram_layers)):
+ ep = params[f"engrams_{site}"]
+ tables = ep["embedding"]
+ fetched = tables[jnp.arange(tables.shape[0]), idx] * ngram_ok[..., None]
+ e = fetched.reshape(B, W + S, -1)
+ k = e @ ep["key_proj"]["kernel"]
+ v = e @ ep["value_proj"]["kernel"]
+ taps = ep["taps"]
+ v = sum(taps[j] * _shift_right(v, j * max(orders)) * tap_ok[j][..., None]
+ for j in range(ENGRAM_CONV_TAPS))
+ ks.append(k[:, W:]); vs.append(v[:, W:])
+ return jnp.stack(ks), jnp.stack(vs)
+
+
+def _mhc(params, cfg):
+ st = params["stack"]
+ n, L = cfg.mhc_lanes, cfg.num_layers
+ lane = jnp.eye(n, dtype=jnp.float32)[jnp.arange(L) % n]
+ return {"phi_pre": st["mhc_phi_pre"], "phi_post": st["mhc_phi_post"],
+ "phi_res": st["mhc_phi_res"], "b_pre": st["mhc_b_pre"],
+ "b_post": st["mhc_b_post"], "b_res": st["mhc_b_res"],
+ "a_pre": st["mhc_a_pre"], "a_post": st["mhc_a_post"],
+ "a_res": st["mhc_a_res"], "pre_off": 8 * lane - 4,
+ "post_off": -4 * (1 - lane)}
+
+
+def _forward_cached(params, cfg, tokens, k_cache, v_cache, pos, cos, sin, key_valid=None,
+ flash=False, hist=None, hist_valid=None, sink=None):
+ emb = params["embedding"]["embedding"].astype(jnp.float32)
+ x = emb[tokens] * math.sqrt(cfg.d_model)
+ B, S = tokens.shape
+ cos_s = jax.lax.dynamic_slice_in_dim(cos, pos, S, axis=0)
+ sin_s = jax.lax.dynamic_slice_in_dim(sin, pos, S, axis=0)
+
+ ekv = None
+ if cfg.engram_layers:
+ h = tokens if hist is None else hist
+ hv = jnp.ones(h.shape, bool) if hist_valid is None else hist_valid
+ ekv = _engram_kv(params, cfg, h, hv,
+ jnp.asarray(0, jnp.int32) if hist is None else pos, S)
+
+ n, C = cfg.mhc_lanes, cfg.d_model
+ hc = _mhc(params, cfg)
+ x = jnp.broadcast_to(x[:, :, None, :], (B, S, n, C))
+
+ new_k, new_v = [], []
+ for i in range(cfg.num_layers):
+ nx = _rms_unit(x.reshape(B, S, n * C))
+ hpre = jax.nn.sigmoid(hc["a_pre"][i] * (nx @ hc["phi_pre"][i])
+ + hc["b_pre"][i] + hc["pre_off"][i])
+ u = jnp.einsum("btn,btnc->btc", hpre, x)
+ bx = u
+ if ekv is not None and i in cfg.engram_layers:
+ site = cfg.engram_layers.index(i)
+ ek, ev = ekv
+ alpha = jax.nn.sigmoid(
+ jnp.einsum("btd,btd->bt", _rms_unit(u), _rms_unit(ek[site]))
+ / math.sqrt(C))
+ bx = u + alpha[..., None] * ev[site]
+ y, kc, vc = _block_cached(bx, _layer(params, i), k_cache[i], v_cache[i],
+ pos, cos_s, sin_s, cfg, key_valid, flash, sink)
+ y = y - u
+ hpost = 2 * jax.nn.sigmoid(hc["a_post"][i] * (nx @ hc["phi_post"][i])
+ + hc["b_post"][i] + hc["post_off"][i])
+ res = nx @ hc["phi_res"][i]
+ hres = _sinkhorn(hc["a_res"][i] * res.reshape(B, S, n, n) + hc["b_res"][i])
+ x = (jnp.einsum("btij,btjc->btic", hres, x)
+ + hpost[..., None] * y[:, :, None, :])
+ new_k.append(kc); new_v.append(vc)
+ x = jnp.mean(x, axis=2)
+ x = _zcrms(x, params["stack"]["final_norm"]["scale"])
+ logits = x @ emb.T
+ return logits, jnp.stack(new_k), jnp.stack(new_v)
+
+
+forward_cached = jax.jit(_forward_cached, static_argnums=(1, 9))
+
+
+def _f32(params):
+ return jax.tree.map(lambda a: jnp.asarray(a, jnp.float32), params)
+
+
+def generate_cached(config, params, tokenizer, prompt, max_new_tokens=96, max_len=None,
+ kv_window=0):
+ from .tokenizer import BOS_ID, EOS_ID
+ params = _f32(params)
+ prompt_ids = ([BOS_ID] + tokenizer.encode(prompt))[:config.max_seq_len - 1]
+ max_len = max_len or min(config.max_seq_len, len(prompt_ids) + max_new_tokens)
+ head_dim = _attn_width(config) // config.num_heads
+ cos, sin = precompute_rope_freqs(head_dim, max_len, config.rope_theta)
+
+ dcfg = decode_cfg(config, kv_window=kv_window)
+ kc, vc = init_kv_cache(config, 1, max_len)
+ hist = jnp.zeros((1, max_len), jnp.int32).at[0, :len(prompt_ids)].set(
+ jnp.asarray(prompt_ids, jnp.int32))
+ valid = jnp.ones((1, max_len), bool)
+ sink = None
+ if kv_window:
+ import numpy as _np
+ sk = _np.zeros((1, max_len), bool)
+ sk[0, :_doc_prefix_len(prompt_ids)] = True
+ sink = jnp.asarray(sk)
+ toks = jnp.asarray([prompt_ids], jnp.int32)
+ logits, kc, vc = forward_cached(params, dcfg, toks, kc, vc,
+ jnp.asarray(0, jnp.int32), cos, sin, None, False,
+ hist, valid, sink)
+ nxt = int(jnp.argmax(logits[0, -1]))
+
+ out, pos = [], len(prompt_ids)
+ for _ in range(max_new_tokens):
+ if nxt == EOS_ID or pos >= max_len:
+ break
+ out.append(nxt)
+ hist = hist.at[0, pos].set(nxt)
+ logits, kc, vc = forward_cached(params, dcfg, jnp.asarray([[nxt]], jnp.int32),
+ kc, vc, jnp.asarray(pos, jnp.int32), cos, sin,
+ None, False, hist, valid, sink)
+ nxt = int(jnp.argmax(logits[0, -1]))
+ pos += 1
+ return tokenizer.decode(out)
+
+
+def _shard(x, nd):
+ return x.reshape(nd, x.shape[0] // nd, *x.shape[1:])
+
+
+def _sample(last, rng, temperature):
+ logp = jax.nn.log_softmax(last, axis=-1)
+ nxt = jnp.where(temperature > 0,
+ jax.random.categorical(rng, last / jnp.maximum(temperature, 1e-6), axis=-1),
+ jnp.argmax(last, axis=-1)).astype(jnp.int32)
+ tok_logp = jnp.take_along_axis(logp, nxt[:, None], axis=-1)[:, 0]
+ return nxt, tok_logp
+
+
+def _make_rollout(cfg, max_len, scan_len):
+ hd = cfg.attn_dim // cfg.num_heads
+
+ def rollout(params, buf, valid, sink, cos, sin, rng, temperature):
+ B, max_p = buf.shape
+ z = jnp.zeros((cfg.num_layers, B, cfg.num_kv_heads, max_len, hd), jnp.float32)
+ hist = jnp.zeros((B, max_len), jnp.int32).at[:, :max_p].set(buf)
+ logits, kc, vc = _forward_cached(params, cfg, buf, z, z,
+ jnp.asarray(0, jnp.int32), cos, sin, valid,
+ flash=True, hist=hist, hist_valid=valid,
+ sink=sink)
+ rng, sk = jax.random.split(rng)
+ t0, lp0 = _sample(logits[:, -1], sk, temperature)
+
+ def body(carry, _):
+ kc, vc, last, pos, rng, hist = carry
+ hist = jax.lax.dynamic_update_slice(hist, last[:, None], (0, pos))
+ logits, kc, vc = _forward_cached(params, cfg, last[:, None], kc, vc,
+ pos, cos, sin, valid,
+ hist=hist, hist_valid=valid, sink=sink)
+ rng, sk = jax.random.split(rng)
+ tok, lp = _sample(logits[:, -1], sk, temperature)
+ return (kc, vc, tok, pos + 1, rng, hist), (tok, lp)
+
+ init = (kc, vc, t0, jnp.asarray(max_p, jnp.int32), rng, hist)
+ _, (toks, lps) = jax.lax.scan(body, init, None, length=scan_len)
+ toks = jnp.concatenate([t0[None], toks], axis=0)
+ lps = jnp.concatenate([lp0[None], lps], axis=0)
+ return toks.T, lps.T
+ return rollout
+
+
+_rollout_cache = {}
+
+
+def _rollout_fn(cfg, max_len, scan_len):
+ key = (cfg, max_len, scan_len)
+ if key not in _rollout_cache:
+ _rollout_cache[key] = jax.pmap(_make_rollout(cfg, max_len, scan_len),
+ in_axes=(0, 0, 0, 0, None, None, 0, None))
+ return _rollout_cache[key]
+
+
+def batched_generate(config, params, tokenizer, prompts, max_new_tokens=96,
+ temperature=0.0, seed=0, pad_to=None, kv_window=0):
+ from .tokenizer import BOS_ID, EOS_ID, PAD_ID
+ import numpy as _np
+
+ params = _f32(params)
+ dcfg = decode_cfg(config, kv_window=kv_window)
+ cap = max(1, config.max_seq_len - max_new_tokens)
+ enc = [([BOS_ID] + tokenizer.encode(p))[:cap] for p in prompts]
+ n_real = len(enc)
+
+ nd = jax.local_device_count()
+ B = ((n_real + nd - 1) // nd) * nd
+ while len(enc) < B:
+ enc.append([BOS_ID])
+ max_p = min(cap, pad_to) if pad_to else max(len(e) for e in enc)
+ max_len = min(config.max_seq_len, max_p + max_new_tokens)
+ scan_len = max(max_len - max_p - 1, 0)
+ if scan_len == 0:
+ raise ValueError(
+ f"longest prompt is {max_p} tokens, leaving no room for "
+ f"{max_new_tokens} new ones inside max_seq_len {config.max_seq_len} - "
+ f"the whole batch would generate a single token. Drop prompts longer "
+ f"than max_seq_len - max_new_tokens before calling.")
+
+ buf = _np.full((B, max_p), PAD_ID, _np.int32)
+ valid = _np.zeros((B, max_len), bool)
+ sink = _np.zeros((B, max_len), bool)
+ for b, e in enumerate(enc):
+ start = max_p - len(e)
+ buf[b, start:] = e
+ valid[b, start:] = True
+ if kv_window:
+ sink[b, start:start + _doc_prefix_len(e)] = True
+ valid[:, max_p:] = True
+
+ head_dim = _attn_width(config) // config.num_heads
+ cos, sin = precompute_rope_freqs(head_dim, max_len, config.rope_theta)
+
+ pr = jax.tree.map(lambda x: jnp.broadcast_to(x, (nd,) + x.shape), params)
+ rollout = _rollout_fn(dcfg, max_len, scan_len)
+ rngs = jax.random.split(jax.random.PRNGKey(seed), nd)
+ toks, lps = rollout(pr, _shard(jnp.asarray(buf), nd), _shard(jnp.asarray(valid), nd),
+ _shard(jnp.asarray(sink), nd), cos, sin, rngs,
+ jnp.asarray(temperature, jnp.float32))
+ toks = _np.asarray(toks).reshape(B, -1)
+ lps = _np.asarray(lps).reshape(B, -1)
+
+ keep = _np.cumprod(toks != EOS_ID, axis=1).astype(bool)
+ texts, out_ids, out_lps = [], [], []
+ for b in range(n_real):
+ ids = toks[b][keep[b]].tolist()
+ out_ids.append(ids)
+ out_lps.append(lps[b][keep[b]].tolist())
+ texts.append(tokenizer.decode(ids))
+ return texts, out_ids, out_lps
diff --git a/needle/model/export.py b/needle/model/export.py
index 4fc7185..1ca6c8b 100644
--- a/needle/model/export.py
+++ b/needle/model/export.py
@@ -1,154 +1,526 @@
-"""Export matryoshka sub-models by slicing FFN weights by a shrink factor.
+"""Export a trained checkpoint to a `.cact` blob (needle.cact) - the deployment
+weights the C++ / single-mega-kernel inference reads (mmap or linked read-only
+section; no parsing).
-With FFN interior matryoshka, d_model stays constant — only FFN intermediate
-dimensions (gate_proj, up_proj, down_proj) are sliced.
-"""
+Format is W4A8: the matmul weights (tied embedding + every attention projection)
+are Cactus-Quants INT4; norms, Hadamard diagonals, and gates stay FP16. Weights
+are PRE-TRANSPOSED to [out, in] so each output row is contiguous along the
+reduction axis (what a GEMV/GEMM stream wants, and the axis quant groups run
+along). Tensors are LAYER-MAJOR: embedding, then all of layer 0's tensors, ...,
+then final norm - so the kernel's working set for one layer is contiguous.
+
+================================ byte layout ================================
+All little-endian. A tiny header, then a NAMELESS tensor directory, then 64-byte
+aligned tensor blobs. The architecture geometry (dims, layer/engram/mHC layout)
+is NOT stored - the runtime bakes it in (canon_config in needle_model.cpp) - so
+neither the blob nor the shipped binary reveals the model structure; tensors are
+positional in the fixed canon order below.
+
+HEADER (u32 tag, u32 num_tensors, u32 codebook_len, u32 kv_window, u32
+kv_bits; then codebook_len * f32 - the shared Lloyd-Max unit-sphere codebooks,
+cb2[4] | cb3[8] | cb4[16] concatenated, so mixed-precision blobs need one
+header). kv_window is the sliding-window width the model was trained with (0 =
+size it from the KV budget alone); kv_bits is the KV-cache width it was
+post-trained for (8 = int8, else 2/3/4). Both ride in the blob because a model
+quantized for one must be RUN at it - leaving either to a runtime flag silently
+serves the wrong numerics.
+
+DIRECTORY (num_tensors records, {REC_SIZE} bytes each, no names):
+ u8 dtype, u8 ndim, u16 _pad, u32 shape[4], u64 offset, u64 nbytes,
+ u32 group_size, u32 bits. dtype: 1=FP16, 2=FP32, 3=CQ (bits gives the
+ width: 2, 3 or 4), 4=RAW.
+
+TENSOR ORDER (what the runtime indexes by position):
+ embedding; per layer [norm_in, q_proj, k_proj, v_proj, q_norm, k_norm,
+ gate_proj, out_proj, post_norm, attn_gate, pre_hada, d1, d2, d3]; the 9 mHC
+ blocks; per engram site [tables, key_proj, value_proj, taps]; final_norm;
+ the optional probe heads (see below); then the RAW tokenizer.
+PROBE HEADS (all FP16, appended so pre-head blobs stay loadable). Let
+ extra = num_tensors - (through final_norm) - (tokenizer). extra == 0 means no
+ heads. Otherwise the tensor right after final_norm is a `heads.manifest` of
+ shape (H,), one code per head in canonical order (1 contrastive,
+ 2 confidence), followed by H fixed-stride triples [probes (P, d_model),
+ proj (out, P*d_model), bias (out,)], so H = (extra - 1) / 3.
+
+CQ blob for a logical [out, in] matrix (in padded up to a multiple of group):
+first the packed indices (out * in_pad*bits/8 bytes), then the per-group L2
+norms (out * in_pad/group FP16). Indices are packed LSB-first in chunks of 8:
+each run of 8 consecutive indices is OR'd into a word at offsets i*bits and
+emitted as `bits` little-endian bytes. A chunk is a whole number of bytes, so
+this is equivalently one continuous LSB-first bitstream per row (index k
+occupies bits [k*bits, (k+1)*bits)), which is what the runtime's extractor
+assumes; at bits=4 it is the classic nibble packing (low nibble = even column).
+Reconstruct per group:
+ w_group = (codebook_bits[idx] * norm) @ H # H = normalized Walsh-Hadamard(group)
+
+TERNARY (record bits=5, 3 levels) stores four signed 2-bit crumbs per byte. The
+crumb codes are 3, 0, 1 for trit indices 0, 1, 2, so sign-extending each 2-bit
+field directly produces -1, 0, +1 in the NEON kernel. The codebook is not
+in the header: the 3-level Lloyd-Max centroids {-c, 0, +c} / sqrt(group),
+c = 1.2240064, are analytic.
+
+RAW "tokenizer" blob (self-contained SentencePiece BPE dump; see RefTokenizer
+below for the reference encoder/decoder this specifies):
+ header u32 n_pieces, u32 pad/eos/bos/unk id, u8 add_dummy_prefix,
+ u8 byte_fallback, u16 _pad;
+ then n_pieces records, id order: f32 score, u8 type
+ (0 NORMAL, 1 UNKNOWN, 2 CONTROL, 3 USER_DEFINED, 4 BYTE),
+ u16 surface_len, surface_len UTF-8 bytes.
+=============================================================================
+"""
import os
-import pickle
-from dataclasses import replace
-from pathlib import Path
+import struct
-import jax
-import jax.numpy as jnp
import numpy as np
+from .quantize import (_bits_for, _cq_codebook_np, _cq_hadamard_np,
+ parse_bits_map, TERNARY_BITS)
from .architecture import TransformerConfig
-_FFN_KERNEL_NAMES = {"gate_proj", "up_proj", "down_proj"}
+HEAD_STAGES = ("conf-head", "tool-head")
+TAG = 0x05E12A82
+ALIGN = 64
+FP16, FP32, CQ, RAW = 1, 2, 3, 4
+CB_BITS = (2, 3, 4)
+TERNARY_RECORD_BITS = 5
-def export_submodel(checkpoint_path, factor, output_path):
- """Slice a full matryoshka checkpoint to a sub-model at given shrink factor.
+TK_NORMAL, TK_UNKNOWN, TK_CONTROL, TK_USER_DEFINED, TK_BYTE = 0, 1, 2, 3, 4
+_TK_HDR = "<IIIIIBBH"
+_TK_REC = "<fBH"
- factor: how many times smaller the FFN width (e.g. 2 = half, 4 = quarter).
- Attention, embeddings, and norms are unchanged.
- """
+_HDR_FMT = "<IIIII"
+_REC_FMT = "<BBHIIIIQQII"
+REC_SIZE = struct.calcsize(_REC_FMT)
- with open(checkpoint_path, "rb") as f:
- data = pickle.load(f)
- params = data["params"]
- config = TransformerConfig(**data["config"])
- d_ff_new = config.d_ff // factor
- if d_ff_new == 0:
- raise ValueError(f"factor={factor} too large: would give d_ff=0")
+def _geometry(config):
+ head_dim = (getattr(config, "attn_dim", 0) or config.d_model) // config.num_heads
+ orders = tuple(getattr(config, "engram_orders", (2, 3)))
+ heads = getattr(config, "engram_heads", 0) or max(1, config.d_model // (len(orders) * ENGRAM_SUB_DIM))
+ sub_dim = config.d_model // (len(orders) * heads)
+ sites = tuple(getattr(config, "engram_layers", (2, 15)))
+ return head_dim, orders, heads, sub_dim, sites
- d_ff = config.d_ff
- def slice_leaf(key_path, leaf):
- arr = np.asarray(leaf)
- if arr.ndim not in (2, 3):
- return arr
+ENGRAM_SUB_DIM = 128
+ENGRAM_CONV_TAPS = 4
- parent_name = None
- for part in key_path:
- name = part.key if hasattr(part, "key") else str(part)
- if name in _FFN_KERNEL_NAMES:
- parent_name = name
- break
- if parent_name is None:
- return arr
-
- if arr.ndim == 2:
- rows, cols = arr.shape
- if parent_name in ("gate_proj", "up_proj"):
- if cols == d_ff:
- return arr[:, :d_ff_new]
- elif parent_name == "down_proj":
- if rows == d_ff:
- return arr[:d_ff_new, :]
- elif arr.ndim == 3:
- _, rows, cols = arr.shape
- if parent_name in ("gate_proj", "up_proj"):
- if cols == d_ff:
- return arr[:, :, :d_ff_new]
- elif parent_name == "down_proj":
- if rows == d_ff:
- return arr[:, :d_ff_new, :]
-
- return arr
-
- sliced = jax.tree_util.tree_map_with_path(slice_leaf, params)
-
- new_config = replace(config, d_ff=d_ff_new)
-
- sliced_np = jax.tree.map(
- lambda x: np.asarray(x) if isinstance(x, jnp.ndarray) else x, sliced
+def _align(n):
+ return (n + ALIGN - 1) & ~(ALIGN - 1)
+
+
+def _nearest_idx(x, cb):
+ flat = x.reshape(-1)
+ pos = np.clip(np.searchsorted(cb, flat), 1, len(cb) - 1)
+ left, right = cb[pos - 1], cb[pos]
+ idx = np.where(np.abs(flat - left) <= np.abs(flat - right), pos - 1, pos)
+ return idx.reshape(x.shape).astype(np.uint8)
+
+
+def _pack_lsb(idx, bits):
+ out, in_pad = idx.shape
+ g = idx.reshape(out, in_pad // 8, 8).astype(np.uint64)
+ word = np.zeros(g.shape[:-1], np.uint64)
+ for i in range(8):
+ word |= g[..., i] << (i * bits)
+ packed = np.empty(word.shape + (bits,), np.uint8)
+ for b in range(bits):
+ packed[..., b] = (word >> (8 * b)) & 0xFF
+ return packed.reshape(out, in_pad * bits // 8)
+
+
+def _unpack_lsb(packed, bits, in_pad):
+ out = packed.shape[0]
+ chunks = packed.reshape(out, in_pad // 8, bits).astype(np.uint64)
+ word = np.zeros(chunks.shape[:-1], np.uint64)
+ for b in range(bits):
+ word |= chunks[..., b] << (8 * b)
+ idx = np.empty((out, in_pad // 8, 8), np.uint8)
+ mask = (1 << bits) - 1
+ for i in range(8):
+ idx[..., i] = (word >> (i * bits)) & mask
+ return idx.reshape(out, in_pad)
+
+
+def _packed_row_bytes(in_pad, bits, group):
+ if bits == TERNARY_RECORD_BITS:
+ return in_pad * 2 // 8
+ return in_pad * bits // 8
+
+
+def _pack_ternary_crumbs(idx):
+ crumbs = np.where(idx == 0, 3, idx - 1).astype(np.uint8)
+ return _pack_lsb(crumbs, 2)
+
+
+def _unpack_ternary_crumbs(packed, in_pad):
+ crumbs = _unpack_lsb(packed, 2, in_pad)
+ return np.where(crumbs == 3, 0, crumbs + 1).astype(np.uint8)
+
+
+def _cq_pack(w, bits, group):
+ if bits not in CB_BITS and bits != TERNARY_BITS:
+ raise ValueError(f"CQ packing supports bits in {CB_BITS} or {TERNARY_BITS}; got bits={bits}")
+ cb = _cq_codebook_np(bits, group)
+ H = _cq_hadamard_np(group)
+ out, D = w.shape
+ pad = (-D) % group
+ wp = np.pad(w, ((0, 0), (0, pad))) if pad else w
+ in_pad = wp.shape[1]
+ g = wp.reshape(out, in_pad // group, group).astype(np.float32)
+ rot = g @ H
+ norm = np.sqrt((rot ** 2).sum(-1, keepdims=True))
+ unit = rot / np.maximum(norm, 1e-12)
+ idx = _nearest_idx(unit, cb).reshape(out, in_pad)
+ packed = _pack_ternary_crumbs(idx) if bits == TERNARY_BITS else _pack_lsb(idx, bits)
+ return packed, norm[:, :, 0].astype(np.float16)
+
+
+def _cq_unpack(packed, norms, out, in_dim, bits, group):
+ record_bits = bits
+ if bits == TERNARY_RECORD_BITS:
+ bits = TERNARY_BITS
+ cb = _cq_codebook_np(bits, group)
+ H = _cq_hadamard_np(group)
+ in_pad = (in_dim + group - 1) // group * group
+ if record_bits == TERNARY_RECORD_BITS:
+ idx = _unpack_ternary_crumbs(packed, in_pad)
+ else:
+ idx = _unpack_lsb(packed, bits, in_pad)
+ unit = cb[idx].reshape(out, in_pad // group, group)
+ rot = unit * norms.reshape(out, in_pad // group, 1).astype(np.float32)
+ w = (rot @ H).reshape(out, in_pad)
+ return w[:, :in_dim]
+
+
+class _Tensor:
+ __slots__ = ("name", "dtype", "shape", "blob", "group", "bits", "offset")
+
+ def __init__(self, name, dtype, shape, blob, group=0, bits=0):
+ self.name, self.dtype, self.shape = name, dtype, tuple(shape)
+ self.blob, self.group, self.bits, self.offset = blob, group, bits, 0
+
+
+def _fp16(name, arr):
+ arr = np.asarray(arr, np.float16)
+ return _Tensor(name, FP16, arr.shape, arr.tobytes())
+
+
+def _q(name, mat, bits, group):
+ if isinstance(bits, tuple):
+ bits = _bits_for(name, *bits)
+ packed, norms = _cq_pack(np.asarray(mat, np.float32), bits, group)
+ rec_bits = TERNARY_RECORD_BITS if bits == TERNARY_BITS else bits
+ return _Tensor(name, CQ, mat.shape, packed.tobytes() + norms.tobytes(), group, rec_bits)
+
+
+def _tensors(params, config, bits, group):
+ _, orders, heads, sub_dim, sites = _geometry(config)
+ num_tables = len(orders) * heads
+ ts = [_q("embedding", np.asarray(_get(params, ("embedding", "embedding"))), bits, group)]
+
+ b = params["stack"]["layers"]["block"]
+ sa, ha = b["self_attn"], b["hadamard_mlp"]
+ for i in range(config.num_layers):
+ ts += [
+ _fp16(f"layer{i:02d}.norm_in", b["ZCRMSNorm_0"]["scale"][i]),
+ _q(f"layer{i:02d}.q_proj", np.asarray(sa["q_proj"]["kernel"][i]).T, bits, group),
+ _q(f"layer{i:02d}.k_proj", np.asarray(sa["k_proj"]["kernel"][i]).T, bits, group),
+ _q(f"layer{i:02d}.v_proj", np.asarray(sa["v_proj"]["kernel"][i]).T, bits, group),
+ _fp16(f"layer{i:02d}.q_norm", sa["q_norm"]["scale"][i]),
+ _fp16(f"layer{i:02d}.k_norm", sa["k_norm"]["scale"][i]),
+ _q(f"layer{i:02d}.gate_proj", np.asarray(sa["gate_proj"]["kernel"][i]).T, bits, group),
+ _q(f"layer{i:02d}.out_proj", np.asarray(sa["out_proj"]["kernel"][i]).T, bits, group),
+ _fp16(f"layer{i:02d}.post_norm", b["post_attn_norm"]["scale"][i]),
+ _fp16(f"layer{i:02d}.attn_gate", np.asarray(b["attn_gate"][i]).reshape(1)),
+ _fp16(f"layer{i:02d}.pre_hada", b["pre_hada_norm"]["scale"][i]),
+ _fp16(f"layer{i:02d}.d1", ha["d1"][i]),
+ _fp16(f"layer{i:02d}.d2", ha["d2"][i]),
+ _fp16(f"layer{i:02d}.d3", ha["d3"][i]),
+ ]
+
+ mhc = params["stack"]
+ for name in ("mhc_a_pre", "mhc_a_post", "mhc_a_res", "mhc_b_pre", "mhc_b_post",
+ "mhc_b_res"):
+ ts.append(_fp16(name, np.asarray(mhc[name])))
+ for name in ("mhc_phi_pre", "mhc_phi_post", "mhc_phi_res"):
+ phi = np.asarray(mhc[name])
+ L, nC, lanes = phi.shape
+ ts.append(_q(name, phi.transpose(0, 2, 1).reshape(L * lanes, nC), bits, group))
+
+ for s in range(len(sites)):
+ eg = params[f"engrams_{s}"]
+ tables = np.asarray(eg["embedding"]).reshape(num_tables * config.engram_slots, sub_dim)
+ ts += [
+ _q(f"engram{s}.tables", tables, bits, group),
+ _q(f"engram{s}.key_proj", np.asarray(eg["key_proj"]["kernel"]).T, bits, group),
+ _q(f"engram{s}.value_proj", np.asarray(eg["value_proj"]["kernel"]).T, bits, group),
+ _fp16(f"engram{s}.taps", np.asarray(eg["taps"])),
+ ]
+
+ ts.append(_fp16("final_norm", params["stack"]["final_norm"]["scale"]))
+ ts += _head_tensors(params)
+ return ts
+
+
+HEAD_CODES = (("contrastive_head", 1), ("confidence_head", 2))
+
+
+def _head_tensors(params):
+ present = [(k, c) for k, c in HEAD_CODES if k in params]
+ if not present:
+ return []
+ ts = [_fp16("heads.manifest", np.asarray([c for _, c in present], np.float16))]
+ for key, _ in present:
+ h = params[key]
+ kernel = np.asarray(h["proj"]["kernel"]).T
+ bias = (np.asarray(h["proj"]["bias"]) if "bias" in h["proj"]
+ else np.zeros(kernel.shape[0], np.float16))
+ ts += [
+ _fp16(f"{key}.probes", np.asarray(h["probes"])),
+ _fp16(f"{key}.proj", kernel),
+ _fp16(f"{key}.bias", bias),
+ ]
+ return ts
+
+
+def _get(params, keys):
+ node = params
+ for k in keys:
+ node = node[k]
+ return node
+
+
+def _tokenizer_blob(tok):
+ from .tokenizer import CHAT_MARKERS, PAD_ID, EOS_ID, BOS_ID, UNK_ID
+ sp = tok.sp
+ n = sp.GetPieceSize()
+ markers = set(CHAT_MARKERS)
+ out = bytearray(struct.pack(_TK_HDR, n, PAD_ID, EOS_ID, BOS_ID, UNK_ID, 1, 1, 0))
+ for i in range(n):
+ piece = sp.IdToPiece(i)
+ if sp.IsControl(i):
+ t = TK_CONTROL
+ elif sp.IsUnknown(i):
+ t = TK_UNKNOWN
+ elif sp.IsByte(i):
+ t = TK_BYTE
+ elif piece in markers:
+ t = TK_USER_DEFINED
+ else:
+ t = TK_NORMAL
+ b = piece.encode("utf-8")
+ out += struct.pack(_TK_REC, sp.GetScore(i), t, len(b)) + b
+ return bytes(out)
+
+
+def _pack_cact(params, config, bits, group, tokenizer, kv_window=0):
+ params = {k: v for k, v in params.items()}
+ ts = _tensors(params, config, bits, group)
+ if tokenizer is not None:
+ if tokenizer.vocab_size != config.vocab_size:
+ raise ValueError(f"tokenizer vocab {tokenizer.vocab_size} != config {config.vocab_size}")
+ ts.append(_Tensor("tokenizer", RAW, (), _tokenizer_blob(tokenizer)))
+ cb = np.concatenate([_cq_codebook_np(b, group) for b in CB_BITS]).astype(np.float32)
+ kv_bits = int(getattr(config, "kv_bits", 8) or 8)
+ header = struct.pack(_HDR_FMT, TAG, len(ts), len(cb),
+ int(kv_window or 0), kv_bits) + cb.tobytes()
+
+ pos = len(header) + len(ts) * REC_SIZE
+ for t in ts:
+ pos = _align(pos)
+ t.offset = pos
+ pos += len(t.blob)
+
+ directory = b"".join(
+ struct.pack(_REC_FMT, t.dtype, len(t.shape), 0,
+ *(list(t.shape) + [0, 0, 0, 0])[:4], t.offset, len(t.blob),
+ t.group, t.bits)
+ for t in ts
)
- os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
- with open(output_path, "wb") as f:
- pickle.dump({"params": sliced_np, "config": new_config.__dict__}, f)
-
- orig_count = sum(x.size for x in jax.tree.leaves(params))
- new_count = sum(x.size for x in jax.tree.leaves(sliced_np))
- orig_bytes = sum(x.nbytes for x in jax.tree.leaves(params))
- new_bytes = sum(x.nbytes for x in jax.tree.leaves(sliced_np))
-
- print(f"\n Export complete: {output_path}")
- print(f" ─────────────────────────────────────")
- print(f" {'':>20s} {'Original':>12s} {'Exported':>12s}")
- print(f" {'d_model':>20s} {config.d_model:>12d} {config.d_model:>12d}")
- print(f" {'d_ff':>20s} {config.d_ff:>12d} {d_ff_new:>12d}")
- print(f" {'factor':>20s} {'1x':>12s} {str(factor)+'x':>12s}")
- print(f" {'num_heads':>20s} {config.num_heads:>12d} {config.num_heads:>12d}")
- print(f" {'num_kv_heads':>20s} {config.num_kv_heads:>12d} {config.num_kv_heads:>12d}")
- print(f" {'params':>20s} {orig_count:>12,d} {new_count:>12,d}")
- print(f" {'size (MB)':>20s} {orig_bytes / 1e6:>12.1f} {new_bytes / 1e6:>12.1f}")
- print()
-
-
-def slice_params(params, config, factor):
- """Slice params in-memory to a matryoshka sub-model at given shrink factor.
-
- Returns (sliced_params, new_config).
- """
- d_ff = config.d_ff
- d_ff_new = d_ff // factor
- if d_ff_new == 0:
- raise ValueError(f"factor={factor} too large: would give d_ff=0")
-
- def slice_leaf(key_path, leaf):
- arr = np.asarray(leaf)
- if arr.ndim not in (2, 3):
- return arr
- parent_name = None
- for part in key_path:
- name = part.key if hasattr(part, "key") else str(part)
- if name in _FFN_KERNEL_NAMES:
- parent_name = name
+ buf = bytearray(header + directory)
+ for t in ts:
+ buf.extend(b"\x00" * (t.offset - len(buf)))
+ buf.extend(t.blob)
+ return bytes(buf), len(ts)
+
+
+def build_export(params, config, bits=4, group=128, tokenizer=None, bits_map=None,
+ kv_window=0):
+ b = parse_bits_map(bits_map) if bits_map else bits
+ return _pack_cact(params, config, b, group, tokenizer, kv_window)[0]
+
+
+def write_export(params, config, path, bits=4, group=128, tokenizer=None,
+ bits_map=None, kv_window=0):
+ b = parse_bits_map(bits_map) if bits_map else bits
+ buf, n = _pack_cact(params, config, b, group, tokenizer, kv_window)
+ with open(path, "wb") as f:
+ f.write(buf)
+ return {"path": path, "bytes": len(buf), "tensors": n}
+
+
+def read_export(path):
+ with open(path, "rb") as f:
+ raw = f.read()
+ tag, num_tensors, cb_n, kv_window, kv_bits = struct.unpack_from(_HDR_FMT, raw, 0)
+ assert tag == TAG, "bad tag"
+ off = struct.calcsize(_HDR_FMT)
+ codebook = np.frombuffer(raw[off:off + cb_n * 4], np.float32).copy()
+ off += cb_n * 4
+
+ tensors = []
+ for _ in range(num_tensors):
+ rec = struct.unpack(_REC_FMT, raw[off:off + REC_SIZE]); off += REC_SIZE
+ dtype, ndim = rec[0], rec[1]
+ shape = tuple(rec[3:3 + ndim])
+ offset, nbytes, group, bits = rec[7], rec[8], rec[9], rec[10]
+ blob = raw[offset:offset + nbytes]
+ if dtype == FP16:
+ tensors.append(np.frombuffer(blob, np.float16).reshape(shape).astype(np.float32))
+ elif dtype == FP32:
+ tensors.append(np.frombuffer(blob, np.float32).reshape(shape))
+ elif dtype == CQ:
+ out, in_dim = shape
+ in_pad = (in_dim + group - 1) // group * group
+ n_packed = out * _packed_row_bytes(in_pad, bits, group)
+ packed = np.frombuffer(blob[:n_packed], np.uint8).reshape(out, -1)
+ norms = np.frombuffer(blob[n_packed:], np.float16).reshape(out, in_pad // group)
+ tensors.append(_cq_unpack(packed, norms, out, in_dim, bits, group))
+ elif dtype == RAW:
+ tensors.append(blob)
+ return {"num_tensors": num_tensors, "codebook": codebook,
+ "kv_window": kv_window, "kv_bits": kv_bits}, tensors
+
+
+_SP_META_SPACE = "▁"
+
+
+def parse_tokenizer_blob(blob):
+ off = struct.calcsize(_TK_HDR)
+ n, pad, eos, bos, unk, add_dummy, byte_fb, _ = struct.unpack_from(_TK_HDR, blob, 0)
+ rec = struct.calcsize(_TK_REC)
+ pieces, scores, types = [], [], []
+ for _ in range(n):
+ score, t, ln = struct.unpack_from(_TK_REC, blob, off)
+ off += rec
+ pieces.append(blob[off:off + ln].decode("utf-8"))
+ scores.append(score)
+ types.append(t)
+ off += ln
+ return {"pieces": pieces, "scores": scores, "types": types, "pad_id": pad,
+ "eos_id": eos, "bos_id": bos, "unk_id": unk,
+ "add_dummy_prefix": bool(add_dummy), "byte_fallback": bool(byte_fb)}
+
+
+class RefTokenizer:
+ def __init__(self, meta):
+ self.pieces = meta["pieces"]
+ self.scores = meta["scores"]
+ self.types = meta["types"]
+ self.add_dummy = meta["add_dummy_prefix"]
+ self.byte_fallback = meta["byte_fallback"]
+ self.unk_id = meta["unk_id"]
+ self.p2id = {p: i for i, p in enumerate(self.pieces)}
+ self.byte_id = {int(p[3:5], 16): i
+ for i, (p, t) in enumerate(zip(self.pieces, self.types)) if t == TK_BYTE}
+ self.markers = sorted((p for p, t in zip(self.pieces, self.types) if t == TK_USER_DEFINED),
+ key=len, reverse=True)
+
+ @classmethod
+ def from_cact(cls, path):
+ _, ts = read_export(path)
+ blob = next(t for t in ts if isinstance(t, (bytes, bytearray)))
+ return cls(parse_tokenizer_blob(blob))
+
+ def _bpe(self, seg):
+ syms = list(seg)
+ while len(syms) > 1:
+ best_score, best_j = None, -1
+ for j in range(len(syms) - 1):
+ idx = self.p2id.get(syms[j] + syms[j + 1])
+ if idx is not None and (best_score is None or self.scores[idx] > best_score):
+ best_score, best_j = self.scores[idx], j
+ if best_j < 0:
break
- if parent_name is None:
- return arr
- if arr.ndim == 2:
- rows, cols = arr.shape
- if parent_name in ("gate_proj", "up_proj") and cols == d_ff:
- return arr[:, :d_ff_new]
- elif parent_name == "down_proj" and rows == d_ff:
- return arr[:d_ff_new, :]
- elif arr.ndim == 3:
- _, rows, cols = arr.shape
- if parent_name in ("gate_proj", "up_proj") and cols == d_ff:
- return arr[:, :, :d_ff_new]
- elif parent_name == "down_proj" and rows == d_ff:
- return arr[:, :d_ff_new, :]
- return arr
-
- sliced = jax.tree_util.tree_map_with_path(slice_leaf, params)
- new_config = replace(config, d_ff=d_ff_new)
- return sliced, new_config
+ syms[best_j:best_j + 2] = [syms[best_j] + syms[best_j + 1]]
+ ids = []
+ for s in syms:
+ idx = self.p2id.get(s)
+ if idx is not None:
+ ids.append(idx)
+ elif self.byte_fallback:
+ ids.extend(self.byte_id[b] for b in s.encode("utf-8"))
+ else:
+ ids.append(self.unk_id)
+ return ids
+ def encode(self, text):
+ if not text:
+ return []
+ esc = text.replace(" ", _SP_META_SPACE)
+ if self.add_dummy:
+ esc = _SP_META_SPACE + esc
+ ids, buf, i, n = [], [], 0, len(esc)
+ while i < n:
+ marker = next((m for m in self.markers if esc.startswith(m, i)), None)
+ if marker is not None:
+ ids += self._bpe("".join(buf)); buf = []
+ ids.append(self.p2id[marker])
+ i += len(marker)
+ else:
+ buf.append(esc[i]); i += 1
+ ids += self._bpe("".join(buf))
+ return ids
-def main(args):
- checkpoint = args.checkpoint
- factor = args.factor
- output = args.output
+ def decode(self, ids):
+ buf = bytearray()
+ for i in ids:
+ t = self.types[i]
+ if t == TK_BYTE:
+ buf.append(int(self.pieces[i][3:5], 16))
+ elif t in (TK_CONTROL, TK_UNKNOWN):
+ continue
+ else:
+ buf += self.pieces[i].encode("utf-8")
+ text = buf.decode("utf-8", "replace").replace(_SP_META_SPACE, " ")
+ if self.add_dummy and text.startswith(" "):
+ text = text[1:]
+ return text
- if output is None:
- stem = Path(checkpoint).stem
- parent = Path(checkpoint).parent
- output = str(parent / f"{stem}_{factor}x.pkl")
- export_submodel(checkpoint, factor, output)
+def main(args):
+ from .run import load_checkpoint
+ from .tokenizer import get_tokenizer
+ params, config, run_meta = load_checkpoint(args.checkpoint, return_run=True)
+ out = getattr(args, "out", None) or "needle.cact"
+ bits = getattr(args, "bits", None)
+ bits_map = getattr(args, "bits_map", None)
+ if bits is None and not bits_map:
+ bits_map = getattr(config, "weight_bits", "") or ""
+ if bits_map:
+ print(f"[export] scheme from the checkpoint: {bits_map}")
+ elif (run_meta or {}).get("stage") in ("qapt",) + HEAD_STAGES:
+ raise ValueError(
+ f"{os.path.basename(args.checkpoint)} is a "
+ f"'{run_meta['stage']}' checkpoint with no config.weight_bits. "
+ "Exporting it would silently ship plain W4 - numerics it was "
+ "never post-trained for. Migrate it "
+ "(scripts/migrate_checkpoint_numerics.py) or pass --bits-map.")
+ else:
+ bits = 4
+ elif bits is None:
+ bits = 4
+ from .architecture import effective_kv_window
+ kv_window = effective_kv_window(config)
+ info = write_export(params, config, out, bits=bits, bits_map=bits_map,
+ tokenizer=get_tokenizer(config.vocab_size),
+ kv_window=kv_window)
+ mb = info["bytes"] / 1e6
+ w = f"mixed[{bits_map}]" if bits_map else f"W{bits}"
+ print(f"wrote {info['path']}: {info['tensors']} tensors (+tokenizer), "
+ f"{mb:.2f} MB ({w}A8)")
diff --git a/needle/model/finetune.py b/needle/model/finetune.py
new file mode 100644
index 0000000..492b51d
--- /dev/null
+++ b/needle/model/finetune.py
@@ -0,0 +1,310 @@
+import json
+import os
+import pickle
+import urllib.request
+
+import numpy as np
+
+from .tokenizer import (
+ get_tokenizer, BOS_ID, EOS_ID, PAD_ID,
+ IM_START, IM_END, THINK_START, THINK_END,
+ TOOLS_START, TOOLS_END, TOOL_CALL_START, TOOL_CALL_END,
+)
+
+LORA_TARGETS = ("q_proj", "k_proj", "v_proj", "gate_proj", "out_proj")
+DEFAULT_BASE = "checkpoints/needle2.pkl"
+
+OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
+DEFAULT_MODEL = "deepseek/deepseek-v4-flash"
+
+_GEN_SYSTEM = (
+ "You generate training data for a tool-calling model. Given a set of tool "
+ "schemas, produce realistic, diverse user requests and the exact tool calls "
+ "that satisfy them. Return only JSON."
+)
+
+_GEN_TEMPLATE = """Tools available (JSON schemas):
+{tools}
+
+Produce {n} varied examples as a JSON array. Each element is an object:
+ {{"query": "<a natural user request>",
+ "reasoning": "<one short line deriving each argument from the query>",
+ "answers": [{{"name": "<tool name>", "arguments": {{...}}}}]}}
+
+Rules:
+- Use only the tools above; arguments must match their schemas exactly.
+- Cover single-call, multi-call, and about {refusals} off-topic requests that no
+ tool can serve (for those, "answers" is []).
+- Vary phrasing, values, and which tools are used. Return ONLY the JSON array."""
+
+
+def _openrouter(messages, model, api_key, temperature=0.9):
+ payload = json.dumps({"model": model, "messages": messages,
+ "temperature": temperature}).encode("utf-8")
+ request = urllib.request.Request(OPENROUTER_URL, data=payload, headers={
+ "Authorization": "Bearer " + api_key,
+ "Content-Type": "application/json",
+ "HTTP-Referer": "https://github.com/cactus-compute/needle",
+ "X-Title": "needle",
+ })
+ with urllib.request.urlopen(request, timeout=180) as response:
+ return json.loads(response.read().decode("utf-8"))["choices"][0]["message"]["content"]
+
+
+def _parse_array(text):
+ start, end = text.find("["), text.rfind("]")
+ if start < 0 or end <= start:
+ return []
+ try:
+ rows = json.loads(text[start:end + 1])
+ except json.JSONDecodeError:
+ return []
+ return [r for r in rows if isinstance(r, dict) and "query" in r and "answers" in r]
+
+
+def generate_examples(tools, n=25, model=DEFAULT_MODEL, api_key=None, refusals=3):
+ api_key = api_key or os.environ.get("OPENROUTER_API_KEY")
+ if not api_key:
+ raise RuntimeError("set OPENROUTER_API_KEY to generate data")
+ tools_json = tools if isinstance(tools, str) else json.dumps(tools, indent=2)
+ prompt = _GEN_TEMPLATE.format(tools=tools_json, n=n, refusals=refusals)
+ text = _openrouter([{"role": "system", "content": _GEN_SYSTEM},
+ {"role": "user", "content": prompt}], model, api_key)
+ rows = _parse_array(text)
+ for row in rows:
+ row.setdefault("tools", tools if isinstance(tools, list) else json.loads(tools))
+ return rows
+
+
+def _collect_tools(examples):
+ seen, tools = set(), []
+ for example in examples:
+ for tool in example.get("tools", []):
+ name = tool.get("name")
+ if name and name not in seen:
+ seen.add(name)
+ tools.append(tool)
+ return tools
+
+
+def augment_jsonl(path, num_samples, model=DEFAULT_MODEL, batch_size=25, out_path=None):
+ with open(path) as handle:
+ examples = [json.loads(line) for line in handle if line.strip()]
+ tools = _collect_tools(examples)
+ if not tools:
+ raise RuntimeError("no tool schemas found in " + path)
+ out_path = out_path or path.replace(".jsonl", "") + ".augmented.jsonl"
+ generated = []
+ while len(generated) < num_samples:
+ generated.extend(generate_examples(tools, n=min(batch_size, num_samples - len(generated)),
+ model=model or DEFAULT_MODEL))
+ print(f" generated {len(generated)}/{num_samples}")
+ with open(out_path, "w") as handle:
+ for example in examples + generated[:num_samples]:
+ handle.write(json.dumps(example) + "\n")
+ print(f"wrote {len(examples) + min(len(generated), num_samples)} examples -> {out_path}")
+ return out_path
+
+
+def generate_main(args):
+ model = args.model or DEFAULT_MODEL
+ if args.tools:
+ with open(args.tools) as handle:
+ tools = json.load(handle)
+ rows, out = [], args.output or "needle_data.jsonl"
+ while len(rows) < args.num_samples:
+ rows.extend(generate_examples(tools, n=min(args.batch_size, args.num_samples - len(rows)),
+ model=model))
+ print(f" generated {len(rows)}/{args.num_samples}")
+ with open(out, "w") as handle:
+ for row in rows[:args.num_samples]:
+ handle.write(json.dumps(row) + "\n")
+ print(f"wrote {min(len(rows), args.num_samples)} examples -> {out}")
+ elif args.augment:
+ augment_jsonl(args.augment, args.num_samples, model=model,
+ batch_size=args.batch_size, out_path=args.output)
+ else:
+ raise SystemExit("pass --tools <schemas.json> or --augment <data.jsonl>")
+
+
+def render_example(example):
+ tools = example.get("tools", [])
+ tools_json = tools if isinstance(tools, str) else json.dumps(tools, separators=(",", ":"))
+ answers = example.get("answers", example.get("function_calls", []))
+ answers_json = answers if isinstance(answers, str) else json.dumps(answers, separators=(",", ":"))
+ reasoning = example.get("reasoning", "")
+ system = example.get("system", "")
+ prefix = IM_START + "system\n" + system + IM_END + "\n" if system else ""
+ prompt = (prefix + IM_START + "user\n" + TOOLS_START + tools_json + TOOLS_END + "\n"
+ + example["query"] + IM_END + "\n" + IM_START + "assistant\n")
+ target = (THINK_START + reasoning + THINK_END + "\n"
+ + TOOL_CALL_START + answers_json + TOOL_CALL_END + IM_END)
+ return prompt, target
+
+
+def _encode(tokenizer, example, max_len):
+ prompt, target = render_example(example)
+ prompt_ids = tokenizer.encode(prompt)
+ target_ids = tokenizer.encode(target)
+ ids = [BOS_ID] + prompt_ids + target_ids + [EOS_ID]
+ mask = [0.0] * (1 + len(prompt_ids)) + [1.0] * (len(target_ids) + 1)
+ ids, mask = ids[:max_len], mask[:max_len]
+ pad = max_len - len(ids)
+ return ids + [PAD_ID] * pad, mask + [0.0] * pad
+
+
+def load_jsonl(path, tokenizer, max_len):
+ seqs, masks = [], []
+ with open(path) as handle:
+ for line in handle:
+ line = line.strip()
+ if not line:
+ continue
+ example = json.loads(line)
+ if "query" not in example:
+ continue
+ ids, mask = _encode(tokenizer, example, max_len)
+ seqs.append(ids)
+ masks.append(mask)
+ return np.array(seqs, np.int32), np.array(masks, np.float32)
+
+
+def lora_target_paths(params):
+ from flax.traverse_util import flatten_dict
+ return [path for path in flatten_dict(params)
+ if path[-1] == "kernel" and any(t in path for t in LORA_TARGETS)]
+
+
+def init_lora(params, paths, rank, key):
+ import jax
+ import jax.numpy as jnp
+ from flax.traverse_util import flatten_dict
+ flat = flatten_dict(params)
+ lora = {}
+ for path in paths:
+ weight = flat[path]
+ in_dim, out_dim = weight.shape[-2], weight.shape[-1]
+ lead = weight.shape[:-2]
+ key, sub = jax.random.split(key)
+ lora[path] = {
+ "A": (jax.random.normal(sub, lead + (in_dim, rank), jnp.float32) / rank).astype(weight.dtype),
+ "B": jnp.zeros(lead + (rank, out_dim), weight.dtype),
+ }
+ return lora
+
+
+def merge_lora(params, lora, scale):
+ import jax.numpy as jnp
+ from flax.traverse_util import flatten_dict, unflatten_dict
+ flat = dict(flatten_dict(params))
+ for path, adapter in lora.items():
+ flat[path] = flat[path] + scale * jnp.matmul(adapter["A"], adapter["B"]).astype(flat[path].dtype)
+ return unflatten_dict(flat)
+
+
+def finetune_local(args):
+ import jax
+ import jax.numpy as jnp
+ import optax
+ from .run import load_checkpoint
+ from .architecture import SimpleAttentionNetwork
+
+ base_path = args.checkpoint or DEFAULT_BASE
+ data_path = args.jsonl_path
+ if getattr(args, "generate", 0):
+ data_path = augment_jsonl(data_path, args.generate, model=getattr(args, "model", None))
+
+ params, config = load_checkpoint(base_path)
+ tokenizer = get_tokenizer(config.vocab_size)
+ seqs, masks = load_jsonl(data_path, tokenizer, args.max_len)
+ if len(seqs) == 0:
+ raise SystemExit("no usable examples in " + data_path)
+ print(f"training on {len(seqs)} examples, seq_len {args.max_len}")
+
+ model = SimpleAttentionNetwork(config)
+ paths = lora_target_paths(params)
+ scale = args.lora_alpha / args.lora_rank
+ lora = init_lora(params, paths, args.lora_rank, jax.random.PRNGKey(0))
+ print(f"LoRA rank {args.lora_rank} on {len(paths)} weight groups")
+
+ optimizer = optax.adamw(args.lr)
+ opt_state = optimizer.init(lora)
+
+ def loss_fn(lora, ids, mask):
+ logits = model.apply({"params": merge_lora(params, lora, scale)}, ids)
+ logits, targets, mask = logits[:, :-1], ids[:, 1:], mask[:, 1:]
+ ce = optax.softmax_cross_entropy_with_integer_labels(logits, targets)
+ return (ce * mask).sum() / jnp.maximum(mask.sum(), 1.0)
+
+ @jax.jit
+ def train_step(lora, opt_state, ids, mask):
+ loss, grads = jax.value_and_grad(loss_fn)(lora, ids, mask)
+ updates, opt_state = optimizer.update(grads, opt_state, lora)
+ return optax.apply_updates(lora, updates), opt_state, loss
+
+ batch, count = args.batch_size, len(seqs)
+ for epoch in range(args.epochs):
+ order = np.random.permutation(count)
+ last = 0.0
+ for start in range(0, count, batch):
+ idx = order[start:start + batch]
+ lora, opt_state, loss = train_step(lora, opt_state,
+ jnp.asarray(seqs[idx]), jnp.asarray(masks[idx]))
+ last = float(loss)
+ print(f"epoch {epoch + 1}/{args.epochs} loss {last:.4f}")
+
+ os.makedirs(args.checkpoint_dir, exist_ok=True)
+ out = args.out or os.path.join(args.checkpoint_dir, "needle_lora.pkl")
+ with open(out, "wb") as handle:
+ pickle.dump({
+ "lora": {"/".join(p): {"A": np.asarray(v["A"]), "B": np.asarray(v["B"])}
+ for p, v in lora.items()},
+ "scale": float(scale),
+ "base": base_path,
+ "rank": args.lora_rank,
+ }, handle)
+ print(f"saved LoRA adapter -> {out}")
+ print(f"merge + export with: needle build {base_path} --lora {out}")
+
+
+def build_main(args):
+ import jax.numpy as jnp
+ from .run import load_checkpoint
+ from .architecture import effective_kv_window
+ from .export import write_export
+
+ params, config, _ = load_checkpoint(args.checkpoint, return_run=True)
+
+ if args.lora:
+ with open(args.lora, "rb") as handle:
+ adapter = pickle.load(handle)
+ lora = {tuple(key.split("/")): {"A": jnp.asarray(v["A"]), "B": jnp.asarray(v["B"])}
+ for key, v in adapter["lora"].items()}
+ params = merge_lora(params, lora, adapter["scale"])
+ print(f"merged LoRA adapter {args.lora} into {len(lora)} weight groups")
+
+ bits = args.bits
+ bits_map = None if bits else (getattr(config, "weight_bits", "") or None)
+ if not bits and not bits_map:
+ bits = "4"
+
+ out = args.out or (os.path.splitext(os.path.basename(args.checkpoint))[0] + ".cact")
+ info = write_export(params, config, out,
+ bits=int(bits) if bits else 4,
+ bits_map=bits_map,
+ tokenizer=get_tokenizer(config.vocab_size),
+ kv_window=effective_kv_window(config))
+ scheme = f"mixed[{bits_map}]" if bits_map else f"W{bits}"
+ print(f"exported {info['path']} ({info['bytes'] / 1e6:.2f} MB, {info['tensors']} tensors, {scheme}A8)")
+ print(f"run it with: needle.Needle(weights={out!r}, tools=[...])")
+
+ if args.upload:
+ repo = os.environ.get("NEEDLE_HF_REPO")
+ if not repo:
+ raise SystemExit("set NEEDLE_HF_REPO=<you>/<model> to upload")
+ from huggingface_hub import HfApi
+ api = HfApi()
+ api.create_repo(repo, repo_type="model", exist_ok=True)
+ api.upload_file(path_or_fileobj=out, path_in_repo=os.path.basename(out),
+ repo_id=repo, repo_type="model")
+ print(f"uploaded {os.path.basename(out)} -> {repo}")
diff --git a/needle/model/quantize.py b/needle/model/quantize.py
index dc449e6..fd7a487 100644
--- a/needle/model/quantize.py
+++ b/needle/model/quantize.py
@@ -1,68 +1,366 @@
+import functools
+import math
+import re
+
+import numpy as np
import jax
import jax.numpy as jnp
-def _fake_quantize_int4(w, group_size=32):
- """Symmetric group-wise INT4 fake quantization with STE.
+def fake_quant(w, group_size=128, bits=4):
+ qmax = 2 ** (bits - 1) - 1
+ D = w.shape[-1]
+ pad = (-D) % group_size
+ wp = jnp.pad(w, [(0, 0)] * (w.ndim - 1) + [(0, pad)]) if pad else w
+ g = wp.reshape(*wp.shape[:-1], -1, group_size).astype(jnp.float32)
+ absmax = jnp.max(jnp.abs(g), axis=-1, keepdims=True)
+ scale = jnp.where(absmax > 0, absmax / qmax, 1.0)
+ q = jnp.clip(jnp.round(g / scale), -qmax - 1, qmax) * scale
+ q = q.reshape(wp.shape).astype(w.dtype)
+ if pad:
+ q = q[..., :D]
+ return w + jax.lax.stop_gradient(q - w)
+
+
+def quantize_params(params, group_size=128, bits=4):
+ def q(path, leaf):
+ name = path[-1].key
+ if name in ("kernel", "embedding") and leaf.ndim >= 2:
+ return fake_quant(leaf, group_size, bits)
+ return leaf
+ return jax.tree_util.tree_map_with_path(q, params)
+
+def fake_quant_act(x):
+ return fake_quant(x, x.shape[-1], ACT_BITS)
+
+
+def cq_fake_quant_kv(x, bits, group=64):
+ return x + jax.lax.stop_gradient(cq_quantize(x, bits, group) - x)
+
+
+def maybe_quant_kv(x, quant):
+ if not KV_BITS:
+ return x
+ return jax.lax.cond(
+ quant, lambda t: cq_fake_quant_kv(t, KV_BITS, _KV_GROUP), lambda t: t, x)
+
+
+QAT_EVERY = 0
+_WEIGHT_GROUP = 128
+_WEIGHT_BITS = 4
+ACT_BITS = 8
+KV_BITS = 0
+_KV_GROUP = 64
+
+
+def configure_qat(every, weight_group=128, weight_bits=4):
+ global QAT_EVERY, _WEIGHT_GROUP, _WEIGHT_BITS
+ QAT_EVERY, _WEIGHT_GROUP, _WEIGHT_BITS = int(every), int(weight_group), int(weight_bits)
+
+
+def configure_deploy(act_bits=8, kv_bits=8, kv_group=64):
+ global ACT_BITS, KV_BITS, _KV_GROUP
+ kv_bits = 0 if int(kv_bits) >= 8 else int(kv_bits)
+ changed = (ACT_BITS, KV_BITS, _KV_GROUP) != (int(act_bits), int(kv_bits), int(kv_group))
+ ACT_BITS, KV_BITS, _KV_GROUP = int(act_bits), int(kv_bits), int(kv_group)
+ if changed:
+ jax.clear_caches()
+
+
+def quantize_params_configured(params):
+ return quantize_params(params, _WEIGHT_GROUP, _WEIGHT_BITS)
+
+
+def deploy_quantize(params, config):
+ spec = getattr(config, "weight_bits", "") or ""
+ if spec:
+ bits_map, default_bits = parse_bits_map(spec)
+ return cq_mixed_params(params, bits_map, default_bits), spec
+ return (cq_quantize_params(params, _WEIGHT_BITS, _WEIGHT_GROUP),
+ f"CQ W{_WEIGHT_BITS}")
+
+
+def weight_bits():
+ return _WEIGHT_BITS
+
+
+def maybe_quant_weights(params, do_quantize):
+ if not QAT_EVERY:
+ return params
+ return jax.lax.cond(
+ do_quantize, quantize_params_configured, lambda p: p, params)
+
+
[email protected]_cache(maxsize=None)
+def _lloyd_max_gaussian(bits, iters=200, samples=400000, seed=0):
+ levels = 1 << bits
+ x = np.sort(np.random.RandomState(seed).randn(samples))
+ c = x[((np.arange(levels) + 0.5) / levels * samples).astype(int)].astype(np.float64)
+ for _ in range(iters):
+ bnd = (c[:-1] + c[1:]) / 2.0
+ idx = np.searchsorted(bnd, x)
+ for k in range(levels):
+ m = idx == k
+ if m.any():
+ c[k] = x[m].mean()
+ return np.sort(c)
+
+TERNARY_BITS = 1.58
+_TERNARY_CB = np.array([-1.2240064, 0.0, 1.2240064])
+
+
[email protected]_cache(maxsize=None)
+def _cq_codebook_np(bits, group_size):
+ cb = _TERNARY_CB if bits == TERNARY_BITS else _lloyd_max_gaussian(bits)
+ return (cb / np.sqrt(group_size)).astype(np.float32)
+
+
[email protected]_cache(maxsize=None)
+def _cq_hadamard_np(group_size):
+ H = np.array([[1.0]], dtype=np.float32)
+ while H.shape[0] < group_size:
+ H = np.block([[H, H], [H, -H]])
+ return (H / np.sqrt(group_size)).astype(np.float32)
+
+
+def _cq_nearest(x, cb):
+ flat = x.reshape(-1)
+ pos = jnp.clip(jnp.searchsorted(cb, flat), 1, cb.shape[0] - 1)
+ left, right = cb[pos - 1], cb[pos]
+ idx = jnp.where(jnp.abs(flat - left) <= jnp.abs(flat - right), pos - 1, pos)
+ return cb[idx].reshape(x.shape)
+
+
+def cq_quantize(w, bits, group_size=128, codebook=None):
+ cb = codebook if codebook is not None else jnp.asarray(_cq_codebook_np(bits, group_size))
+ D, g = w.shape[-1], group_size
+ pad = (-D) % g
+ wp = jnp.pad(w, [(0, 0)] * (w.ndim - 1) + [(0, pad)]) if pad else w
+ groups = wp.reshape(*wp.shape[:-1], -1, g).astype(jnp.float32)
+ H = jnp.asarray(_cq_hadamard_np(g))
+ rot = groups @ H
+ norm = jnp.sqrt(jnp.sum(rot ** 2, axis=-1, keepdims=True))
+ unit = rot / jnp.maximum(norm, 1e-12)
+ norm = norm.astype(jnp.float16).astype(jnp.float32)
+ deq = (_cq_nearest(unit, cb) * norm) @ H
+ deq = deq.reshape(wp.shape).astype(w.dtype)
+ return deq[..., :D] if pad else deq
+
+
+def _is_quant_leaf(path, leaf):
+ key = path[-1].key
+ return ((key in ("kernel", "embedding") or key.startswith("mhc_phi"))
+ and getattr(leaf, "ndim", 0) >= 2)
+
+
+def _reduces_second_last(path):
+ key = path[-1].key
+ return key == "kernel" or key.startswith("mhc_phi")
+
- Divides the input dimension (axis 0) into groups of `group_size` elements,
- each with its own scale factor. Falls back to per-channel if in_features < group_size.
- """
- in_feat, out_feat = w.shape
- gs = min(group_size, in_feat)
+def cq_quantize_params(params, bits, group_size=128):
+ cb = jnp.asarray(_cq_codebook_np(bits, group_size))
- pad = (gs - in_feat % gs) % gs
- if pad > 0:
- w_padded = jnp.pad(w, ((0, pad), (0, 0)))
- else:
- w_padded = w
+ def q(path, leaf):
+ if not _is_quant_leaf(path, leaf):
+ return leaf
+ if _reduces_second_last(path):
+ rotated = cq_quantize(jnp.swapaxes(leaf, -1, -2), bits, group_size, cb)
+ return jnp.swapaxes(rotated, -1, -2)
+ return cq_quantize(leaf, bits, group_size, cb)
- num_groups = w_padded.shape[0] // gs
- w_grouped = w_padded.reshape(num_groups, gs, out_feat)
+ return jax.tree_util.tree_map_with_path(q, params)
- scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 7.0
- scale = jnp.maximum(scale, 1e-8)
- w_q = jnp.clip(jnp.round(w_grouped / scale), -8, 7) * scale
- w_q = w_q.reshape(-1, out_feat)[:in_feat]
+def cq_model_bytes(params, bits, group_size=128):
+ acc = {"q": 0, "groups": 0, "other": 0}
- return w + jax.lax.stop_gradient(w_q - w)
+ def visit(path, leaf):
+ n = int(np.prod(leaf.shape))
+ if _is_quant_leaf(path, leaf):
+ acc["q"] += n
+ per = leaf.shape[-2] if _reduces_second_last(path) else leaf.shape[-1]
+ acc["groups"] += (n // per) * (-(-per // group_size))
+ else:
+ acc["other"] += n
+ return leaf
+
+ jax.tree_util.tree_map_with_path(visit, params)
+ return acc["q"] * bits / 8 + acc["groups"] * 2 + acc["other"] * 2
+
+
+def model_bytes_fp16(params):
+ return 2 * sum(int(np.prod(l.shape)) for l in jax.tree_util.tree_leaves(params))
+
+
+
+
+CQ_GROUP_SIZE = 128
+CQ_BITS = (2, 3, 4)
+MIN_BITS, MAX_BITS = 1, 8
+
+
+def canonical_tensor_name(name):
+ n = re.sub(r"^layer\d+\.", "attn.", name)
+ n = re.sub(r"^engrams?_?(\d+)[./]?", r"engram\1.", n)
+ if n.startswith("stack/"):
+ n = n[len("stack/"):]
+ n = n.replace("layers/block/self_attn/", "attn.")
+ n = n.replace("mtp_block/self_attn/", "mtp.")
+ if n.endswith("/kernel"):
+ n = n[:-len("/kernel")]
+ if n == "embedding/embedding":
+ n = "embedding"
+ n = re.sub(r"^(engram\d+\.)embedding$", r"\1tables", n)
+ return n.replace("/", ".")
+
+
+def _bits_for(name, bits_map, default_bits):
+ canon = canonical_tensor_name(name)
+ hits = [k for k in bits_map if canon.startswith(k)]
+ return bits_map[max(hits, key=len)] if hits else default_bits
+
+
[email protected]_cache(maxsize=None)
+def cq_distortion(bits, group_size=CQ_GROUP_SIZE, rows=2048, seed=0):
+ w = np.random.RandomState(seed).randn(rows, 4 * group_size).astype(np.float32)
+ deq = np.asarray(cq_quantize(jnp.asarray(w), bits, group_size))
+ return float(np.mean((deq - w) ** 2) / np.mean(w ** 2))
+
+
+def noise_scale(bits, group_size=CQ_GROUP_SIZE):
+ b = min(max(float(bits), MIN_BITS), MAX_BITS)
+ lo, hi = int(np.floor(b)), int(np.ceil(b))
+ d_lo = cq_distortion(lo, group_size)
+ if hi == lo:
+ return float(np.sqrt(d_lo))
+ d_hi = cq_distortion(hi, group_size)
+ t = b - lo
+ return float(np.sqrt(np.exp((1.0 - t) * np.log(d_lo) + t * np.log(d_hi))))
+
+
+def add_cq_noise(w, key, scale, group_size=CQ_GROUP_SIZE):
+ D, g = w.shape[-1], group_size
+ pad = (-D) % g
+ wp = jnp.pad(w, [(0, 0)] * (w.ndim - 1) + [(0, pad)]) if pad else w
+ groups = wp.reshape(*wp.shape[:-1], -1, g).astype(jnp.float32)
+ rms = jnp.sqrt(jnp.mean(jnp.square(groups), axis=-1, keepdims=True))
+ sigma = jax.lax.stop_gradient(rms) * scale
+ eps = jax.random.normal(key, groups.shape, dtype=jnp.float32)
+ noisy = (groups + sigma * eps).reshape(wp.shape).astype(w.dtype)
+ return noisy[..., :D] if pad else noisy
+
+
+def _map_quant_leaves(params, fn):
+ counter = [0]
+ def q(path, leaf):
+ if not _is_quant_leaf(path, leaf):
+ return leaf
+ i = counter[0]
+ counter[0] += 1
+ if _reduces_second_last(path):
+ return jnp.swapaxes(fn(jnp.swapaxes(leaf, -1, -2), i), -1, -2)
+ return fn(leaf, i)
-def _fake_quantize_int8(w, group_size=32):
- """Symmetric group-wise INT8 fake quantization with STE.
+ return jax.tree_util.tree_map_with_path(q, params)
- Same structure as INT4 but with [-128, 127] range (8-bit signed).
- """
- in_feat, out_feat = w.shape
- gs = min(group_size, in_feat)
- pad = (gs - in_feat % gs) % gs
- if pad > 0:
- w_padded = jnp.pad(w, ((0, pad), (0, 0)))
- else:
- w_padded = w
+def noise_params(params, key, scale, group_size=CQ_GROUP_SIZE):
+ return _map_quant_leaves(
+ params,
+ lambda w, i: add_cq_noise(w, jax.random.fold_in(key, i), scale, group_size),
+ )
- num_groups = w_padded.shape[0] // gs
- w_grouped = w_padded.reshape(num_groups, gs, out_feat)
- scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 127.0
- scale = jnp.maximum(scale, 1e-8)
- w_q = jnp.clip(jnp.round(w_grouped / scale), -128, 127) * scale
+def noise_params_only(params, key, scale, name, group_size=CQ_GROUP_SIZE):
+ names = [n for n, _ in quant_leaf_names(params)]
+ return _map_quant_leaves(
+ params,
+ lambda w, i: (add_cq_noise(w, key, scale, group_size)
+ if names[i].startswith(name) else w),
+ )
- w_q = w_q.reshape(-1, out_feat)[:in_feat]
- return w + jax.lax.stop_gradient(w_q - w)
+def leaf_name(path):
+ return "/".join(getattr(p, "key", str(p)) for p in path)
-def _quantize_params(params, group_size=32, precision="int4"):
- """Fake-quantize all Dense kernels in the param tree."""
- qfn = _fake_quantize_int8 if precision == "int8" else _fake_quantize_int4
- def _maybe_quantize(path, leaf):
- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel" and leaf.ndim == 3:
- return jax.vmap(lambda w: qfn(w, group_size=group_size), in_axes=(0,))(leaf)
- if name == "kernel" and leaf.ndim == 2:
- return qfn(leaf, group_size=group_size)
+def quant_leaf_names(params):
+ out = []
+
+ def visit(path, leaf):
+ if _is_quant_leaf(path, leaf):
+ out.append((leaf_name(path), int(np.prod(leaf.shape))))
return leaf
- return jax.tree_util.tree_map_with_path(_maybe_quantize, params)
+
+ jax.tree_util.tree_map_with_path(visit, params)
+ return out
+
+
+def parse_bits_map(spec):
+ m, default = {}, None
+ for part in str(spec or "").split(","):
+ if not part.strip():
+ continue
+ k, v = part.split("=", 1)
+ b = TERNARY_BITS if float(v) == TERNARY_BITS else int(v)
+ if k.strip() == "default":
+ default = b
+ else:
+ m[canonical_tensor_name(k.strip())] = b
+ if default is None:
+ raise ValueError(f"bits map {spec!r} needs a default=<b> entry")
+ bad = {b for b in list(m.values()) + [default]
+ if b not in CQ_BITS and b != TERNARY_BITS}
+ if bad:
+ raise ValueError(f"bits map {spec!r} has unsupported widths {sorted(bad)}")
+ return m, default
+
+
+def cq_mixed_params(params, bits_map, default_bits, group_size=CQ_GROUP_SIZE):
+ names = [n for n, _ in quant_leaf_names(params)]
+
+ def fn(w, i):
+ b = _bits_for(names[i], bits_map, default_bits)
+ return cq_quantize(w, b, group_size)
+
+ return _map_quant_leaves(params, fn)
+
+
+def cq_mixed_stats(params, bits_map, default_bits, group_size=CQ_GROUP_SIZE):
+ qbits = qn = 0
+ other = 0
+ groups = 0
+
+ def visit(path, leaf):
+ nonlocal qbits, qn, other, groups
+ n = int(np.prod(leaf.shape))
+ if _is_quant_leaf(path, leaf):
+ b = _bits_for(leaf_name(path), bits_map, default_bits)
+ qbits += b * n
+ qn += n
+ per = leaf.shape[-2] if _reduces_second_last(path) else leaf.shape[-1]
+ groups += (n // per) * (-(-per // group_size))
+ else:
+ other += n
+ return leaf
+
+ jax.tree_util.tree_map_with_path(visit, params)
+ mb = (qbits / 8 + groups * 2 + other * 2) / 1e6
+ return qbits / max(qn, 1), mb
+
+
+def cq_ste(w, bits, group_size=CQ_GROUP_SIZE):
+ return w + jax.lax.stop_gradient(cq_quantize(w, bits, group_size) - w)
+
+
+def cq_ste_params(params, bits, group_size=CQ_GROUP_SIZE):
+ return _map_quant_leaves(params, lambda w, i: cq_ste(w, bits, group_size))
+
+
+def cq_ste_mixed_params(params, bits_map, default_bits, group_size=CQ_GROUP_SIZE):
+ names = [n for n, _ in quant_leaf_names(params)]
+ return _map_quant_leaves(
+ params,
+ lambda w, i: cq_ste(w, _bits_for(names[i], bits_map, default_bits), group_size))
diff --git a/needle/model/run.py b/needle/model/run.py
index 954022e..11f690f 100644
--- a/needle/model/run.py
+++ b/needle/model/run.py
@@ -1,358 +1,226 @@
-import argparse
-import json as _json
+import os
import pickle
-import re as _re
+import re
import sys
import jax
import jax.numpy as jnp
import numpy as np
-from ..dataset.dataset import get_tokenizer, to_snake_case, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_GEN_LEN
-from .architecture import (
- SimpleAttentionNetwork,
- TransformerConfig,
- make_causal_mask,
- make_padding_mask,
-)
+from .tokenizer import get_tokenizer, BOS_ID, EOS_ID, PAD_ID
+from .architecture import SimpleAttentionNetwork, TransformerConfig
+CHECKPOINT_FORMAT_VERSION = 2
-def normalize_tools(tools_json):
- """Normalize tool names in a tools JSON string to snake_case.
-
- Returns (normalized_json, name_map) where name_map maps
- snake_name -> original_name for reverse mapping.
- """
- try:
- tools = _json.loads(tools_json)
- except (_json.JSONDecodeError, TypeError):
- return tools_json, {}
- name_map = {}
- for t in tools:
- if isinstance(t, dict) and "name" in t:
- orig = t["name"]
- snake = to_snake_case(orig)
- name_map[snake] = orig
- t["name"] = snake
- return _json.dumps(tools, separators=(",", ":")), name_map
-
-
-def restore_tool_names(pred_text, name_map):
- """Replace snake_case tool names in model output back to original names."""
- if not name_map:
- return pred_text
- try:
- calls = _json.loads(pred_text)
- except (_json.JSONDecodeError, TypeError):
- # Fallback: string-level replacement, longest names first
- for snake, orig in sorted(name_map.items(), key=lambda x: len(x[0]), reverse=True):
- pred_text = pred_text.replace(snake, orig)
- return pred_text
- if isinstance(calls, list):
- for c in calls:
- if isinstance(c, dict) and "name" in c:
- c["name"] = name_map.get(c["name"], c["name"])
- elif isinstance(calls, dict) and "name" in calls:
- calls["name"] = name_map.get(calls["name"], calls["name"])
- return _json.dumps(calls, separators=(",", ":"))
+_decode_fn_cache = {}
-_decode_fn_cache = {}
+def average_checkpoints(paths, out):
+ acc, meta, steps = None, None, []
+ def add(dst, src, scale):
+ for k, v in src.items():
+ if isinstance(v, dict):
+ add(dst.setdefault(k, {}), v, scale)
+ else:
+ arr = np.asarray(v, np.float32) * scale
+ dst[k] = dst[k] + arr if k in dst else arr
+
+ scale = 1.0 / len(paths)
+ for p in paths:
+ with open(p, "rb") as f:
+ ckpt = pickle.load(f)
+ if ckpt.get("format_version") != CHECKPOINT_FORMAT_VERSION:
+ raise ValueError(f"{p} is not a format-v{CHECKPOINT_FORMAT_VERSION} checkpoint")
+ acc = acc if acc is not None else {}
+ add(acc, ckpt["params"], scale)
+ meta = ckpt
+ steps.append(ckpt.get("step"))
+
+ def to_fp16(t):
+ return {k: to_fp16(v) if isinstance(v, dict) else v.astype(np.float16)
+ for k, v in t.items()}
+
+ with open(out, "wb") as f:
+ pickle.dump({
+ "format_version": CHECKPOINT_FORMAT_VERSION,
+ "params": to_fp16(acc),
+ "config": meta["config"],
+ "step": meta.get("step"),
+ "run": {**(meta.get("run") or {}), "averaged_from": steps},
+ }, f)
+ print(f"Averaged {len(paths)} checkpoints (steps {steps}) -> {out}")
+
+
+def head_output_name(checkpoint, suffix):
+ stem = os.path.splitext(os.path.basename(checkpoint))[0]
+ stem = re.sub(r"_step_\d+$", "", stem)
+ return f"{stem}_{suffix}"
+
+
+def load_checkpoint(path, return_run=False):
+ if not os.path.exists(path):
+ from huggingface_hub import hf_hub_download
+ from huggingface_hub.errors import EntryNotFoundError
+ from .tokenizer import HF_REPO
+ print(f"{path} not found locally, downloading from HF...", flush=True)
+ rel = os.path.normpath(path).replace(os.sep, "/")
+ names = []
+ if not os.path.isabs(path):
+ names.append(rel if rel.startswith("checkpoints/") else "checkpoints/" + rel)
+ flat = "checkpoints/" + os.path.basename(path)
+ if flat not in names:
+ names.append(flat)
+ err = None
+ for name in names:
+ try:
+ path = hf_hub_download(HF_REPO, name, repo_type="model", local_dir=".")
+ break
+ except EntryNotFoundError as e:
+ err = e
+ else:
+ raise err
+ with open(path, "rb") as f:
+ ckpt = pickle.load(f)
+ version = ckpt.get("format_version") if isinstance(ckpt, dict) else None
+ if version != CHECKPOINT_FORMAT_VERSION:
+ raise ValueError(
+ f"{path} is not a format-v{CHECKPOINT_FORMAT_VERSION} checkpoint "
+ f"(got format_version={version!r}). Old encoder-decoder/tool-calling "
+ f"checkpoints are incompatible with this branch."
+ )
+ config = ckpt["config"]
+ if isinstance(config, dict):
+ config = TransformerConfig(**config)
+ if return_run:
+ return ckpt["params"], config, ckpt.get("run") or {}
+ return ckpt["params"], config
-def _get_decode_fn(model, max_gen_len):
- """Return a JIT-compiled decode function, cached by (model, max_gen_len).
- params is an explicit argument (not closed over) so the same compiled
- function can be reused across calls with different params.
- """
- key = (id(model), max_gen_len)
+def _get_decode_fn(model, buf_len):
+ key = (id(model), buf_len)
if key not in _decode_fn_cache:
- tgt_mask = make_causal_mask(max_gen_len)
-
@jax.jit
- def decode_step(params, dec_buffer, encoder_out, cross_mask):
- return model.apply(
- {"params": params}, dec_buffer, encoder_out,
- self_mask=tgt_mask, cross_mask=cross_mask, method="decode",
- )
-
- _decode_fn_cache[key] = decode_step
+ def decode_fn(params, tokens):
+ return model.apply({"params": params}, tokens)
+ _decode_fn_cache[key] = decode_fn
return _decode_fn_cache[key]
-def load_checkpoint(path):
- with open(path, "rb") as f:
- data = pickle.load(f)
- params = jax.tree.map(lambda x: jnp.array(x, dtype=jnp.bfloat16), data["params"])
- config = TransformerConfig(**data["config"])
- return params, config
-
-
-def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LEN):
- """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""
- tools_sep_id = tokenizer.tools_token_id
- q_toks = tokenizer.encode(query)
- t_toks = tokenizer.encode(tools)
-
- max_query = max_enc_len - 2
- if len(q_toks) > max_query:
- q_toks = q_toks[:max_query]
- remaining = max_enc_len - len(q_toks) - 1
- t_toks = t_toks[:remaining]
- return q_toks + [tools_sep_id] + t_toks
-
-
-def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, seed=0, stream=True, task_token_id=None, normalize=True, constrained=True):
- """Generate tool-call output.
-
- Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
- Decoder: prefilled with [EOS], model predicts <tool_call> first, then answer tokens.
- """
- name_map = {}
- if normalize:
- tools, name_map = normalize_tools(tools)
-
- enc_tokens = _build_encoder_input(tokenizer, query, tools, max_enc_len)
- enc_input = jnp.array([enc_tokens])
-
- pad_id = tokenizer.pad_token_id
- eos_id = tokenizer.eos_token_id
-
- src_mask = make_padding_mask(enc_input, pad_id)
- encoder_out, enc_mask = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, method="encode"
- )
-
- dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
- dec_buffer = dec_buffer.at[0, 0].set(eos_id)
-
- decode_fn = _get_decode_fn(model, max_gen_len)
-
- generated_tokens = []
-
- constrained_decoder = None
- if constrained:
- from .constrained import build_constrained_decoder
- constrained_decoder = build_constrained_decoder([tools], tokenizer)
-
- if stream:
- sys.stdout.write(f"\n")
- sys.stdout.flush()
-
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
+def _step_stats(step_logits):
+ logp = jax.nn.log_softmax(step_logits.astype(jnp.float32), axis=-1)
+ p = jnp.exp(logp)
+ chosen = jnp.max(p, axis=-1)
+ entropy = -jnp.sum(p * logp, axis=-1)
+ return np.asarray(chosen), np.asarray(entropy)
+
+
+def batch_generate(model, params, tokenizer, prompts, max_new_tokens=96,
+ return_signals=False):
+ enc = [([BOS_ID] + tokenizer.encode(p))[:model.config.max_seq_len - 1] for p in prompts]
+ plens = [len(e) for e in enc]
+ buf_len = min(model.config.max_seq_len,
+ -(-(max(plens) + max_new_tokens) // BUF_BUCKET) * BUF_BUCKET)
+ B = len(prompts)
+ buffer = np.full((B, buf_len), PAD_ID, dtype=np.int32)
+ for i, e in enumerate(enc):
+ buffer[i, :plens[i]] = e[:buf_len]
+ decode_fn = _get_decode_fn(model, buf_len)
+ params = jax.device_put(params)
+
+ done = [plens[i] >= buf_len for i in range(B)]
+ gen = [[] for _ in range(B)]
+ probs = [[] for _ in range(B)]
+ ents = [[] for _ in range(B)]
+ for pos in range(min(plens) - 1, buf_len - 1):
+ logits = decode_fn(params, jnp.asarray(buffer))
+ step = logits[:, pos]
+ nxt = np.asarray(jnp.argmax(step, axis=-1))
+ chosen_p, chosen_h = _step_stats(step) if return_signals else (None, None)
+ for i in range(B):
+ if done[i] or pos < plens[i] - 1:
+ continue
+ t = int(nxt[i])
+ if t == EOS_ID:
+ done[i] = True
+ continue
+ gen[i].append(t)
+ if return_signals:
+ probs[i].append(float(chosen_p[i]))
+ ents[i].append(float(chosen_h[i]))
+ buffer[i, pos + 1] = t
+ if len(gen[i]) >= max_new_tokens:
+ done[i] = True
+ if all(done):
+ break
- for i in range(0, max_gen_len - 1):
- next_logits = logits[0, i]
+ texts = [tokenizer.decode(g) for g in gen]
+ if not return_signals:
+ return texts
- if constrained_decoder and constrained_decoder.is_active(0):
- logits_np = np.array(next_logits)
- logits_np = constrained_decoder.constrain_logits(logits_np, 0)
- next_token = int(np.argmax(logits_np))
+ out = []
+ for i in range(B):
+ pr = probs[i]
+ out.append({
+ "text": texts[i],
+ "tokens": gen[i],
+ "token_probs": pr,
+ "token_entropies": ents[i],
+ "min_prob": min(pr) if pr else 0.0,
+ "mean_logprob": float(np.mean(np.log(np.clip(pr, 1e-12, 1.0)))) if pr else float("-inf"),
+ "max_entropy": max(ents[i]) if ents[i] else 0.0,
+ })
+ return out
+
+
+def generate(model, params, tokenizer, prompt, max_new_tokens=256, temperature=0.0,
+ seed=0, stream=True):
+ prompt_ids = [BOS_ID] + tokenizer.encode(prompt)
+ buf_len = min(model.config.max_seq_len, len(prompt_ids) + max_new_tokens)
+ if len(prompt_ids) >= buf_len:
+ raise ValueError(f"Prompt ({len(prompt_ids)} tokens) does not fit in max_seq_len={model.config.max_seq_len}")
+
+ buffer = jnp.full((1, buf_len), PAD_ID, dtype=jnp.int32)
+ buffer = buffer.at[0, :len(prompt_ids)].set(jnp.array(prompt_ids, dtype=jnp.int32))
+ decode_fn = _get_decode_fn(model, buf_len)
+ rng = jax.random.PRNGKey(seed)
+
+ generated = []
+ printed = ""
+ for pos in range(len(prompt_ids) - 1, buf_len - 1):
+ logits = decode_fn(params, buffer)[0, pos]
+ if temperature <= 0.0:
+ next_token = int(jnp.argmax(logits))
else:
- next_token = int(jnp.argmax(next_logits))
-
- if constrained_decoder:
- constrained_decoder.update(0, next_token)
-
- if next_token == eos_id:
+ rng, sample_rng = jax.random.split(rng)
+ next_token = int(jax.random.categorical(sample_rng, logits / temperature))
+ if next_token == EOS_ID:
break
-
- generated_tokens.append(next_token)
- dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
-
+ generated.append(next_token)
+ buffer = buffer.at[0, pos + 1].set(next_token)
if stream:
- sys.stdout.write(tokenizer.decode([next_token]))
+ text = tokenizer.decode(generated)
+ sys.stdout.write(text[len(printed):])
sys.stdout.flush()
+ printed = text
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
+ text = tokenizer.decode(generated)
if stream:
- sys.stdout.write("\n")
-
- result = tokenizer.decode(generated_tokens)
- # Strip leading <tool_call> token text from output if present
- if result.startswith("<tool_call>"):
- result = result[len("<tool_call>"):]
- if normalize and name_map:
- result = restore_tool_names(result, name_map)
- return result
-
-
-def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, normalize=True, constrained=True):
- """Batch-generate tool-call outputs for multiple examples at once.
-
- Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.
- Decoder: prefilled with [EOS], model predicts <tool_call> first, then answer tokens.
-
- Returns a list of decoded strings, one per example.
- """
- name_maps = []
- if normalize:
- normed_tools = []
- for t in tools_list:
- nt, nm = normalize_tools(t)
- normed_tools.append(nt)
- name_maps.append(nm)
- tools_list = normed_tools
-
- B = len(queries)
- pad_id = tokenizer.pad_token_id
- eos_id = tokenizer.eos_token_id
-
- # --- Encode: [query..., <tools>, tools...] per example, truncated ---
- enc_token_lists = []
- for q, t in zip(queries, tools_list):
- enc_token_lists.append(_build_encoder_input(tokenizer, q, t, max_enc_len))
- max_enc = max(len(toks) for toks in enc_token_lists)
- enc_input = np.full((B, max_enc), pad_id, dtype=np.int32)
- for i, toks in enumerate(enc_token_lists):
- enc_input[i, :len(toks)] = toks
- enc_input = jnp.array(enc_input)
- src_mask = make_padding_mask(enc_input, pad_id)
-
- encoder_out, enc_mask = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, method="encode"
- )
-
- dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)
- dec_buffer[:, 0] = eos_id
- dec_buffer = jnp.array(dec_buffer)
-
- decode_fn = _get_decode_fn(model, max_gen_len)
-
- finished = [False] * B
- gen_tokens = [[] for _ in range(B)]
-
- constrained_decoder = None
- if constrained:
- from .constrained import build_constrained_decoder
- constrained_decoder = build_constrained_decoder(tools_list, tokenizer)
-
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
- for pos in range(0, max_gen_len - 1):
- for i in range(B):
- if finished[i]:
- continue
- if constrained_decoder and constrained_decoder.is_active(i):
- logits_np = np.array(logits[i, pos])
- logits_np = constrained_decoder.constrain_logits(logits_np, i)
- next_token = int(np.argmax(logits_np))
- else:
- next_token = int(jnp.argmax(logits[i, pos]))
- if constrained_decoder:
- constrained_decoder.update(i, next_token)
- if next_token == eos_id:
- finished[i] = True
- continue
- gen_tokens[i].append(next_token)
- dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)
-
- if all(finished):
- break
-
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
- results = []
- for i in range(B):
- text = tokenizer.decode(gen_tokens[i])
- if text.startswith("<tool_call>"):
- text = text[len("<tool_call>"):]
- results.append(text)
- if normalize and name_maps:
- results = [restore_tool_names(r, nm) for r, nm in zip(results, name_maps)]
- return results
+ sys.stdout.write(text[len(printed):] + "\n")
+ sys.stdout.flush()
+ return text
def main(args):
- print(f"Loading checkpoint: {args.checkpoint}")
params, config = load_checkpoint(args.checkpoint)
-
model = SimpleAttentionNetwork(config)
- tokenizer = get_tokenizer()
-
- param_count = sum(x.size for x in jax.tree.leaves(params))
- print(f"Model parameters: {param_count:,}")
-
- use_constrained = not getattr(args, "no_constrained", False)
-
- query = getattr(args, "query", None)
- tools = getattr(args, "tools", None) or "[]"
-
- if query:
- queries = [(query, tools)]
- else:
- queries = [
- ('What is the weather in San Francisco?', '[{"name": "get_weather", "description": "Get current weather for a city.", "parameters": {"location": {"type": "string", "description": "City name.", "required": true}}}]'),
- ('Send an email to [email protected] saying hello', '[{"name": "send_email", "description": "Send an email to a recipient.", "parameters": {"to": {"type": "string", "description": "The recipient email address.", "required": true}, "body": {"type": "string", "description": "The email body text.", "required": true}}}]'),
- ('Get the current stock price of AAPL', '[{"name": "get_stock_price", "description": "Get the current stock price.", "parameters": {"symbol": {"type": "string", "description": "Ticker symbol.", "required": true}}}]'),
- ]
-
- for i, (q, t) in enumerate(queries):
- print(f"\nQuery: {q}")
- print(f"Tools: {t[:80]}{'...' if len(t) > 80 else ''}")
- generate(
- model,
- params,
- tokenizer,
- q,
- tools=t,
- max_gen_len=args.max_len,
- seed=args.seed + i,
- stream=True,
- constrained=use_constrained,
- )
-
-
-def encode_for_retrieval(model, params, tokenizer, texts, max_len=256, batch_size=64):
- """Encode texts into contrastive embeddings for retrieval. Returns (N, contrastive_dim) numpy array."""
- pad_id = tokenizer.pad_token_id
- all_embs = []
-
- for start in range(0, len(texts), batch_size):
- batch_texts = texts[start:start + batch_size]
- token_lists = [tokenizer.encode(t)[:max_len] for t in batch_texts]
- max_t = max(len(toks) for toks in token_lists)
- tokens = np.full((len(batch_texts), max_t), pad_id, dtype=np.int32)
- for i, toks in enumerate(token_lists):
- tokens[i, :len(toks)] = toks
- embs = model.apply(
- {"params": params}, jnp.array(tokens),
- deterministic=True, method="encode_contrastive",
- )
- all_embs.append(np.array(embs))
-
- return np.concatenate(all_embs, axis=0)
-
-
-def retrieve_tools(model, params, tokenizer, query, tool_descriptions, top_k=5, max_len=256):
- """Retrieve top-k tools by cosine similarity to query.
-
- Args:
- query: query string
- tool_descriptions: list of tool description strings (one per tool)
- top_k: number of results to return
-
- Returns:
- list of (index, score) tuples sorted by descending similarity
- """
- q_emb = encode_for_retrieval(model, params, tokenizer, [query], max_len=max_len)
- t_emb = encode_for_retrieval(model, params, tokenizer, tool_descriptions, max_len=max_len)
- scores = (q_emb @ t_emb.T)[0] # (N_tools,)
- top_indices = np.argsort(-scores)[:top_k]
- return [(int(idx), float(scores[idx])) for idx in top_indices]
-
-
-def parse_args():
- parser = argparse.ArgumentParser(description="Generate tool calls with trained transformer")
- parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
- parser.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
- parser.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
- parser.add_argument("--max-len", type=int, default=512)
- parser.add_argument("--seed", type=int, default=0)
- return parser.parse_args()
-
-
-if __name__ == "__main__":
- main(parse_args())
+ tokenizer = get_tokenizer(config.vocab_size)
+
+ prompt = args.prompt or "The most surprising thing about"
+ print(f"prompt: {prompt!r}")
+ generate(
+ model, params, tokenizer, prompt,
+ max_new_tokens=args.max_new_tokens,
+ temperature=args.temperature,
+ seed=args.seed,
+ )
diff --git a/needle/model/tokenizer.py b/needle/model/tokenizer.py
new file mode 100644
index 0000000..df55296
--- /dev/null
+++ b/needle/model/tokenizer.py
@@ -0,0 +1,119 @@
+import os
+
+import sentencepiece as spm
+
+TOKENIZER_DIR = os.path.dirname(__file__)
+TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "tokenizer")
+NANO_TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "tokenizer_nano")
+NANO_VOCAB_SIZE = 4096
+
+PAD_ID = 0
+EOS_ID = 1
+BOS_ID = 2
+UNK_ID = 3
+IM_START = "<|im_start|>"
+IM_END = "<|im_end|>"
+THINK_START = "<think>"
+THINK_END = "</think>"
+TOOLS_START = "<tools>"
+TOOLS_END = "</tools>"
+TOOL_CALL_START = "<tool_call>"
+TOOL_CALL_END = "</tool_call>"
+TOOL_RESULT_START = "<tool_result>"
+TOOL_RESULT_END = "</tool_result>"
+CHAT_MARKERS = [
+ IM_START, IM_END, THINK_START, THINK_END,
+ TOOLS_START, TOOLS_END, TOOL_CALL_START, TOOL_CALL_END,
+ TOOL_RESULT_START, TOOL_RESULT_END,
+]
+IM_START_ID, IM_END_ID, THINK_START_ID, THINK_END_ID = 4, 5, 6, 7
+(TOOLS_START_ID, TOOLS_END_ID, TOOL_CALL_START_ID, TOOL_CALL_END_ID,
+ TOOL_RESULT_START_ID, TOOL_RESULT_END_ID) = range(8, 14)
+
+HF_REPO = "Cactus-Compute/needle-prod"
+_HF_TOKENIZER_DIR = "tokenizer"
+
+
+class SANTokenizer:
+
+ def __init__(self, model_path):
+ self.sp = spm.SentencePieceProcessor()
+ self.sp.Load(model_path)
+ self.model_path = model_path
+
+ @property
+ def md5(self):
+ import hashlib
+ with open(self.model_path, "rb") as f:
+ return hashlib.md5(f.read()).hexdigest()
+
+ @property
+ def pad_token_id(self):
+ return PAD_ID
+
+ @property
+ def eos_token_id(self):
+ return EOS_ID
+
+ @property
+ def bos_token_id(self):
+ return BOS_ID
+
+ @property
+ def vocab_size(self):
+ return self.sp.GetPieceSize()
+
+ def encode(self, text):
+ return self.sp.Encode(text, out_type=int)
+
+ def decode(self, ids):
+ if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):
+ return [self.sp.Decode(seq) for seq in ids]
+ return self.sp.Decode(list(ids))
+
+ def __call__(self, texts, truncation=True, max_length=None, **kwargs):
+ all_ids = []
+ for text in texts:
+ ids = self.sp.Encode(text, out_type=int)
+ if truncation and max_length:
+ ids = ids[:max_length]
+ all_ids.append(ids)
+ return {"input_ids": all_ids}
+
+
+def _prefix_for(vocab_size):
+ return NANO_TOKENIZER_PREFIX if vocab_size == NANO_VOCAB_SIZE else TOKENIZER_PREFIX
+
+
+def _download_tokenizer_from_hf(prefix):
+ from huggingface_hub import hf_hub_download
+
+ os.makedirs(TOKENIZER_DIR, exist_ok=True)
+ base = os.path.basename(prefix)
+ for ext in (".model", ".vocab"):
+ fname = base + ext
+ hf_hub_download(
+ repo_id=HF_REPO,
+ filename=f"{_HF_TOKENIZER_DIR}/{fname}",
+ repo_type="model",
+ local_dir=TOKENIZER_DIR,
+ )
+ nested = os.path.join(TOKENIZER_DIR, _HF_TOKENIZER_DIR, fname)
+ dst = os.path.join(TOKENIZER_DIR, fname)
+ if os.path.exists(nested) and not os.path.exists(dst):
+ os.rename(nested, dst)
+
+
+def get_tokenizer(vocab_size=None):
+ prefix = _prefix_for(vocab_size) if vocab_size is not None else TOKENIZER_PREFIX
+ model_path = prefix + ".model"
+ if not os.path.exists(model_path):
+ try:
+ print(f"Downloading {os.path.basename(prefix)} from HuggingFace...")
+ _download_tokenizer_from_hf(prefix)
+ except Exception as e:
+ raise RuntimeError(
+ f"No pretraining tokenizer at {model_path} and HF download failed ({e}). "
+ f"Run `needle tokenizer-train` (add --upload to share it via HF hub)."
+ ) from e
+ return SANTokenizer(model_path)
diff --git a/needle/training/__init__.py b/needle/training/__init__.py
deleted file mode 100644
index 5732bd7..0000000
--- a/needle/training/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from .optim import create_train_state, _wsd_schedule # noqa: F401
-from .train import train # noqa: F401
-from .pretrain import pretrain # noqa: F401
-from .finetune import finetune_local # noqa: F401
diff --git a/needle/training/eval.py b/needle/training/eval.py
deleted file mode 100644
index 1b0d1dc..0000000
--- a/needle/training/eval.py
+++ /dev/null
@@ -1,661 +0,0 @@
-import argparse
-import math
-import time
-
-import jax
-import jax.numpy as jnp
-import numpy as np
-import optax
-
-from ..dataset.dataset import get_tokenizer, load_tool_calls, load_prepared_data, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN
-from ..model.architecture import (
- SimpleAttentionNetwork,
- TransformerConfig,
- make_causal_mask,
- make_causal_packing_mask,
- make_padding_mask,
- make_packing_mask,
- make_cross_packing_mask,
-)
-from ..model.run import load_checkpoint, _get_decode_fn
-
-
-def compute_perplexity_packed(model, params, packed_data, batch_size):
- """Compute perplexity from pre-packed data with segment IDs."""
- enc = packed_data["packed_enc"]
- dec_in = packed_data["packed_dec_in"]
- dec_tgt = packed_data["packed_dec_tgt"]
- enc_seg = packed_data["packed_enc_seg"]
- dec_seg = packed_data["packed_dec_seg"]
-
- total_loss = 0.0
- total_tokens = 0
- n = len(enc)
-
- for i in range(0, n - batch_size + 1, batch_size):
- src = jnp.asarray(enc[i:i+batch_size], dtype=jnp.int32)
- tgt_in = jnp.asarray(dec_in[i:i+batch_size], dtype=jnp.int32)
- tgt_out = jnp.asarray(dec_tgt[i:i+batch_size], dtype=jnp.int32)
- es = jnp.asarray(enc_seg[i:i+batch_size], dtype=jnp.int32)
- ds = jnp.asarray(dec_seg[i:i+batch_size], dtype=jnp.int32)
-
- src_mask = make_packing_mask(es)
- tgt_mask = make_causal_packing_mask(ds)
- cross_mask = make_cross_packing_mask(es, ds)
-
- logits = model.apply(
- {"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- )
-
- loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
- padding_mask = (ds > 0).astype(jnp.float32)
- total_loss += float(jnp.sum(loss * padding_mask))
- total_tokens += int(jnp.sum(padding_mask))
-
- avg_nll = total_loss / max(total_tokens, 1)
- return math.exp(min(avg_nll, 20))
-
-
-def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is the weather?', max_gen_len=64):
- enc_tokens = tokenizer.encode(prompt)
- enc_input = jnp.array([enc_tokens])
- pad_id = tokenizer.pad_token_id
- eos_id = tokenizer.eos_token_id
-
- src_mask = make_padding_mask(enc_input, pad_id)
- decode_fn = _get_decode_fn(model, max_gen_len)
-
- # Warmup JIT
- encoder_out, enc_mask = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, method="encode"
- )
- dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
- dec_buffer = dec_buffer.at[0, 0].set(eos_id)
- decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
- tokens_generated = []
- latencies = []
-
- for run in range(num_runs):
- rng = jax.random.PRNGKey(run)
- dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
- dec_buffer = dec_buffer.at[0, 0].set(eos_id)
-
- start = time.perf_counter()
- encoder_out, enc_mask = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, method="encode"
- )
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
- num_tokens = 0
- for i in range(max_gen_len - 1):
- rng, sample_rng = jax.random.split(rng)
- next_token = jax.random.categorical(sample_rng, logits[0, i]).item()
-
- if next_token == eos_id:
- break
-
- num_tokens += 1
- dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
- elapsed = time.perf_counter() - start
- tokens_generated.append(num_tokens)
- latencies.append(elapsed)
-
- avg_tokens = np.mean(tokens_generated)
- avg_latency = np.mean(latencies)
- tokens_per_sec = sum(tokens_generated) / sum(latencies)
-
- return {
- "avg_tokens_generated": avg_tokens,
- "avg_latency_s": avg_latency,
- "tokens_per_second": tokens_per_sec,
- }
-
-
-def compute_repetition_rate(texts):
- bigram_rep_rates = []
- for text in texts:
- words = text.lower().split()
- if len(words) < 2:
- bigram_rep_rates.append(0.0)
- continue
- bigrams = [(words[i], words[i + 1]) for i in range(len(words) - 1)]
- unique = len(set(bigrams))
- bigram_rep_rates.append(1.0 - unique / len(bigrams))
- return float(np.mean(bigram_rep_rates))
-
-
-def benchmark_generation_quality(model, params, tokenizer, prompts, max_gen_len=128, temperature=0.8):
- from ..model.run import generate
-
- generations = []
- for i, prompt in enumerate(prompts):
- text = generate(model, params, tokenizer, prompt, max_gen_len, temperature, seed=i, stream=False)
- generations.append(text)
-
- lengths = [len(tokenizer.encode(t)) for t in generations]
- rep_rate = compute_repetition_rate(generations)
-
- return {
- "avg_generation_length": float(np.mean(lengths)),
- "min_generation_length": int(np.min(lengths)),
- "max_generation_length": int(np.max(lengths)),
- "bigram_repetition_rate": rep_rate,
- "generations": list(zip(prompts, generations)),
- }
-
-
-def compute_wer(hypotheses, references):
- """Compute word error rate using edit distance."""
- total_edits = 0
- total_ref_words = 0
-
- for hyp, ref in zip(hypotheses, references):
- hyp_words = hyp.lower().split()
- ref_words = ref.lower().split()
- n = len(ref_words)
- m = len(hyp_words)
-
- # DP edit distance
- d = [[0] * (m + 1) for _ in range(n + 1)]
- for i in range(n + 1):
- d[i][0] = i
- for j in range(m + 1):
- d[0][j] = j
- for i in range(1, n + 1):
- for j in range(1, m + 1):
- if ref_words[i - 1] == hyp_words[j - 1]:
- d[i][j] = d[i - 1][j - 1]
- else:
- d[i][j] = 1 + min(d[i - 1][j], d[i][j - 1], d[i - 1][j - 1])
-
- total_edits += d[n][m]
- total_ref_words += n
-
- return total_edits / max(total_ref_words, 1)
-
-
-def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, constrained=True, ds=None):
- """Generate tool-call predictions and compute structured metrics."""
- import json
- from ..model.run import generate_batch, normalize_tools, restore_tool_names
- from ..dataset.dataset import load_tool_calls, to_snake_case
-
- if ds is None:
- ds = load_tool_calls("validation", max_samples=num_samples)
-
- queries = [ex["query"] for ex in ds]
- tools_list = [ex["tools"] for ex in ds]
- all_preds = generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=max_gen_len, max_enc_len=max_enc_len, normalize=True, constrained=constrained)
-
- total = 0
- exact_match = 0
- tp_names = 0
- fp_names = 0
- fn_names = 0
- tp_calls = 0
- fp_calls = 0
- fn_calls = 0
- json_parse_errors = 0
- empty_ref = 0
- empty_pred = 0
- args_correct = 0
- args_total = 0
- halluc_params = 0
- total_pred_params = 0
- missing_params = 0
- total_ref_params = 0
- correct_values = 0
- matched_params = 0
- samples = []
- failures = [] # (query, tools, ref, pred, reasons)
-
- def _normalize_value(v):
- """Normalize a value for fuzzy comparison."""
- if isinstance(v, str):
- # Try parsing as number to handle "74.006" vs "74.0060"
- try:
- f = float(v)
- # Preserve sign, normalize trailing zeros
- v = str(f)
- except ValueError:
- pass
- # Normalize time-like strings: strip leading "at ", lowercase
- s = v.strip().lower()
- if s.startswith("at "):
- s = s[3:].strip()
- # "today at 21:00" → "21:00", "tonight" is harder but normalize what we can
- if s.startswith("today at "):
- s = s[len("today at "):].strip()
- return s
- if isinstance(v, float):
- return str(v)
- return v
-
- def _normalize_args(args):
- """Normalize all argument values for comparison."""
- if not isinstance(args, dict):
- return args
- return {k: _normalize_value(v) for k, v in args.items()}
-
- def call_key(c):
- if not isinstance(c, dict):
- return None
- norm_args = _normalize_args(c.get("arguments", {}))
- return json.dumps({"name": c.get("name"), "arguments": norm_args}, sort_keys=True)
-
- for i, (ex, pred_text) in enumerate(zip(ds, all_preds)):
- ref_text = ex["answers"]
- pred_text = pred_text.strip()
-
- try:
- ref_calls = json.loads(ref_text)
- except (json.JSONDecodeError, TypeError):
- ref_calls = []
-
- # Normalize ref tool names to snake_case for consistent comparison
- for rc in ref_calls:
- if isinstance(rc, dict) and "name" in rc:
- rc["name"] = to_snake_case(rc["name"])
-
- # Normalize pred tool names to snake_case too (restore_tool_names in
- # generate_batch maps back to originals, but we want snake_case comparison)
- try:
- pred_calls_raw = json.loads(pred_text)
- if isinstance(pred_calls_raw, list):
- for pc in pred_calls_raw:
- if isinstance(pc, dict) and "name" in pc:
- pc["name"] = to_snake_case(pc["name"])
- elif isinstance(pred_calls_raw, dict) and "name" in pred_calls_raw:
- pred_calls_raw["name"] = to_snake_case(pred_calls_raw["name"])
- pred_text = json.dumps(pred_calls_raw)
- except (json.JSONDecodeError, TypeError):
- pass
-
- # Also normalize tool definitions for param validation
- try:
- tool_defs_raw = json.loads(ex["tools"])
- for td in tool_defs_raw:
- if isinstance(td, dict) and "name" in td:
- td["name"] = to_snake_case(td["name"])
- tools_normalized = json.dumps(tool_defs_raw)
- except (json.JSONDecodeError, TypeError):
- tools_normalized = ex["tools"]
-
- try:
- pred_calls = json.loads(pred_text)
- if not isinstance(pred_calls, list):
- pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
- except (json.JSONDecodeError, TypeError):
- json_parse_errors += 1
- pred_calls = []
-
- total += 1
- if i < 10:
- samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))
-
- ref_is_empty = ref_text.strip() in ("", "[]")
- pred_is_empty = pred_text.strip() in ("", "[]")
- if ref_is_empty:
- empty_ref += 1
- if pred_is_empty:
- empty_pred += 1
-
- if ref_is_empty and pred_is_empty:
- exact_match += 1
- elif not ref_is_empty and not pred_is_empty:
- # Order-independent comparison: sort calls by their canonical key
- ref_sorted = sorted([call_key(c) for c in ref_calls if call_key(c)])
- pred_sorted = sorted([call_key(c) for c in pred_calls if call_key(c)])
- if ref_sorted == pred_sorted and len(ref_sorted) == len(ref_calls) and len(pred_sorted) == len(pred_calls):
- exact_match += 1
-
- ref_name_set = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
- pred_name_set = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}
- tp_names += len(pred_name_set & ref_name_set)
- fp_names += len(pred_name_set - ref_name_set)
- fn_names += len(ref_name_set - pred_name_set)
-
- ref_keys = {call_key(c) for c in ref_calls} - {None}
- pred_keys = {call_key(c) for c in pred_calls} - {None}
- tp_calls += len(pred_keys & ref_keys)
- fp_calls += len(pred_keys - ref_keys)
- fn_calls += len(ref_keys - pred_keys)
-
- ref_by_name = {}
- for c in ref_calls:
- if isinstance(c, dict) and "name" in c:
- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))
- for c in pred_calls:
- if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:
- args_total += 1
- pa = json.dumps(_normalize_args(c.get("arguments", {})), sort_keys=True)
- if any(pa == json.dumps(_normalize_args(ra), sort_keys=True) for ra in ref_by_name[c["name"]]):
- args_correct += 1
-
- try:
- tool_defs = json.loads(tools_normalized)
- tool_param_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tool_defs if isinstance(t, dict) and "name" in t}
- except (json.JSONDecodeError, TypeError):
- tool_param_map = {}
- for c in pred_calls:
- if not isinstance(c, dict) or "name" not in c:
- continue
- cname = c["name"]
- if cname not in tool_param_map:
- continue
- schema_keys = tool_param_map[cname]
- p_keys = set((c.get("arguments") or {}).keys())
- total_pred_params += len(p_keys)
- halluc_params += len(p_keys - schema_keys)
- if cname in ref_by_name:
- ref_args = ref_by_name[cname][0]
- r_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())
- total_ref_params += len(r_keys)
- missing_params += len(r_keys - p_keys)
- m_keys = p_keys & r_keys
- matched_params += len(m_keys)
- for k in m_keys:
- pv_norm = _normalize_value(c.get("arguments", {})[k])
- rv_norm = _normalize_value(ref_args[k])
- if json.dumps(pv_norm, sort_keys=True) == json.dumps(rv_norm, sort_keys=True):
- correct_values += 1
-
- # Diagnose failures for this example
- is_exact = (ref_is_empty and pred_is_empty) or (
- not ref_is_empty and not pred_is_empty
- and sorted([call_key(c) for c in ref_calls if call_key(c)])
- == sorted([call_key(c) for c in pred_calls if call_key(c)])
- and len(ref_calls) == len(pred_calls)
- )
- if not is_exact and len(failures) < 50:
- reasons = []
- if json_parse_errors > 0 and i == total - 1:
- # Check if this specific example had a parse error
- try:
- json.loads(pred_text)
- except (json.JSONDecodeError, TypeError):
- reasons.append("json_parse_error")
-
- ex_fp_names = pred_name_set - ref_name_set
- ex_fn_names = ref_name_set - pred_name_set
- if ex_fp_names:
- reasons.append(f"wrong_tools:{','.join(sorted(ex_fp_names))}")
- if ex_fn_names:
- reasons.append(f"missing_tools:{','.join(sorted(ex_fn_names))}")
-
- if not ex_fp_names and not ex_fn_names and pred_name_set:
- # Right tools but wrong args — find which values differ
- for c in pred_calls:
- if not isinstance(c, dict) or "name" not in c:
- continue
- cname = c["name"]
- if cname not in ref_by_name:
- continue
- pred_args = c.get("arguments", {})
- ref_args = ref_by_name[cname][0]
- for k in set(pred_args.keys()) | set(ref_args.keys()):
- pv = pred_args.get(k, "<MISSING>")
- rv = ref_args.get(k, "<MISSING>")
- pv_n = _normalize_value(pv)
- rv_n = _normalize_value(rv)
- if json.dumps(pv_n, sort_keys=True) != json.dumps(rv_n, sort_keys=True):
- reasons.append(f"value_mismatch:{cname}.{k}={json.dumps(pv)[:60]}!={json.dumps(rv)[:60]}")
-
- if ref_is_empty and not pred_is_empty:
- reasons.append("false_positive:should_be_empty")
- if not ref_is_empty and pred_is_empty:
- reasons.append("false_negative:predicted_empty")
-
- if not reasons:
- reasons.append("unknown")
-
- failures.append({
- "query": ex["query"][:200],
- "ref": ref_text[:300],
- "pred": pred_text[:300],
- "reasons": reasons,
- })
-
- name_precision = tp_names / max(tp_names + fp_names, 1)
- name_recall = tp_names / max(tp_names + fn_names, 1)
- name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)
-
- call_precision = tp_calls / max(tp_calls + fp_calls, 1)
- call_recall = tp_calls / max(tp_calls + fn_calls, 1)
- call_f1 = 2 * call_precision * call_recall / max(call_precision + call_recall, 1e-9)
-
- return {
- "num_samples": total,
- "exact_match": exact_match / max(total, 1),
- "json_parse_rate": 1.0 - json_parse_errors / max(total, 1),
- "name_precision": name_precision,
- "name_recall": name_recall,
- "name_f1": name_f1,
- "args_acc": args_correct / max(args_total, 1),
- "param_haluc": halluc_params / max(total_pred_params, 1),
- "param_miss": missing_params / max(total_ref_params, 1),
- "value_acc": correct_values / max(matched_params, 1),
- "call_precision": call_precision,
- "call_recall": call_recall,
- "call_f1": call_f1,
- "empty_ref_pct": empty_ref / max(total, 1),
- "empty_pred_pct": empty_pred / max(total, 1),
- "samples": samples,
- "failures": failures,
- }
-
-
-def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256, ks=(1, 2, 3, 4, 5), ds=None):
- """Benchmark contrastive retrieval: Recall@k and MRR over validation set.
-
- For each query, ranks all tools from that example by cosine similarity
- and checks if the positive (called) tools appear in top-k.
- """
- import json
- from ..model.run import encode_for_retrieval
-
- if ds is None:
- ds = load_tool_calls("validation", max_samples=num_samples)
- elif num_samples and len(ds) > num_samples:
- ds = ds.select(range(num_samples))
-
- queries = []
- all_tool_strs = []
- tool_groups = [] # (start_idx, count, set_of_positive_indices_within_group)
-
- for ex in ds:
- query = ex["query"]
- try:
- tools = json.loads(ex["tools"])
- except (ValueError, TypeError):
- continue
- if not isinstance(tools, list) or len(tools) == 0:
- continue
- try:
- calls = json.loads(ex["answers"])
- except (ValueError, TypeError):
- calls = []
- pos_names = set()
- if isinstance(calls, list):
- for c in calls:
- if isinstance(c, dict) and "name" in c:
- pos_names.add(c["name"])
- if not pos_names:
- continue
-
- start = len(all_tool_strs)
- pos_indices = set()
- for j, tool in enumerate(tools):
- if not isinstance(tool, dict):
- continue
- all_tool_strs.append(json.dumps(tool, separators=(",", ":")))
- if tool.get("name", "") in pos_names:
- pos_indices.add(len(all_tool_strs) - 1 - start)
-
- if pos_indices:
- queries.append(query)
- tool_groups.append((start, len(all_tool_strs) - start, pos_indices))
-
- if not queries:
- return {"recall@k": {}, "mrr": 0.0, "num_queries": 0}
-
- # Encode all queries and tools
- q_embs = encode_for_retrieval(model, params, tokenizer, queries, max_len=max_len)
- t_embs = encode_for_retrieval(model, params, tokenizer, all_tool_strs, max_len=max_len)
-
- max_k = max(ks)
- recall = {k: 0 for k in ks}
- mrr_sum = 0.0
-
- for i, (start, count, pos_set) in enumerate(tool_groups):
- group_embs = t_embs[start:start + count] # (count, dim)
- scores = q_embs[i:i+1] @ group_embs.T # (1, count)
- ranked = np.argsort(-scores[0])
-
- # MRR: reciprocal rank of first positive
- for rank, idx in enumerate(ranked):
- if idx in pos_set:
- mrr_sum += 1.0 / (rank + 1)
- break
-
- # Recall@k
- for k in ks:
- top_k_set = set(ranked[:k].tolist())
- if top_k_set & pos_set:
- recall[k] += 1
-
- n = len(queries)
- return {
- "recall@k": {k: recall[k] / n for k in ks},
- "mrr": mrr_sum / n,
- "num_queries": n,
- }
-
-
-def main(args):
- import json as _json_mod
-
- params, config = load_checkpoint(args.checkpoint)
- model = SimpleAttentionNetwork(config)
- tokenizer = get_tokenizer()
-
- param_count = sum(x.size for x in jax.tree.leaves(params))
- print(f"\ncheckpoint: {args.checkpoint}")
- print(f"parameters: {param_count:,}")
- print(f"config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")
-
- # --- Perplexity from pre-packed data (no re-tokenization) ---
- print(f"\nevaluating perplexity (pre-packed val data)...")
- try:
- val_data = load_prepared_data("val", mmap=True)
- n_bins = len(val_data["packed_enc"])
- ppl = compute_perplexity_packed(model, params, val_data, args.batch_size)
- print(f" perplexity: {ppl:.2f} ({n_bins} packed bins)")
- except FileNotFoundError:
- print(" Skipped — no pre-packed val data. Run 'needle tokenize' first.")
- ppl = None
-
- # --- Tool-call accuracy with stratified sampling (matches training eval) ---
- tc_samples = getattr(args, "tool_call_samples", 200)
- use_constrained = not getattr(args, "no_constrained", False)
-
- if tc_samples > 0:
- val_ds = load_tool_calls("validation")
- rng = np.random.RandomState(42)
-
- def _classify(ex):
- try:
- answers = _json_mod.loads(ex["answers"])
- except (ValueError, TypeError):
- return "empty"
- if not answers or not isinstance(answers, list):
- return "empty"
- return "single" if len(answers) == 1 else "multi"
-
- pool_names = ["single", "multi"]
- pool_buckets = {name: {t: [] for t in range(11)} for name in pool_names}
-
- for k in range(len(val_ds)):
- ex = val_ds[k]
- try:
- nc = min(len(_json_mod.loads(ex["tools"])), 10)
- except (ValueError, TypeError):
- nc = 0
- ct = _classify(ex)
- if ct != "empty":
- pool_buckets[ct][nc].append(k)
-
- per_bucket = max(1, tc_samples // 14) # ~7 tool-count buckets * 2 pools
-
- for name in pool_names:
- label = "Single-Call" if name == "single" else "Multi-Call"
- pool = []
- for t in range(11):
- b = np.array(pool_buckets[name][t])
- if len(b) > 0:
- rng.shuffle(b)
- pool.extend(b[:per_bucket].tolist())
- rng.shuffle(pool)
- examples = [val_ds[k] for k in pool]
-
- if not examples:
- continue
-
- print(f"\n evaluating {label} ({len(examples)} samples, constrained={use_constrained})...")
- tc = benchmark_tool_calls(
- model, params, tokenizer, num_samples=len(examples),
- max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len,
- constrained=use_constrained, ds=examples,
- )
- print(f" ─── {label} ({len(examples)} samples) ───")
- print(f" JSON parse {tc['json_parse_rate']:>10.1%}")
- print(f" Name F1 {tc['name_f1']:>10.1%}")
- print(f" Param haluc {tc['param_haluc']:>10.1%}")
- print(f" Param miss {tc['param_miss']:>10.1%}")
- print(f" Value acc {tc['value_acc']:>10.1%}")
- print(f" Args acc {tc['args_acc']:>10.1%}")
- print(f" Call F1 {tc['call_f1']:>10.1%}")
- print(f" Exact match {tc['exact_match']:>10.1%}")
- if tc["samples"]:
- print(f" samples:")
- for query, ref, pred in tc["samples"][:3]:
- print(f" Q: {query}")
- print(f" R: {ref}")
- print(f" P: {pred}")
- print()
- if tc.get("failures"):
- print(f" ─── Failures ({len(tc['failures'])} captured) ───")
- for j, fail in enumerate(tc["failures"][:15]):
- print(f" [{j+1}] Q: {fail['query'][:120]}")
- print(f" Ref: {fail['ref'][:200]}")
- print(f" Pred: {fail['pred'][:200]}")
- print(f" Why: {', '.join(fail['reasons'])}")
- print()
-
- print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")
- throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)
- print(f"avg tokens: {throughput['avg_tokens_generated']:.1f}")
- print(f"avg latency: {throughput['avg_latency_s']:.3f}s")
- print(f"tokens/sec: {throughput['tokens_per_second']:.1f}")
-
-
-def parse_args():
- parser = argparse.ArgumentParser(description="Benchmark transformer on generation tasks")
- parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
- parser.add_argument("--batch-size", type=int, default=32)
- parser.add_argument("--max-eval-samples", type=int, default=1000)
- parser.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
- parser.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
- parser.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)
- parser.add_argument("--throughput-runs", type=int, default=10)
- parser.add_argument("--tool-call-samples", type=int, default=200,
- help="Samples for tool-call accuracy eval (default: 200)")
- return parser.parse_args()
-
-
-if __name__ == "__main__":
- main(parse_args())
diff --git a/needle/training/finetune.py b/needle/training/finetune.py
deleted file mode 100644
index 3f02645..0000000
--- a/needle/training/finetune.py
+++ /dev/null
@@ -1,386 +0,0 @@
-import argparse
-import glob
-import json
-import os
-import random
-import shutil
-import tempfile
-import time
-from collections import Counter
-
-from ..dataset import dataset as data_mod
-from ..dataset.dataset import _cache_key, _save_cache_metadata, get_tokenizer, pack_sequences, prepare_tool_call_pairs
-
-
-def _write_split(split, examples, tokenizer, cache_dir, max_enc_len, max_dec_len):
- from datasets import Dataset
-
- ds = Dataset.from_list(examples)
- enc_vl, dec_in_vl, dec_tgt_vl, loss_vl, kept, _ = prepare_tool_call_pairs(
- ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len, shuffle_tools=True,
- )
- # Cache key weights must match prepare_tool_call_pairs (1.0)
- cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
- 1.0, 1.0, 1.0, True)
- cache_path = os.path.join(cache_dir, cache_id)
- pack_sequences(cache_path, enc_vl, dec_in_vl, dec_tgt_vl, loss_vl)
- _save_cache_metadata(split, cache_id, len(kept), max_enc_len, max_dec_len, 256)
- return len(kept)
-
-
-def _ensure_best_checkpoint(checkpoint_dir, run_id):
- """Find or promote the best checkpoint scoped to this run's unique ID."""
- prefix = f"needle_finetuned_{run_id}_"
- best = sorted(
- p for p in glob.glob(os.path.join(checkpoint_dir, f"{prefix}*_best.pkl"))
- )
- if best:
- return best[-1]
-
- candidates = sorted(
- p for p in glob.glob(os.path.join(checkpoint_dir, f"{prefix}*.pkl"))
- if not p.endswith("_best.pkl")
- )
- if not candidates:
- return None
-
- latest = candidates[-1]
- best_name = os.path.basename(latest).rsplit("_", 1)[0] + "_best.pkl"
- best_path = os.path.join(checkpoint_dir, best_name)
- shutil.copy2(latest, best_path)
- print(f"Promoted final checkpoint to best: {best_path}")
- return best_path
-
-
-def _call_key(c):
- """Canonical key for a tool call (name + sorted arguments). Matches train.py."""
- if not isinstance(c, dict):
- return None
- return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
-
-
-def _quick_tool_eval(model, params, tokenizer, examples, max_gen_len=512, max_enc_len=1024):
- """Tool-call eval matching needle's F1 methodology (TP/FP/FN)."""
- from ..model.run import generate_batch
-
- samples = [ex for ex in examples
- if ex.get("answers", "").strip() not in ("", "[]")]
- if not samples:
- return {}
-
- _BATCH = 32
- all_preds = []
- for i in range(0, len(samples), _BATCH):
- chunk = samples[i:i + _BATCH]
- all_preds.extend(generate_batch(
- model, params, tokenizer,
- [s["query"] for s in chunk],
- [s["tools"] for s in chunk],
- max_gen_len=max_gen_len, max_enc_len=max_enc_len, constrained=True,
- ))
-
- n = len(samples)
- parse_ok, exact = 0, 0
- name_tp, name_fp, name_fn = 0, 0, 0
- call_tp, call_fp, call_fn = 0, 0, 0
- args_correct, args_total = 0, 0
- per_tool = {}
-
- for ex, pred_text in zip(samples, all_preds):
- pred_text = pred_text.strip()
- ref_text = ex["answers"].strip()
- try:
- ref_calls = json.loads(ref_text)
- except (ValueError, TypeError):
- ref_calls = []
- try:
- pred_calls = json.loads(pred_text)
- if not isinstance(pred_calls, list):
- pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
- parse_ok += 1
- except (ValueError, TypeError):
- pred_calls = []
-
- ref_keys = sorted(_call_key(c) for c in ref_calls if _call_key(c))
- pred_keys = sorted(_call_key(c) for c in pred_calls if _call_key(c))
- if ref_keys == pred_keys and len(ref_keys) == len(ref_calls) and len(pred_keys) == len(pred_calls):
- exact += 1
-
- ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
- pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}
- name_tp += len(pred_names & ref_names)
- name_fp += len(pred_names - ref_names)
- name_fn += len(ref_names - pred_names)
-
- rk = {_call_key(c) for c in ref_calls} - {None}
- pk = {_call_key(c) for c in pred_calls} - {None}
- call_tp += len(pk & rk)
- call_fp += len(pk - rk)
- call_fn += len(rk - pk)
-
- ref_by_name = {}
- for c in ref_calls:
- if isinstance(c, dict) and "name" in c:
- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))
- for c in pred_calls:
- if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:
- args_total += 1
- pred_args = json.dumps(c.get("arguments", {}), sort_keys=True)
- if any(pred_args == json.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):
- args_correct += 1
-
- for c in ref_calls:
- if not isinstance(c, dict) or "name" not in c:
- continue
- rname = c["name"]
- rargs = json.dumps(c.get("arguments", {}), sort_keys=True)
- if rname not in per_tool:
- per_tool[rname] = {"correct": 0, "total": 0}
- per_tool[rname]["total"] += 1
- for pc in pred_calls:
- if isinstance(pc, dict) and pc.get("name") == rname:
- if json.dumps(pc.get("arguments", {}), sort_keys=True) == rargs:
- per_tool[rname]["correct"] += 1
- break
-
- name_p = name_tp + name_fp
- name_r = name_tp + name_fn
- call_p = call_tp + call_fp
- call_r = call_tp + call_fn
-
- return {
- "call_f1": round(2 * call_tp / max(call_p + call_r, 1), 4),
- "name_f1": round(2 * name_tp / max(name_p + name_r, 1), 4),
- "exact_match": round(exact / max(n, 1), 4),
- "parse_rate": round(parse_ok / max(n, 1), 4),
- "args_acc": round(args_correct / max(args_total, 1), 4),
- "n": n,
- "per_tool": per_tool,
- }
-
-
-def _per_tool_split(examples, val_per_tool=10, test_per_tool=10, seed=42):
- """Split examples per tool: remainder->train, val_per_tool->val, test_per_tool->test.
-
- Each tool gets exactly val_per_tool examples in val and test_per_tool in test.
- If a tool has fewer than val_per_tool + test_per_tool + 1, the split is
- proportional (at least 1 in each when possible).
- """
- rng = random.Random(seed)
-
- # Group by primary tool (first tool in answers)
- tool_buckets = {}
- for i, ex in enumerate(examples):
- try:
- calls = json.loads(ex.get("answers", "[]"))
- except (ValueError, TypeError):
- calls = []
- names = [c["name"] for c in calls if isinstance(c, dict) and "name" in c]
- primary = names[0] if names else "__no_tool__"
- tool_buckets.setdefault(primary, []).append(i)
-
- train_idx, val_idx, test_idx = [], [], []
- for tool, indices in tool_buckets.items():
- rng.shuffle(indices)
- n = len(indices)
- needed = val_per_tool + test_per_tool
- if n < needed:
- if n == 1:
- n_test, n_val, n_train = 1, 0, 0
- elif n == 2:
- n_test, n_val, n_train = 1, 1, 0
- else:
- n_test = max(1, n // 3)
- n_val = max(1, (n - n_test) // 3)
- n_train = n - n_val - n_test
- else:
- n_test = test_per_tool
- n_val = val_per_tool
- n_train = n - n_val - n_test
-
- test_idx.extend(indices[:n_test])
- val_idx.extend(indices[n_test:n_test + n_val])
- train_idx.extend(indices[n_test + n_val:])
-
- train = [examples[i] for i in train_idx]
- val = [examples[i] for i in val_idx]
- test = [examples[i] for i in test_idx]
- return train, val, test
-
-
-def _emit(tag, data):
- print(f"{tag}:{json.dumps(data)}", flush=True)
-
-
-def _resolve_checkpoint(path):
- """Resolve checkpoint path, always downloading from HuggingFace to ensure freshness."""
- from huggingface_hub import hf_hub_download
- local_dir = "checkpoints"
- os.makedirs(local_dir, exist_ok=True)
- filename = os.path.basename(path) if path else "needle.pkl"
- print(f"Downloading {filename} from Cactus-Compute/needle...")
- return hf_hub_download(
- repo_id="Cactus-Compute/needle",
- filename=filename,
- repo_type="model",
- local_dir=local_dir,
- force_download=True,
- )
-
-
-def finetune_local(args):
- args.checkpoint = _resolve_checkpoint(args.checkpoint)
-
- with open(args.jsonl_path) as f:
- examples = [json.loads(line) for line in f if line.strip()]
-
- print(f"Loaded {len(examples)} examples from {args.jsonl_path}")
- if len(examples) < 3:
- raise ValueError("finetune requires at least 3 examples for train/val/test splits")
- tool_counts = Counter()
- for ex in examples:
- for call in json.loads(ex.get("answers", "[]")):
- if call.get("name"):
- tool_counts[call["name"]] += 1
- low_tools = {t: c for t, c in tool_counts.items() if c < 120}
- if low_tools:
- print(
- f"\n⚠ WARNING: The following tools have fewer than 120 examples "
- f"(need at least 120 per tool: 100 train / 10 val / 10 test):"
- )
- for tool, count in sorted(low_tools.items()):
- print(f" • {tool}: {count}")
- print(
- f" Finetuning with too few examples will overfit (perfect training "
- f"metrics but poor real-world performance). Add more diverse examples.\n"
- )
-
- from ..dataset.tokenizer import DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN
- max_enc_len = getattr(args, "max_enc_len", None) or DEFAULT_MAX_ENC_LEN
- max_dec_len = getattr(args, "max_dec_len", None) or DEFAULT_MAX_DEC_LEN
-
- _owns_cache = args.cache_dir is None
- cache_dir = args.cache_dir or tempfile.mkdtemp(prefix="needle-finetune-cache-")
- os.makedirs(cache_dir, exist_ok=True)
-
- train_examples, val_examples, test_examples = _per_tool_split(examples)
-
- if len(train_examples) == 0:
- all_avail = val_examples + test_examples
- random.Random(42).shuffle(all_avail)
- n = len(all_avail)
- n_test = max(1, n // 5)
- n_val = max(1, n // 5)
- n_train = n - n_val - n_test
- if n_train <= 0:
- raise ValueError(f"Not enough data ({n} examples) for train/val/test splits")
- test_examples = all_avail[:n_test]
- val_examples = all_avail[n_test:n_test + n_val]
- train_examples = all_avail[n_test + n_val:]
-
- if len(train_examples) == 0:
- raise ValueError("Not enough data — need at least 3 examples per tool for train/val/test splits")
-
- print(f"Split: {len(train_examples)} train / {len(val_examples)} val / {len(test_examples)} test (per-tool)")
-
- # Load checkpoint config
- import pickle as _pkl
- with open(args.checkpoint, "rb") as f:
- ckpt_config = _pkl.load(f)["config"]
-
- # Unique run ID for checkpoint scoping
- run_id = f"{time.strftime('%Y%m%d%H%M%S')}_{os.getpid()}"
- experiment_name = f"needle_finetuned_{run_id}"
-
- original_cache_dir = data_mod.CACHE_DIR
- data_mod.CACHE_DIR = cache_dir
- try:
- tokenizer = get_tokenizer()
- train_kept = _write_split("train", train_examples, tokenizer, cache_dir, max_enc_len, max_dec_len)
- val_kept = _write_split("val", val_examples, tokenizer, cache_dir, max_enc_len, max_dec_len)
-
- if train_kept == 0:
- raise ValueError(f"Tokenization produced 0 training sequences from {len(train_examples)} examples")
- if val_kept == 0:
- raise ValueError(f"Tokenization produced 0 validation sequences from {len(val_examples)} examples")
- print(f"Tokenized: {train_kept} train / {val_kept} val sequences")
-
- from datasets import Dataset as _Dataset
- val_ds = _Dataset.from_list(val_examples)
-
- # Base model eval on TEST set (never seen by model)
- from ..model.run import load_checkpoint
- from ..model.architecture import SimpleAttentionNetwork
- print(f"Evaluating base model on {len(test_examples)} test examples...")
- base_params, base_config = load_checkpoint(args.checkpoint)
- base_model = SimpleAttentionNetwork(base_config)
- base_metrics = _quick_tool_eval(
- base_model, base_params, tokenizer, test_examples,
- max_gen_len=min(max_dec_len, 512), max_enc_len=max_enc_len,
- )
- del base_params, base_model
- if base_metrics:
- _emit("BASE_EVAL", base_metrics)
- print(f" Base: call_f1={base_metrics['call_f1']:.1%}, "
- f"exact={base_metrics['exact_match']:.1%}")
-
- # Training (val_ds used for checkpoint selection inside train())
- # Architecture from checkpoint config
- print("Starting training...")
- approx_steps = max(1, (train_kept // args.batch_size) * args.epochs)
- cfg = ckpt_config
- train_args = argparse.Namespace(
- name=experiment_name, checkpoint=args.checkpoint, init_from=None,
- epochs=args.epochs, batch_size=args.batch_size, lr=3e-5, muon_lr=0.02,
- d_model=cfg["d_model"],
- num_heads=cfg["num_heads"],
- num_kv_heads=cfg.get("num_kv_heads", cfg["num_heads"]),
- num_layers=cfg["num_encoder_layers"],
- num_dec_layers=cfg["num_decoder_layers"],
- d_ff=cfg.get("d_ff", cfg["d_model"] * 4),
- max_enc_len=max_enc_len, max_dec_len=max_dec_len, max_samples=None,
- warmup_ratio=0.05, decay_ratio=0.05, wandb=False,
- dtype=cfg.get("dtype", "bfloat16"),
- checkpoint_dir=args.checkpoint_dir, seed=42,
- eval_every=max(1, approx_steps), max_eval_samples=min(val_kept, 50),
- contrastive_weight=0.1,
- contrastive_dim=cfg.get("contrastive_dim", 128),
- num_memory_slots=cfg.get("num_memory_slots", 64),
- w_name=2.0, w_value=4.0, w_key=1.5,
- val_ds=val_ds,
- )
-
- from ..training.train import train
- train(train_args)
- best_path = _ensure_best_checkpoint(args.checkpoint_dir, run_id)
-
- # Finetuned model eval on TEST set (same held-out set as base eval)
- if best_path and os.path.exists(best_path) and test_examples:
- print(f"Evaluating finetuned model on {len(test_examples)} test examples...")
- ft_params, ft_config = load_checkpoint(best_path)
- ft_model = SimpleAttentionNetwork(ft_config)
- ft_metrics = _quick_tool_eval(
- ft_model, ft_params, tokenizer, test_examples,
- max_gen_len=min(max_dec_len, 512), max_enc_len=max_enc_len,
- )
- del ft_params, ft_model
- if ft_metrics:
- _emit("FINETUNED_EVAL", ft_metrics)
- print(f" Finetuned: call_f1={ft_metrics['call_f1']:.1%}, "
- f"exact={ft_metrics['exact_match']:.1%}")
- finally:
- data_mod.CACHE_DIR = original_cache_dir
- if _owns_cache:
- shutil.rmtree(cache_dir, ignore_errors=True)
-
-if __name__ == "__main__":
- p = argparse.ArgumentParser()
- p.add_argument("jsonl_path")
- p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=64)
- p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
- p.add_argument("--checkpoint", type=str, default=None)
- p.add_argument("--cache-dir", type=str, default=None)
- p.add_argument("--max-enc-len", type=int, default=None)
- p.add_argument("--max-dec-len", type=int, default=None)
- finetune_local(p.parse_args())
diff --git a/needle/training/optim.py b/needle/training/optim.py
deleted file mode 100644
index be8890b..0000000
--- a/needle/training/optim.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import math
-from typing import NamedTuple
-import jax
-import jax.numpy as jnp
-import optax
-from flax.training import train_state
-from ..model.architecture import SimpleAttentionNetwork
-
-
-def _newton_schulz(G, steps=5):
- """Approximate polar decomposition via Newton-Schulz iteration."""
- a, b, c = 3.4445, -4.7750, 2.0315
- orig_dtype = G.dtype
- G = G.astype(jnp.float32)
- X = G / (jnp.linalg.norm(G) + 1e-7)
- transposed = G.shape[0] > G.shape[1]
- if transposed:
- X = X.T
- for _ in range(steps):
- A = X @ X.T
- B = b * A + c * A @ A
- X = a * X + B @ X
- if transposed:
- X = X.T
- return X.astype(orig_dtype)
-
-
-class MuonState(NamedTuple):
- mu: optax.Updates
-
-
-def scale_by_muon(momentum=0.95, ns_steps=5):
- """Muon gradient transform: orthogonalize 2D+ grads, then Nesterov momentum."""
-
- def init_fn(params):
- return MuonState(mu=jax.tree.map(jnp.zeros_like, params))
-
- def update_fn(updates, state, params=None):
- del params
-
- def ortho(g):
- if g.ndim == 3:
- return jax.vmap(_newton_schulz, in_axes=(0,))(g)
- if g.ndim == 2:
- return _newton_schulz(g, steps=ns_steps)
- return g
-
- ortho_g = jax.tree.map(ortho, updates)
- new_mu = jax.tree.map(lambda m, g: momentum * m + g, state.mu, ortho_g)
- new_updates = jax.tree.map(
- lambda g, m: g + momentum * m, ortho_g, new_mu
- )
- return new_updates, MuonState(mu=new_mu)
-
- return optax.GradientTransformation(init_fn, update_fn)
-
-
-def _param_labels(params):
- """Label each param: 'muon' for Dense kernels, 'adam' for the rest."""
-
- def _label(path, leaf):
- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel" and leaf.ndim in (2, 3):
- return "muon"
- return "adam"
-
- return jax.tree_util.tree_map_with_path(_label, params)
-
-
-def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
- """Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""
- decay_steps = max(1, int(total_steps * decay_ratio))
- stable_steps = total_steps - warmup_steps - decay_steps
- return optax.join_schedules(
- [
- optax.linear_schedule(0.0, peak_value, warmup_steps),
- optax.constant_schedule(peak_value),
- optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),
- ],
- boundaries=[warmup_steps, warmup_steps + stable_steps],
- )
-
-
-def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):
- model = SimpleAttentionNetwork(config)
-
- rng, init_rng = jax.random.split(rng)
- dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
- dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
- variables = model.init(
- {"params": init_rng},
- dummy_src, dummy_tgt,
- method="init_all",
- )
-
- adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps, decay_ratio)
- muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps, decay_ratio)
-
- muon_opt = optax.chain(
- scale_by_muon(momentum=0.95, ns_steps=5),
- optax.add_decayed_weights(weight_decay=0.01),
- optax.scale_by_schedule(muon_schedule),
- optax.scale(-1.0),
- )
- adam_opt = optax.chain(
- optax.adamw(adam_schedule, b2=0.95, weight_decay=0.0),
- )
-
- tx = optax.chain(
- optax.clip_by_global_norm(1.0),
- optax.multi_transform(
- {"muon": muon_opt, "adam": adam_opt},
- _param_labels,
- ),
- )
- return train_state.TrainState.create(
- apply_fn=model.apply,
- params=variables["params"],
- tx=tx,
- )
diff --git a/needle/training/pretrain.py b/needle/training/pretrain.py
deleted file mode 100644
index 6c1bf83..0000000
--- a/needle/training/pretrain.py
+++ /dev/null
@@ -1,413 +0,0 @@
-"""Streaming pretraining on PleIAs/SYNTH dataset.
-
-Streams 80M examples from HuggingFace, tokenizes on-the-fly, and trains
-with simple CE loss (uniform weights). Saves needle_base.pkl every N steps.
-
-Usage:
- needle pretrain --wandb
- needle pretrain --wandb --max-steps 10000
- needle tpu train large --name pretrain --wandb
-"""
-
-import math
-import os
-import pickle
-import time
-import queue
-import threading
-
-import jax
-import jax.numpy as jnp
-import numpy as np
-import optax
-from tqdm import tqdm
-
-from ..dataset.tokenizer import get_tokenizer, EOS_ID, TOOLS_ID, PAD_ID
-from ..model.architecture import (
- SimpleAttentionNetwork,
- TransformerConfig,
- make_padding_mask,
- make_causal_mask,
-)
-from .optim import create_train_state, _wsd_schedule
-from ..utils.distributed import _replicate, _unreplicate, shard_batch, _upload_checkpoint
-
-_HF_PRETRAIN_REPO = "PleIAs/SYNTH"
-
-
-def _stream_batches(tokenizer, batch_size, max_enc_len, max_dec_len, seed=42):
- """Stream batches from PleIAs/SYNTH, tokenizing on-the-fly.
-
- Encoder: [query_tokens, <tools>, seed_text_tokens] truncated to max_enc_len
- Decoder input: [EOS, answer_tokens] truncated to max_dec_len
- Decoder target: [answer_tokens, EOS] truncated to max_dec_len
-
- Yields (enc, dec_in, dec_tgt) as numpy int32 arrays of shape (batch_size, seq_len).
- """
- from datasets import load_dataset
-
- ds = load_dataset(_HF_PRETRAIN_REPO, split="train", streaming=True)
- # Small shuffle buffer so first batch yields quickly on resume.
- # Dataset is already split across 500 shards so it's already partially shuffled.
- ds = ds.shuffle(seed=seed, buffer_size=1000)
-
- tools_sep_id = TOOLS_ID
- eos_id = EOS_ID
- pad_id = PAD_ID
-
- enc_batch = np.full((batch_size, max_enc_len), pad_id, dtype=np.int32)
- dec_in_batch = np.full((batch_size, max_dec_len), pad_id, dtype=np.int32)
- dec_tgt_batch = np.full((batch_size, max_dec_len), pad_id, dtype=np.int32)
- idx = 0
-
- for example in ds:
- query = example.get("query") or ""
- seed_text = example.get("query_seed_text") or ""
- answer = example.get("synthetic_answer") or ""
-
- if not query.strip() or not answer.strip():
- continue
-
- # Tokenize
- q_toks = tokenizer.encode(query)
- s_toks = tokenizer.encode(seed_text) if seed_text.strip() else []
- a_toks = tokenizer.encode(answer)
-
- # Build encoder input: [query, <tools>, seed_text]
- max_query = max_enc_len - 2
- if len(q_toks) > max_query:
- q_toks = q_toks[:max_query]
- remaining = max_enc_len - len(q_toks) - 1
- s_toks = s_toks[:remaining]
- enc_tokens = q_toks + [tools_sep_id] + s_toks
-
- # Build decoder: input=[EOS, answer], target=[answer, EOS]
- max_ans = max_dec_len - 1
- a_toks = a_toks[:max_ans]
- if len(a_toks) == 0:
- continue
- dec_in_tokens = [eos_id] + a_toks
- dec_tgt_tokens = a_toks + [eos_id]
-
- # Fill batch
- enc_batch[idx, :len(enc_tokens)] = enc_tokens
- dec_in_batch[idx, :len(dec_in_tokens)] = dec_in_tokens
- dec_tgt_batch[idx, :len(dec_tgt_tokens)] = dec_tgt_tokens
- idx += 1
-
- if idx == batch_size:
- yield enc_batch.copy(), dec_in_batch.copy(), dec_tgt_batch.copy()
- enc_batch[:] = pad_id
- dec_in_batch[:] = pad_id
- dec_tgt_batch[:] = pad_id
- idx = 0
-
-
-class _PrefetchStream:
- """Prefetch streaming batches in a background thread."""
-
- def __init__(self, generator_fn, prefetch=4):
- self._queue = queue.Queue(maxsize=prefetch)
- self._stop = threading.Event()
- self._thread = threading.Thread(target=self._produce, args=(generator_fn,), daemon=True)
- self._thread.start()
-
- def _produce(self, gen_fn):
- try:
- for batch in gen_fn():
- if self._stop.is_set():
- return
- self._queue.put(batch)
- self._queue.put(None)
- except Exception as e:
- self._queue.put(e)
-
- def __iter__(self):
- return self
-
- def __next__(self):
- item = self._queue.get()
- if item is None:
- raise StopIteration
- if isinstance(item, Exception):
- raise item
- return item
-
- def close(self):
- self._stop.set()
-
-
-def _pretrain_step(state, src, tgt_in, tgt_out, rng):
- """Simple CE loss train step for pretraining. No QAT, no contrastive, no pruning."""
-
- def loss_fn(params):
- src_mask = make_padding_mask(src, PAD_ID)
- tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, PAD_ID)
- cross_mask = src_mask
-
- logits = state.apply_fn(
- {"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- )
- logits_f32 = logits.astype(jnp.float32)
-
- padding_mask = (tgt_out != PAD_ID).astype(jnp.float32)
- num_tokens = jnp.maximum(jnp.sum(padding_mask), 1.0)
- ce = jnp.sum(
- optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * padding_mask
- ) / num_tokens
- z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- return ce + z_loss
-
- loss, grads = jax.value_and_grad(loss_fn)(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- loss = jax.lax.pmean(loss, axis_name="batch")
- state = state.apply_gradients(grads=grads)
- return state, loss
-
-
-_p_pretrain_step = None
-
-
-def _make_p_pretrain_step():
- return jax.pmap(_pretrain_step, axis_name="batch", donate_argnums=(0,))
-
-
-def pretrain(args):
- num_devices = jax.local_device_count()
- num_hosts = jax.process_count()
- host_id = jax.process_index()
- total_devices = jax.device_count()
- is_main = host_id == 0
-
- experiment_name = getattr(args, "name", "pretrain")
- use_wandb = getattr(args, "wandb", False)
- if use_wandb and is_main:
- import wandb
- if wandb.run is None:
- wandb.init(project="needle-v1", name=experiment_name, config=vars(args))
-
- print(f"\n[1/3] Detecting devices...")
- print(f" {num_devices} local, {total_devices} total across {num_hosts} hosts")
-
- print(f"\n[2/3] Loading tokenizer...")
- tokenizer = get_tokenizer()
-
- # If resuming, download checkpoint from HF if not local, then load config
- resume_checkpoint_arg = getattr(args, "checkpoint", None)
- _preloaded_ckpt = None
- if resume_checkpoint_arg:
- if not os.path.exists(resume_checkpoint_arg):
- print(f" Checkpoint not found locally, downloading from HF...", flush=True)
- try:
- from huggingface_hub import hf_hub_download
- local_dir = os.path.dirname(resume_checkpoint_arg) or "checkpoints"
- os.makedirs(local_dir, exist_ok=True)
- local_path = hf_hub_download(
- repo_id="Cactus-Compute/checkpoints",
- filename=os.path.basename(resume_checkpoint_arg),
- repo_type="model",
- local_dir=local_dir,
- )
- resume_checkpoint_arg = local_path
- print(f" Downloaded to {local_path}", flush=True)
- except Exception as e:
- print(f" ERROR downloading checkpoint: {e}", flush=True)
- return
-
- print(f" Pre-loading config from {resume_checkpoint_arg}", flush=True)
- with open(resume_checkpoint_arg, "rb") as f:
- _preloaded_ckpt = pickle.load(f)
- config = TransformerConfig(**_preloaded_ckpt["config"])
- print(f" Config from ckpt: d={config.d_model}, "
- f"heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}",
- flush=True)
- else:
- config = TransformerConfig(
- d_model=args.d_model,
- num_heads=args.num_heads,
- num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,
- num_encoder_layers=args.num_layers,
- num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),
- d_ff=getattr(args, "d_ff", None) or args.d_model * 4,
- max_seq_len=max(args.max_enc_len, args.max_dec_len),
- dtype=args.dtype,
- activation=getattr(args, "activation", "swiglu"),
- no_feedforward=getattr(args, "no_feedforward", True),
- contrastive_dim=getattr(args, "contrastive_dim", 128),
- )
-
- effective_batch_size = args.batch_size * num_devices
- global_batch_size = effective_batch_size * num_hosts
-
- # Estimate total steps: 80M rows / global_batch_size
- estimated_rows = 80_000_000
- max_steps = getattr(args, "max_steps", None)
- estimated_steps = estimated_rows // global_batch_size
- if max_steps:
- estimated_steps = min(estimated_steps, max_steps)
- total_steps = estimated_steps * args.epochs
- warmup_steps = max(1, int(total_steps * args.warmup_ratio))
-
- scaled_lr = args.lr * total_devices
- muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(total_devices)
- decay_ratio = getattr(args, "decay_ratio", 0.05)
-
- resume_step = 0
- resume_checkpoint = resume_checkpoint_arg
- ckpt_data = _preloaded_ckpt
-
- print(f"\n[3/3] Initializing model...")
- rng = jax.random.PRNGKey(args.seed)
- rng, init_rng = jax.random.split(rng)
- state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)
-
- if resume_checkpoint and ckpt_data is not None:
- print(f" Resuming from {resume_checkpoint}", flush=True)
- ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])
- state = state.replace(params=ckpt_params)
- del ckpt_params
- print(f" Loaded checkpoint params", flush=True)
- # Use --resume-step override, else read from checkpoint, else 0
- if ckpt_data is not None:
- manual_step = getattr(args, "resume_step", None)
- if manual_step is not None:
- resume_step = manual_step
- else:
- resume_step = ckpt_data.get("pretrain_step", 0)
- if resume_step > 0:
- print(f" Resuming from step {resume_step}", flush=True)
-
- print(f" Replicating state across {num_devices} local devices...", flush=True)
- state = _replicate(state)
- print(f" Replicated.", flush=True)
-
- param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))
- print(f" Params: {param_count:,}", flush=True)
-
- global _p_pretrain_step
- _p_pretrain_step = _make_p_pretrain_step()
- print(f" pmap ready.", flush=True)
-
- save_every = getattr(args, "save_every", 1000)
- checkpoint_dir = getattr(args, "checkpoint_dir", "checkpoints")
- os.makedirs(checkpoint_dir, exist_ok=True)
-
- if is_main:
- decay_steps = max(1, int(total_steps * decay_ratio))
- stable_steps = total_steps - warmup_steps - decay_steps
- print(f"\n ─────────────────────────────────────")
- print(f" Pretraining on PleIAs/SYNTH")
- print(f" ─────────────────────────────────────")
- print(f" Parameters {param_count:>12,}")
- print(f" d_model {config.d_model:>12}")
- print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")
- print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")
- print(f" Dtype {config.dtype:>12}")
- print(f" ─────────────────────────────────────")
- print(f" Hosts {num_hosts:>12}")
- print(f" Devices {num_devices:>5}/host, {total_devices} total")
- print(f" Batch {args.batch_size:>7} x {total_devices} = {global_batch_size}")
- print(f" Adam LR {args.lr:>7} x {total_devices} = {scaled_lr}")
- print(f" Muon LR {getattr(args, 'muon_lr', 0.02):>7.4f} -> {muon_lr:.4f}")
- print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")
- print(f" Est. steps {estimated_steps:>12,}")
- print(f" Save every {save_every:>12,}")
- print(f" ─────────────────────────────────────\n")
-
- global_step = resume_step
-
- for epoch in range(args.epochs):
- # Use a different seed on resume to get fresh data ordering
- # (avoids re-seeing exactly the same examples we saw before the crash)
- stream_seed = args.seed + epoch + resume_step
- batch_stream = _PrefetchStream(
- lambda: _stream_batches(tokenizer, global_batch_size,
- args.max_enc_len, args.max_dec_len,
- seed=stream_seed),
- prefetch=8,
- )
-
- pbar = tqdm(desc=f"Pretrain epoch {epoch + 1}/{args.epochs}",
- total=estimated_steps, initial=resume_step,
- disable=not is_main)
-
- for batch in batch_stream:
- if max_steps and global_step >= max_steps:
- break
-
- src, tgt_in, tgt_out = batch
- t0 = time.perf_counter()
-
- # Slice this host's portion
- host_slice = slice(host_id * effective_batch_size,
- (host_id + 1) * effective_batch_size)
- src = src[host_slice]
- tgt_in = tgt_in[host_slice]
- tgt_out = tgt_out[host_slice]
-
- # Shard for pmap
- src_b = shard_batch(src, num_devices)
- tgt_in_b = shard_batch(tgt_in, num_devices)
- tgt_out_b = shard_batch(tgt_out, num_devices)
-
- rng, step_rng = jax.random.split(rng)
- step_rngs = jax.random.split(step_rng, num_devices)
-
- state, loss = _p_pretrain_step(state, src_b, tgt_in_b, tgt_out_b, step_rngs)
-
- loss_val = float(loss.addressable_shards[0].data[0])
- ppl = math.exp(min(loss_val, 20))
- dt = time.perf_counter() - t0
- global_step += 1
-
- pbar.update(1)
- pbar.set_postfix(loss=f"{loss_val:.4f}", ppl=f"{ppl:.2f}",
- tok_s=f"{global_batch_size * (args.max_enc_len + args.max_dec_len) / dt:.0f}")
-
- if use_wandb and is_main:
- import wandb
- wandb.log({
- "pretrain/loss": loss_val,
- "pretrain/ppl": ppl,
- "pretrain/step": global_step,
- "pretrain/tokens_per_sec": global_batch_size * (args.max_enc_len + args.max_dec_len) / dt,
- })
-
- # Save checkpoint every N steps
- if is_main and global_step % save_every == 0:
- _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step)
-
- batch_stream.close()
- pbar.close()
-
- # Final save
- if is_main:
- ckpt_path = _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step)
- print(f"\nPretraining complete. {global_step} steps.")
- print(f"Base checkpoint: {ckpt_path}")
-
- if use_wandb and is_main:
- import wandb
- wandb.finish()
-
- # Sync all hosts before exit
- jax.experimental.multihost_utils.sync_global_devices("pretrain_done")
-
-
-def _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step):
- """Save and upload needle_base.pkl."""
- params = _unreplicate(state).params
- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)
-
- ckpt_path = os.path.join(checkpoint_dir, "needle_base.pkl")
- with open(ckpt_path, "wb") as f:
- pickle.dump({"params": params_np, "config": config.__dict__,
- "pretrain_step": global_step}, f)
-
- param_count = sum(x.size for x in jax.tree.leaves(params_np))
- size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6
- print(f"\n [step {global_step}] Saved {ckpt_path} ({param_count:,} params, {size_mb:.1f} MB)")
-
- _upload_checkpoint(ckpt_path)
- return ckpt_path
diff --git a/needle/training/train.py b/needle/training/train.py
deleted file mode 100644
index 657559f..0000000
--- a/needle/training/train.py
+++ /dev/null
@@ -1,947 +0,0 @@
-import math
-import os
-import pickle
-import time
-
-import jax
-import jax.numpy as jnp
-import numpy as np
-import optax
-from tqdm import tqdm
-
-from ..dataset.dataset import (
- get_batches, get_tokenizer,
- load_prepared_data, load_tool_calls,
- PrefetchIterator, count_batches,
- get_contrastive_batches,
-)
-from ..model.architecture import (
- SimpleAttentionNetwork,
- TransformerConfig,
- make_packing_mask,
- make_causal_packing_mask,
- make_cross_packing_mask,
-)
-from .optim import create_train_state, _wsd_schedule
-from ..utils.distributed import (
- _replicate, _unreplicate, shard_batch, _upload_checkpoint,
- load_pretrained_params, partial_load_params,
-)
-from ..model.quantize import _quantize_params
-
-
-_PRECISION = "int4"
-_CONTRASTIVE_WEIGHT = 0.1
-# Weight map: class 0=base, 1=name, 2=value, 3=key (set from CLI args at train time)
-# Use numpy at module level to avoid initializing JAX backend before jax.distributed.initialize()
-_LOSS_WEIGHT_MAP = np.array([1.0, 3.0, 2.0, 1.5], dtype=np.float32)
-
-
-def _clip_contrastive_loss(q_emb, t_emb, log_temp):
- """CLIP-style symmetric contrastive loss with learnable temperature."""
- temp = jnp.exp(jnp.clip(log_temp, -jnp.log(100.0), jnp.log(100.0)))
- logits = q_emb @ t_emb.T / temp # (B, B)
- B = logits.shape[0]
- labels = jnp.arange(B)
- loss_q = optax.softmax_cross_entropy_with_integer_labels(logits, labels)
- loss_t = optax.softmax_cross_entropy_with_integer_labels(logits.T, labels)
- return (jnp.mean(loss_q) + jnp.mean(loss_t)) / 2.0
-
-
-def _text_loss_fn(state, params, src, tgt_in, tgt_out, rng, loss_mask, do_quantize,
- enc_seg_ids, dec_seg_ids):
- q_params = jax.lax.cond(
- do_quantize,
- lambda p: _quantize_params(p, precision=_PRECISION),
- lambda p: p,
- params,
- )
- src_mask = make_packing_mask(enc_seg_ids)
- tgt_mask = make_causal_packing_mask(dec_seg_ids)
- cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
- logits, slot_div = state.apply_fn(
- {"params": q_params},
- src, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,
- cross_mask=cross_mask,
- deterministic=False,
- method="forward_masked",
- rngs={"dropout": rng},
- )
- logits_f32 = logits.astype(jnp.float32)
- # loss_mask contains class labels (0=base,1=name,2=value,3=key) from packed data;
- # non-padding positions have seg_id > 0, padding has class 0 and seg_id 0.
- # Convert to weights via lookup, then zero out padding using dec_seg_ids.
- token_weights = _LOSS_WEIGHT_MAP[loss_mask.astype(jnp.int32)]
- padding_mask = (dec_seg_ids > 0).astype(jnp.float32)
- mask = token_weights * padding_mask
- num_tokens = jnp.maximum(jnp.sum(padding_mask), 1.0)
- ce_loss = jnp.sum(
- optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
- ) / num_tokens
- z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- return ce_loss + z_loss
-
-
-def _contrastive_loss_fn(state, params, query_tokens, tool_tokens, rng, do_quantize):
- """Compute CLIP contrastive loss on query/tool pairs."""
- q_params = jax.lax.cond(
- do_quantize,
- lambda p: _quantize_params(p, precision=_PRECISION),
- lambda p: p,
- params,
- )
- q_emb, t_emb, log_temp = state.apply_fn(
- {"params": q_params},
- query_tokens, tool_tokens,
- deterministic=False,
- method="forward_contrastive",
- rngs={"dropout": rng},
- )
- return _clip_contrastive_loss(q_emb, t_emb, log_temp)
-
-
-
-def _train_step(state, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids, rng, loss_mask, query_tokens, tool_tokens, cl_rng, do_quantize, do_contrastive):
- """Unified train step with boolean flag for contrastive loss."""
-
- def combined_loss(p):
- text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, rng, loss_mask, do_quantize,
- enc_seg_ids, dec_seg_ids)
- cl_loss = jax.lax.cond(
- do_contrastive,
- lambda: _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng, do_quantize),
- lambda: 0.0,
- )
- return text_loss + _CONTRASTIVE_WEIGHT * cl_loss, text_loss
-
- (loss, text_loss), grads = jax.value_and_grad(combined_loss, has_aux=True)(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- text_loss = jax.lax.pmean(text_loss, axis_name="batch")
- grad_norm = optax.global_norm(grads)
- state = state.apply_gradients(grads=grads)
- return state, text_loss, grad_norm
-
-
-def _make_p_train_step():
- return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0,))
-
-
-def _make_val_loss_fn(apply_fn):
- @jax.jit
- def val_loss_batch(params, src, tgt_in, tgt_out, loss_mask, enc_seg_ids, dec_seg_ids):
- src_mask = make_packing_mask(enc_seg_ids)
- tgt_mask = make_causal_packing_mask(dec_seg_ids)
- cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
- logits = apply_fn(
- {"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- )
- loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
- # Val uses uniform weights for consistent PPL — just mask padding via seg_ids
- padding_mask = (dec_seg_ids > 0).astype(jnp.float32)
- return jnp.sum(loss * padding_mask), jnp.sum(padding_mask)
- return val_loss_batch
-
-
-def train(args):
- num_devices = jax.local_device_count()
-
- experiment_name = getattr(args, "name", "baseline")
- use_wandb = getattr(args, "wandb", False)
- if use_wandb and jax.process_index() == 0:
- import wandb
- if wandb.run is None:
- wandb.init(project="needle-v1", name=experiment_name, config=vars(args))
-
- print(f"\n[1/3] Detecting devices...")
- print(f" {num_devices} local device(s), {jax.device_count()} total across {jax.process_count()} hosts")
-
- print(f"\n[2/3] Loading tokenizer...")
- tokenizer = get_tokenizer(max_samples=args.max_samples)
-
- print(f"\n[3/3] Loading prepared data from disk (mmap)...")
- train_data = load_prepared_data("train", mmap=True)
- val_data = load_prepared_data("val", mmap=True)
-
- enc_inputs = train_data["packed_enc"]
- dec_inputs = train_data["packed_dec_in"]
- dec_targets = train_data["packed_dec_tgt"]
- train_loss_mask = train_data["packed_loss"]
- train_enc_seg = train_data["packed_enc_seg"]
- train_dec_seg = train_data["packed_dec_seg"]
-
- val_enc = val_data["packed_enc"]
- val_dec_in = val_data["packed_dec_in"]
- val_dec_tgt = val_data["packed_dec_tgt"]
- val_loss_mask = val_data["packed_loss"]
- val_enc_seg = val_data["packed_enc_seg"]
- val_dec_seg = val_data["packed_dec_seg"]
- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val packed bins (memory-mapped)")
-
- val_ds = getattr(args, "val_ds", None)
- if val_ds is None:
- try:
- val_ds = load_tool_calls("val", max_samples=args.max_samples)
- except (FileNotFoundError, Exception) as e:
- print(f" WARNING: could not load val dataset for eval samples: {e}")
- val_ds = None
-
- cl_query_tokens = train_data.get("query_only")
- cl_tool_tokens = train_data.get("tool_individual")
- cl_tool_ex_idx = train_data.get("tool_ex_idx")
- cl_tool_is_pos = train_data.get("tool_is_pos")
- has_contrastive = all(x is not None for x in [cl_query_tokens, cl_tool_tokens, cl_tool_ex_idx, cl_tool_is_pos])
- if has_contrastive:
- print(f" Contrastive: {len(cl_query_tokens):,} queries, {len(cl_tool_tokens):,} tools")
-
- num_hosts = jax.process_count()
- host_id = jax.process_index()
- total_devices = jax.device_count()
- effective_batch_size = args.batch_size * num_devices # per-host batch for pmap
-
- resume_checkpoint = getattr(args, "checkpoint", None)
- if resume_checkpoint:
- if not os.path.exists(resume_checkpoint):
- print(f" Checkpoint not found locally, downloading from HF...", flush=True)
- from huggingface_hub import hf_hub_download
- local_dir = os.path.dirname(resume_checkpoint) or "checkpoints"
- os.makedirs(local_dir, exist_ok=True)
- resume_checkpoint = hf_hub_download(
- repo_id="Cactus-Compute/checkpoints",
- filename=os.path.basename(resume_checkpoint),
- repo_type="model",
- local_dir=local_dir,
- )
- print(f" Downloaded to {resume_checkpoint}", flush=True)
- print(f"Resuming from checkpoint: {resume_checkpoint}")
- with open(resume_checkpoint, "rb") as f:
- ckpt_data = pickle.load(f)
- config = TransformerConfig(**ckpt_data["config"])
- ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])
- print(f" Config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")
- else:
- config = TransformerConfig(
- d_model=args.d_model,
- num_heads=args.num_heads,
- num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,
- num_encoder_layers=args.num_layers,
- num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),
- d_ff=getattr(args, "d_ff", None) or args.d_model * 4,
- max_seq_len=max(args.max_enc_len, args.max_dec_len),
- dtype=args.dtype,
- num_memory_slots=getattr(args, "num_memory_slots", 64),
- contrastive_dim=getattr(args, "contrastive_dim", 128),
- )
-
- global _PRECISION, _CONTRASTIVE_WEIGHT, _LOSS_WEIGHT_MAP
- _PRECISION = getattr(args, "precision", "int4")
- _CONTRASTIVE_WEIGHT = getattr(args, "contrastive_weight", 0.1)
- _LOSS_WEIGHT_MAP = jnp.array([
- 1.0, # 0: base (boilerplate)
- getattr(args, "w_name", 2.0), # 1: tool name
- getattr(args, "w_value", 4.0), # 2: argument value
- getattr(args, "w_key", 1.5), # 3: argument key
- ], dtype=jnp.float32)
- p_train_step = _make_p_train_step()
-
- np.random.seed(args.seed)
- rng = jax.random.PRNGKey(args.seed)
- rng, init_rng = jax.random.split(rng)
-
- unique_batch_size = (effective_batch_size // num_devices) * num_devices
- # Each host processes unique_batch_size samples; total across hosts = unique_batch_size * num_hosts
- global_batch_size = unique_batch_size * num_hosts
- text_batches_per_epoch = count_batches(len(enc_inputs), global_batch_size)
- num_batches = text_batches_per_epoch
- total_steps = num_batches * args.epochs
- warmup_steps = max(1, int(total_steps * args.warmup_ratio))
-
- scaled_lr = args.lr * total_devices
- muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(total_devices)
- decay_ratio = getattr(args, "decay_ratio", 0.15)
- state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)
- val_loss_fn = _make_val_loss_fn(state.apply_fn)
-
- if resume_checkpoint:
- # Cast each loaded leaf to the dtype of the freshly-initialized param
- # (most weights -> bf16, scalars like log_temp stay fp32)
- ckpt_params = jax.tree.map(
- lambda ref, loaded: jnp.asarray(loaded, dtype=ref.dtype),
- state.params, ckpt_params,
- )
- state = state.replace(params=ckpt_params)
- print(f" Loaded checkpoint params into train state")
-
- init_from = getattr(args, "init_from", None)
- if init_from and not resume_checkpoint:
- print(f"\n Initializing from pretrained base: {init_from}")
- loaded_params, _loaded_cfg, local_path = load_pretrained_params(init_from)
- merged, stats = partial_load_params(state.params, loaded_params)
- state = state.replace(params=merged)
- total = stats["loaded"] + stats["random_init"]
- print(f" [init-from] {stats['loaded']}/{total} leaves loaded from {os.path.basename(local_path)}, "
- f"{stats['random_init']} kept at random init")
- if stats["shape_mismatches"]:
- print(f" [init-from] {len(stats['shape_mismatches'])} shape mismatch(es):")
- for path, loaded_shape, init_shape in stats["shape_mismatches"][:8]:
- print(f" {'/'.join(path)}: ckpt={loaded_shape} vs init={init_shape}")
- if len(stats["shape_mismatches"]) > 8:
- print(f" ... ({len(stats['shape_mismatches']) - 8} more)")
- if stats["extra_in_ckpt"]:
- print(f" [init-from] {len(stats['extra_in_ckpt'])} extra leaf(s) in ckpt (ignored)")
- del loaded_params, merged
-
- state = _replicate(state)
-
- param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))
- decay_steps = max(1, int(total_steps * decay_ratio))
- stable_steps = total_steps - warmup_steps - decay_steps
- best_call_f1 = 0.0
- best_ckpt_path = None
-
- is_main = host_id == 0
- if is_main:
- print(f"\n ─────────────────────────────────────")
- print(f" Parameters {param_count:>12,}")
- print(f" d_model {config.d_model:>12}")
- print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")
- print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")
- print(f" Dtype {config.dtype:>12}")
- print(f" ─────────────────────────────────────")
- print(f" Hosts {num_hosts:>12}")
- print(f" Devices {num_devices:>5}/host, {total_devices} total")
- print(f" Batch {args.batch_size:>7} x {total_devices} = {global_batch_size}")
- print(f" Adam LR {args.lr:>7} x {total_devices} = {scaled_lr}")
- print(f" Muon LR {args.muon_lr:>7.4f} -> {muon_lr:.4f}")
- print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")
- print(f" Total steps {total_steps:>12,}")
- print(f" Epochs {args.epochs:>12}")
- print(f" ─────────────────────────────────────\n")
-
- os.makedirs(args.checkpoint_dir, exist_ok=True)
- global_step = 0
-
- adam_schedule = _wsd_schedule(scaled_lr, total_steps, warmup_steps)
- muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)
- tokens_per_batch = global_batch_size * (args.max_enc_len + args.max_dec_len)
-
- eval_model = SimpleAttentionNetwork(config)
-
- last_val_ppl = None
- # Dummy contrastive arrays — used when contrastive is inactive
- dummy_cl_tokens = jnp.zeros((num_devices, args.batch_size, 128), dtype=jnp.int32)
- dummy_cl_rng = jax.random.split(jax.random.PRNGKey(0), num_devices)
-
- for epoch in range(args.epochs):
- text_losses = []
- text_batch_iter = PrefetchIterator(
- lambda: get_batches(enc_inputs, dec_inputs, dec_targets, global_batch_size,
- loss_mask=train_loss_mask,
- enc_seg_ids=train_enc_seg,
- dec_seg_ids=train_dec_seg),
- prefetch=4,
- )
-
- # Contrastive batch iterator (cycles through contrastive data alongside text)
- cl_batch_iter = None
- if has_contrastive and _CONTRASTIVE_WEIGHT > 0:
- cl_batch_iter = PrefetchIterator(
- lambda: get_contrastive_batches(
- cl_query_tokens, cl_tool_tokens, cl_tool_ex_idx, cl_tool_is_pos,
- global_batch_size),
- prefetch=4,
- )
-
- pbar = tqdm(range(text_batches_per_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")
-
- for step_i in pbar:
- t0 = time.perf_counter()
-
- batch = next(text_batch_iter)
-
- # Slice this host's portion from the global batch
- host_slice = slice(host_id * unique_batch_size, (host_id + 1) * unique_batch_size)
- src, tgt_in, tgt_out, lm, enc_seg, dec_seg = [b[host_slice] for b in batch]
-
- # Upcast from int16/float16 storage to int32/float32 for JAX
- src = np.asarray(src, dtype=np.int32)
- tgt_in = np.asarray(tgt_in, dtype=np.int32)
- tgt_out = np.asarray(tgt_out, dtype=np.int32)
- lm = np.asarray(lm, dtype=np.int32) # class labels (0=base,1=name,2=value,3=key)
- enc_seg = np.asarray(enc_seg, dtype=np.int32)
- dec_seg = np.asarray(dec_seg, dtype=np.int32)
-
- src_b = shard_batch(src, num_devices)
- tgt_in_b = shard_batch(tgt_in, num_devices)
- tgt_out_b = shard_batch(tgt_out, num_devices)
- lm_b = shard_batch(lm, num_devices)
- enc_seg_b = shard_batch(enc_seg, num_devices)
- dec_seg_b = shard_batch(dec_seg, num_devices)
-
- rng, text_rng = jax.random.split(rng)
- text_rngs = jax.random.split(text_rng, num_devices)
-
- do_qat = jnp.broadcast_to(jnp.array(global_step > 0 and global_step % 100 == 0), (num_devices,))
-
- do_contrastive = cl_batch_iter is not None and global_step > 0 and global_step % 1000 == 0
- cl_q_b, cl_t_b, cl_rngs = dummy_cl_tokens, dummy_cl_tokens, dummy_cl_rng
- if do_contrastive:
- try:
- cl_q, cl_t = next(cl_batch_iter)
- cl_q = cl_q[host_slice]
- cl_t = cl_t[host_slice]
- cl_q_b = shard_batch(cl_q, num_devices)
- cl_t_b = shard_batch(cl_t, num_devices)
- rng, cl_rng = jax.random.split(rng)
- cl_rngs = jax.random.split(cl_rng, num_devices)
- except StopIteration:
- cl_batch_iter = None
- do_contrastive = False
-
- do_cl = jnp.broadcast_to(jnp.array(do_contrastive), (num_devices,))
-
- state, loss, grad_norm = p_train_step(
- state, src_b, tgt_in_b, tgt_out_b, enc_seg_b, dec_seg_b,
- text_rngs, lm_b,
- cl_q_b, cl_t_b, cl_rngs, do_qat, do_cl,
- )
-
- text_loss_val = float(loss.addressable_shards[0].data[0])
- text_losses.append(text_loss_val)
- step_grad_norm = float(grad_norm.addressable_shards[0].data[0])
- global_step += 1
-
- dt = time.perf_counter() - t0
- eval_every = getattr(args, "eval_every", 100)
- if is_main and (global_step % eval_every == 0 or global_step == total_steps):
- _eval_params = _unreplicate(state).params
- total_loss, total_toks = 0.0, 0.0
- _val_n = count_batches(len(val_enc), args.batch_size)
- for vb in tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False,
- loss_mask=val_loss_mask, enc_seg_ids=val_enc_seg, dec_seg_ids=val_dec_seg),
- total=_val_n, desc=" val loss", leave=False):
- src_v, di_v, dt_v, lm_v, es_v, ds_v = vb
- src_v = jnp.asarray(src_v, dtype=jnp.int32)
- di_v = jnp.asarray(di_v, dtype=jnp.int32)
- dt_v = jnp.asarray(dt_v, dtype=jnp.int32)
- lm_v = jnp.asarray(lm_v, dtype=jnp.float32)
- es_v = jnp.asarray(es_v, dtype=jnp.int32)
- ds_v = jnp.asarray(ds_v, dtype=jnp.int32)
- vl, vt = val_loss_fn(_eval_params, src_v, di_v, dt_v, lm_v, es_v, ds_v)
- total_loss += float(vl)
- total_toks += float(vt)
- last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))
-
- del _eval_params
-
- postfix = {
- "text_loss": f"{text_loss_val:.4f}",
- "text_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?",
- }
- pbar.set_postfix(**postfix)
-
- if use_wandb and is_main:
- log_dict = {
- "train/text_loss": text_loss_val,
- "train/grad_norm": step_grad_norm,
- "train/adam_lr": float(adam_schedule(global_step)),
- "train/muon_lr": float(muon_schedule(global_step)),
- "train/tokens_per_sec": tokens_per_batch / dt,
- "train/step": global_step,
- }
- if global_step % eval_every == 0 or global_step == total_steps:
- log_dict["val/text_ppl"] = last_val_ppl
- wandb.log(log_dict)
-
- text_batch_iter.close()
- if cl_batch_iter is not None:
- cl_batch_iter.close()
-
- epoch_avg_loss = sum(text_losses) / len(text_losses) if text_losses else float("nan")
- final_loss = text_losses[-1] if text_losses else float("nan")
- final_ppl = math.exp(min(final_loss, 20)) if not math.isnan(final_loss) else float("nan")
-
- # ── All hosts: unreplicate params for eval ──
- eval_params = _unreplicate(state).params
-
- # ── Host 0 only: val PPL + checkpoint save (fast) ──
- quant_val_ppl = 0.0
- total_params = 0
- ckpt_path = ""
-
- if is_main:
- q_params = _quantize_params(eval_params, precision=_PRECISION)
-
- full_loss, full_toks = 0.0, 0.0
- q_loss, q_toks = 0.0, 0.0
-
- _val_n = count_batches(len(val_enc), args.batch_size)
- _val_bar = tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size,
- shuffle=False, loss_mask=val_loss_mask,
- enc_seg_ids=val_enc_seg, dec_seg_ids=val_dec_seg),
- total=_val_n, desc=" eval val loss", leave=False)
- for vb in _val_bar:
- src, dec_in, dec_tgt, lm, es, ds = vb
- src = jnp.asarray(src, dtype=jnp.int32)
- dec_in = jnp.asarray(dec_in, dtype=jnp.int32)
- dec_tgt = jnp.asarray(dec_tgt, dtype=jnp.int32)
- lm = jnp.asarray(lm, dtype=jnp.float32)
- es = jnp.asarray(es, dtype=jnp.int32)
- ds = jnp.asarray(ds, dtype=jnp.int32)
- vl, vt = val_loss_fn(eval_params, src, dec_in, dec_tgt, lm, es, ds)
- full_loss += float(vl); full_toks += float(vt)
- vl, vt = val_loss_fn(q_params, src, dec_in, dec_tgt, lm, es, ds)
- q_loss += float(vl); q_toks += float(vt)
- last_val_ppl = float(math.exp(min(full_loss / max(full_toks, 1), 20)))
- quant_val_ppl = float(math.exp(min(q_loss / max(q_toks, 1), 20)))
- del q_params
-
- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), eval_params)
- total_params = sum(x.size for x in jax.tree.leaves(params_np))
-
- ckpt_name = f"{experiment_name}_{args.num_layers}_{args.d_model}_{global_step}.pkl"
- ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)
- with open(ckpt_path, "wb") as f:
- pickle.dump({"params": params_np, "config": config.__dict__}, f)
- del params_np
-
- # ── All hosts: distributed generation eval ──
- from ..model.run import generate_batch
- import json as _json_mod
-
- _pool_names = ["single", "multi"]
- display_pairs = []
- eval_pools = {name: [] for name in _pool_names}
-
- if val_ds is not None:
- val_kept = val_data["kept_indices"]
-
- sample_rng = np.random.RandomState(epoch + 7)
- sample_pool = sample_rng.permutation(len(val_kept))
- display_with, display_without = [], []
- for k in sample_pool:
- if len(display_with) >= 4 and len(display_without) >= 1:
- break
- local_idx = int(val_kept[k])
- ex = val_ds[local_idx]
- is_empty = ex["answers"].strip() in ("", "[]")
- if not is_empty and len(display_with) < 4:
- display_with.append(ex)
- elif is_empty and len(display_without) < 1:
- display_without.append(ex)
- display_pairs = display_with + display_without
-
- tc_per_bucket = 50
- tc_rng = np.random.RandomState(epoch + 42)
-
- def _classify_sample(ex):
- try:
- answers = _json_mod.loads(ex["answers"])
- except (ValueError, TypeError):
- return "empty"
- if not answers or not isinstance(answers, list):
- return "empty"
- return "single" if len(answers) == 1 else "multi"
-
- _pool_buckets = {name: {t: [] for t in range(11)} for name in _pool_names}
- for k in range(len(val_kept)):
- ex = val_ds[int(val_kept[k])]
- try:
- nc = min(len(_json_mod.loads(ex["tools"])), 10)
- except (ValueError, TypeError):
- nc = 0
- call_type = _classify_sample(ex)
- if call_type == "empty":
- continue
- _pool_buckets[call_type][nc].append(k)
-
- def _balanced_sample(buckets, per_bucket, rng):
- pool = []
- for t in range(11):
- b = np.array(buckets[t])
- if len(b) > 0:
- rng.shuffle(b)
- pool.extend(b[:per_bucket].tolist())
- rng.shuffle(pool)
- return [val_ds[int(val_kept[k])] for k in pool]
-
- for name in _pool_names:
- eval_pools[name] = _balanced_sample(_pool_buckets[name], tc_per_bucket, tc_rng)
-
- all_eval_examples = display_pairs
- _pool_offsets = {}
- for name in _pool_names:
- _pool_offsets[name] = len(all_eval_examples)
- all_eval_examples = all_eval_examples + eval_pools[name]
- eval_gen_len = min(args.max_dec_len, 512)
- _EVAL_BATCH = 32
-
- # Prepare generation model (all hosts)
- _gen_params = _quantize_params(eval_params, precision=_PRECISION)
- _gen_model = eval_model
- _gen_label = f"full {_PRECISION.upper()}"
-
- # Split eval examples across hosts — each host generates its slice
- total_eval = len(all_eval_examples)
- per_host_eval = (total_eval + num_hosts - 1) // num_hosts
- my_eval_start = host_id * per_host_eval
- my_eval_end = min(my_eval_start + per_host_eval, total_eval)
- my_eval_examples = all_eval_examples[my_eval_start:my_eval_end]
- _my_n_chunks = (len(my_eval_examples) + _EVAL_BATCH - 1) // _EVAL_BATCH
-
- my_preds = []
- _gen_t0 = time.perf_counter()
- for _ei in tqdm(range(0, len(my_eval_examples), _EVAL_BATCH),
- total=_my_n_chunks,
- desc=f" eval generate h{host_id} ({_gen_label})",
- leave=False, disable=not is_main):
- _chunk = my_eval_examples[_ei:_ei + _EVAL_BATCH]
- my_preds.extend(generate_batch(
- _gen_model, _gen_params, tokenizer,
- [ex["query"] for ex in _chunk],
- [ex["tools"] for ex in _chunk],
- max_gen_len=eval_gen_len,
- max_enc_len=args.max_enc_len,
- constrained=False,
- ))
- _gen_elapsed = time.perf_counter() - _gen_t0
- del _gen_params
-
- # All-gather predictions across hosts via padded token ID arrays
- _MAX_PRED_TOKENS = eval_gen_len
- pred_ids = np.zeros((per_host_eval, _MAX_PRED_TOKENS), dtype=np.int32)
- for i, p in enumerate(my_preds):
- toks = tokenizer.encode(p)[:_MAX_PRED_TOKENS]
- pred_ids[i, :len(toks)] = toks
- all_pred_ids = jax.experimental.multihost_utils.process_allgather(
- jnp.array(pred_ids), tiled=True)
- all_pred_ids = np.array(all_pred_ids)[:total_eval]
-
- # Decode predictions on all hosts (cheap), metrics computed on host 0
- all_preds = []
- for row in all_pred_ids:
- nonzero = np.nonzero(row)[0]
- if len(nonzero) > 0:
- all_preds.append(tokenizer.decode(row[:nonzero[-1] + 1].tolist()))
- else:
- all_preds.append("")
- _gen_total_toks = sum(len(tokenizer.encode(p)) for p in all_preds)
- _gen_tok_per_sec = _gen_total_toks / max(_gen_elapsed, 1e-6)
-
- # ── Non-main hosts: done with this epoch ──
- if not is_main:
- jax.experimental.multihost_utils.sync_global_devices("epoch_eval")
- continue
-
- display_preds = all_preds[:len(display_pairs)]
- pool_preds = {}
- for name in _pool_names:
- start = _pool_offsets[name]
- end = start + len(eval_pools[name])
- pool_preds[name] = all_preds[start:end]
-
- unified_samples = []
- for ex, pred in zip(display_pairs, display_preds):
- try:
- ref = _json_mod.dumps(_json_mod.loads(ex["answers"]), separators=(",", ":"))
- except (ValueError, TypeError):
- ref = ex["answers"].strip() or "[]"
- unified_samples.append({
- "query": ex["query"],
- "tools": ex["tools"],
- "ref": ref,
- "text": pred.strip(),
- })
-
- def _call_key(c):
- if not isinstance(c, dict): return None
- return _json_mod.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
-
- _pc_keys = ("n", "exact", "name_tp", "name_fp", "name_fn",
- "call_tp", "call_fp", "call_fn", "parse_err")
-
- def _eval_pool(eval_pairs, preds):
- """Compute tool-call metrics for a pool of (example, prediction) pairs."""
- m_n, m_exact, m_name_tp, m_name_fp, m_name_fn = 0, 0, 0, 0, 0
- m_call_tp, m_call_fp, m_call_fn, m_parse_err = 0, 0, 0, 0
- m_args_correct, m_args_total = 0, 0
- m_halluc, m_total_pred_params = 0, 0
- m_missing, m_total_ref_params = 0, 0
- m_correct_values, m_matched_params = 0, 0
- m_per_count = {t: {k: 0 for k in _pc_keys} for t in range(11)}
- m_failures = []
-
- for ex, pred_text in zip(eval_pairs, preds):
- try:
- tool_defs = _json_mod.loads(ex["tools"])
- num_tools = min(len(tool_defs), 10)
- except (ValueError, TypeError):
- tool_defs = []
- num_tools = 0
- pc = m_per_count[num_tools]
-
- ref_text = ex["answers"].strip()
- pred_text = pred_text.strip()
- ref_is_empty = ref_text in ("", "[]")
- pred_is_empty = pred_text in ("[]", "")
- try:
- ref_calls = _json_mod.loads(ref_text) if not ref_is_empty else []
- except (ValueError, TypeError):
- ref_calls = []
- try:
- pred_calls = _json_mod.loads(pred_text) if not pred_is_empty else []
- if not isinstance(pred_calls, list):
- pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
- except (ValueError, TypeError):
- m_parse_err += 1
- pc["parse_err"] += 1
- pred_calls = []
- m_n += 1
- pc["n"] += 1
- if ref_is_empty and pred_is_empty:
- m_exact += 1
- pc["exact"] += 1
- elif not ref_is_empty and not pred_is_empty:
- ref_sorted = sorted([_call_key(c) for c in ref_calls if _call_key(c)])
- pred_sorted = sorted([_call_key(c) for c in pred_calls if _call_key(c)])
- if ref_sorted == pred_sorted and len(ref_sorted) == len(ref_calls) and len(pred_sorted) == len(pred_calls):
- m_exact += 1
- pc["exact"] += 1
- ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
- pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}
- m_name_tp += len(pred_names & ref_names)
- m_name_fp += len(pred_names - ref_names)
- m_name_fn += len(ref_names - pred_names)
- pc["name_tp"] += len(pred_names & ref_names)
- pc["name_fp"] += len(pred_names - ref_names)
- pc["name_fn"] += len(ref_names - pred_names)
- rk = {_call_key(c) for c in ref_calls} - {None}
- pk = {_call_key(c) for c in pred_calls} - {None}
- m_call_tp += len(pk & rk)
- m_call_fp += len(pk - rk)
- m_call_fn += len(rk - pk)
- pc["call_tp"] += len(pk & rk)
- pc["call_fp"] += len(pk - rk)
- pc["call_fn"] += len(rk - pk)
-
- ref_by_name = {}
- for c in ref_calls:
- if isinstance(c, dict) and "name" in c:
- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))
- for c in pred_calls:
- if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:
- m_args_total += 1
- pred_args = _json_mod.dumps(c.get("arguments", {}), sort_keys=True)
- if any(pred_args == _json_mod.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):
- m_args_correct += 1
-
- tool_param_map = {}
- for t in tool_defs:
- if isinstance(t, dict) and "name" in t:
- tool_param_map[t["name"]] = set((t.get("parameters") or {}).keys())
- for c in pred_calls:
- if not isinstance(c, dict) or "name" not in c:
- continue
- cname = c["name"]
- if cname not in tool_param_map:
- continue
- pred_keys = set((c.get("arguments") or {}).keys())
- m_total_pred_params += len(pred_keys)
- m_halluc += len(pred_keys - tool_param_map[cname])
- if cname in ref_by_name:
- ref_args = ref_by_name[cname][0]
- ref_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())
- m_total_ref_params += len(ref_keys)
- m_missing += len(ref_keys - pred_keys)
- matched_keys = pred_keys & ref_keys
- m_matched_params += len(matched_keys)
- for k in matched_keys:
- if _json_mod.dumps(c.get("arguments", {})[k], sort_keys=True) == _json_mod.dumps(ref_args[k], sort_keys=True):
- m_correct_values += 1
-
- # Failure diagnosis
- is_exact = (ref_is_empty and pred_is_empty) or (
- not ref_is_empty and not pred_is_empty
- and sorted([_call_key(c) for c in ref_calls if _call_key(c)])
- == sorted([_call_key(c) for c in pred_calls if _call_key(c)])
- and len(ref_calls) == len(pred_calls)
- )
- if not is_exact and len(m_failures) < 30:
- reasons = []
- ex_fp = pred_names - ref_names
- ex_fn = ref_names - pred_names
- if ex_fp:
- reasons.append(f"wrong_tools:{','.join(sorted(ex_fp))}")
- if ex_fn:
- reasons.append(f"missing_tools:{','.join(sorted(ex_fn))}")
- if not ex_fp and not ex_fn and pred_names:
- for c in pred_calls:
- if not isinstance(c, dict) or "name" not in c or c["name"] not in ref_by_name:
- continue
- pa = c.get("arguments", {})
- ra = ref_by_name[c["name"]][0]
- for k in set(pa.keys()) | set(ra.keys()):
- pv, rv = pa.get(k, "<MISSING>"), ra.get(k, "<MISSING>")
- if _json_mod.dumps(pv, sort_keys=True) != _json_mod.dumps(rv, sort_keys=True):
- reasons.append(f"{c['name']}.{k}={_json_mod.dumps(pv)[:50]}!={_json_mod.dumps(rv)[:50]}")
- if ref_is_empty and not pred_is_empty:
- reasons.append("false_positive")
- if not ref_is_empty and pred_is_empty:
- reasons.append("false_negative")
- if not reasons:
- reasons.append("unknown")
- m_failures.append({
- "query": ex["query"][:150],
- "ref": ref_text[:200],
- "pred": pred_text[:200],
- "reasons": reasons,
- })
-
- metrics = {}
- if m_n > 0:
- metrics["n"] = m_n
- metrics["parse_rate"] = 1.0 - m_parse_err / m_n
- metrics["exact_match"] = m_exact / m_n
- np_ = m_name_tp + m_name_fp
- nr_ = m_name_tp + m_name_fn
- metrics["name_f1"] = 2 * m_name_tp / max(np_ + nr_, 1)
- cp_ = m_call_tp + m_call_fp
- cr_ = m_call_tp + m_call_fn
- metrics["call_f1"] = 2 * m_call_tp / max(cp_ + cr_, 1)
- metrics["args_acc"] = m_args_correct / max(m_args_total, 1)
- metrics["param_haluc"] = m_halluc / max(m_total_pred_params, 1)
- metrics["param_miss"] = m_missing / max(m_total_ref_params, 1)
- metrics["value_acc"] = m_correct_values / max(m_matched_params, 1)
- metrics["failures"] = m_failures
- return metrics, m_per_count
-
- pool_metrics = {}
- pool_pc = {}
- for name in _pool_names:
- pool_metrics[name], pool_pc[name] = _eval_pool(eval_pools[name], pool_preds[name])
-
- _best_metric = pool_metrics.get("single", {}).get("call_f1", 0)
- if _best_metric > best_call_f1:
- best_call_f1 = _best_metric
- best_ckpt_path = os.path.join(args.checkpoint_dir, f"{experiment_name}_{args.num_layers}_{args.d_model}_best.pkl")
- import shutil as _shutil
- _shutil.copy2(ckpt_path, best_ckpt_path)
- print(f" ** New best single call_f1={best_call_f1:.1%} → {best_ckpt_path}")
- _upload_checkpoint(best_ckpt_path)
-
- retrieval_metrics = None
- if has_contrastive and _CONTRASTIVE_WEIGHT > 0 and val_ds is not None:
- from ..training.eval import benchmark_retrieval
- retrieval_metrics = benchmark_retrieval(
- eval_model, eval_params, tokenizer,
- num_samples=min(500, getattr(args, "max_eval_samples", 500)),
- ds=val_ds,
- )
-
- del eval_params
-
- print(f"\n ─────────────────────────────────────")
- print(f" Epoch {epoch + 1}/{args.epochs}")
- print(f" ─────────────────────────────────────")
- print(f" Text loss {final_loss:>12.4f}")
- print(f" Text val ppl {last_val_ppl:>12.2f}")
- print(f" Quant val ppl {quant_val_ppl:>12.2f} ({_PRECISION.upper()})")
- def _print_tc_metrics(label, metrics, pc):
- if not metrics:
- return
- n = metrics["n"]
- print(f" ─── {label} ({n} samples) ──")
- print(f" JSON parse {metrics['parse_rate']:>10.1%}")
- print(f" Name F1 {metrics['name_f1']:>10.1%}")
- print(f" Param haluc {metrics['param_haluc']:>10.1%}")
- print(f" Param miss {metrics['param_miss']:>10.1%}")
- print(f" Value acc {metrics['value_acc']:>10.1%}")
- print(f" Args acc {metrics['args_acc']:>10.1%}")
- print(f" Call F1 {metrics['call_f1']:>10.1%}")
- print(f" Exact match {metrics['exact_match']:>10.1%}")
- has_any_pc = any(pc[t]["n"] > 0 for t in range(11))
- if has_any_pc:
- print(f" {'#tools':>6} {'n':>4} {'name_f1':>8} {'nTP':>4} {'nFP':>4} {'nFN':>4} {'call_f1':>8} {'cTP':>4} {'cFP':>4} {'cFN':>4} {'exact':>6} {'parse':>6}")
- for t in range(11):
- d = pc[t]
- if d["n"] == 0:
- continue
- np_ = d["name_tp"] + d["name_fp"]
- nr_ = d["name_tp"] + d["name_fn"]
- nf1 = 2 * d["name_tp"] / max(np_ + nr_, 1)
- cp_ = d["call_tp"] + d["call_fp"]
- cr_ = d["call_tp"] + d["call_fn"]
- cf1 = 2 * d["call_tp"] / max(cp_ + cr_, 1)
- ex_ = d["exact"] / d["n"]
- pr_ = 1.0 - d["parse_err"] / d["n"]
- print(f" {t:>6} {d['n']:>4} {nf1:>7.1%} {d['name_tp']:>4} {d['name_fp']:>4} {d['name_fn']:>4} {cf1:>7.1%} {d['call_tp']:>4} {d['call_fp']:>4} {d['call_fn']:>4} {ex_:>5.1%} {pr_:>5.1%}")
- if metrics.get("failures"):
- print(f" ─── Failures ({len(metrics['failures'])} captured) ───")
- for j, fail in enumerate(metrics["failures"][:10]):
- print(f" [{j+1}] Q: {fail['query'][:120]}")
- print(f" Ref: {fail['ref'][:200]}")
- print(f" Pred: {fail['pred'][:200]}")
- print(f" Why: {', '.join(fail['reasons'])}")
- print()
-
- _label_map = {
- "single": "Single-Call",
- "multi": "Multi-Call",
- }
- for name in _pool_names:
- _print_tc_metrics(_label_map[name], pool_metrics[name], pool_pc[name])
- if retrieval_metrics and retrieval_metrics["num_queries"] > 0:
- rm = retrieval_metrics
- print(f" ─── Retrieval ({rm['num_queries']} queries) ─────")
- for k, v in sorted(rm["recall@k"].items()):
- print(f" Recall@{k:<3} {v:>10.1%}")
- print(f" MRR {rm['mrr']:>10.3f}")
- print(f" ─────────────────────────────────────")
- print(f" Throughput {_gen_tok_per_sec:>10.1f} tok/s ({len(all_eval_examples)} samples, {_gen_elapsed:.1f}s, {_gen_label})")
- if unified_samples:
- print(f" ─── Samples ({len(unified_samples)}) ───────────────────")
- for j, s in enumerate(unified_samples):
- print(f" [{j+1}] Query: {s['query'][:120]}")
- tools_short = s["tools"][:120]
- if len(s["tools"]) > 120:
- tools_short += "..."
- print(f" Tools: {tools_short}")
- print(f" Ref: {s['ref'][:200]}")
- print(f" Text: {s['text'][:200] or '(empty)'}")
- if j < len(unified_samples) - 1:
- print()
- print(f" ─────────────────────────────────────")
- print(f" Checkpoint: {ckpt_path}")
- print(f" ─────────────────────────────────────\n")
-
- if use_wandb:
- log_dict = {
- "epoch/text_loss": final_loss,
- "epoch/text_val_ppl": last_val_ppl,
- "epoch/quant_val_ppl": quant_val_ppl,
- "epoch": epoch + 1,
- }
- for name in _pool_names:
- m = pool_metrics.get(name)
- if not m:
- continue
- for k in ("parse_rate", "exact_match", "name_f1", "call_f1",
- "args_acc", "param_haluc", "param_miss", "value_acc"):
- log_dict[f"epoch/{name}_{k}"] = m[k]
- if retrieval_metrics and retrieval_metrics["num_queries"] > 0:
- for k, v in retrieval_metrics["recall@k"].items():
- log_dict[f"epoch/retrieval_recall@{k}"] = v
- log_dict["epoch/retrieval_mrr"] = retrieval_metrics["mrr"]
- wandb.log(log_dict)
-
- # Sync with non-main hosts that are waiting at the barrier
- jax.experimental.multihost_utils.sync_global_devices("epoch_eval")
-
- if use_wandb and is_main:
- wandb.finish()
- if is_main and best_ckpt_path:
- print(f"\nBest checkpoint (call_f1={best_call_f1:.1%}): {best_ckpt_path}")
- if is_main:
- print("\nTraining complete.")
-
-
diff --git a/needle/ui/server.py b/needle/ui/server.py
deleted file mode 100644
index d75bbc9..0000000
--- a/needle/ui/server.py
+++ /dev/null
@@ -1,780 +0,0 @@
-import json as _json
-import os
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-import threading
-import time
-import urllib.parse
-from email.parser import BytesParser
-from email.policy import default as _email_policy
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
-
-_STATIC_DIR = Path(__file__).parent / "static"
-_CONTENT_TYPES = {
- ".html": "text/html; charset=utf-8",
- ".css": "text/css; charset=utf-8",
- ".js": "application/javascript; charset=utf-8",
-}
-_DEFAULT_MAX_GEN_LEN = 512
-_MAX_MAX_GEN_LEN = 4096
-_MAX_JSON_BYTES = 1024 * 1024
-_MAX_UPLOAD_BYTES = 512 * 1024 * 1024
-_SOCKET_TIMEOUT = 30
-
-_model = None
-_params = None
-_tokenizer = None
-_current_model = ""
-_current_model_path = None
-_lock = threading.Lock()
-_finetune_lock = threading.Lock()
-_finetune_status = {
- "running": False, "step": "", "log": [], "checkpoint": None,
- "validation": None, "eval_results": None, "base_eval": None,
-}
-
-
-def _is_local_request(client_ip):
- return client_ip in {"127.0.0.1", "::1", "::ffff:127.0.0.1"}
-
-
-def _is_same_origin(handler):
- """Reject cross-origin POST requests (CSRF protection).
-
- Browsers always send Origin on cross-origin POSTs. If Origin is present
- it must match the Host header; if absent we allow (non-browser client).
- """
- origin = handler.headers.get("Origin")
- if origin is None:
- return True
- try:
- parsed = urllib.parse.urlparse(origin)
- origin_host = parsed.hostname or ""
- default_port = 443 if parsed.scheme == "https" else 80
- origin_port = parsed.port or default_port
- host_header = handler.headers.get("Host", "")
- if ":" in host_header:
- h_host, h_port = host_header.rsplit(":", 1)
- h_port = int(h_port)
- else:
- h_host, h_port = host_header, default_port
- return origin_host == h_host and origin_port == h_port
- except Exception:
- return False
-
-
-def _project_root():
- return Path(__file__).parent.parent.parent
-
-
-def _checkpoints_dir():
- return _project_root() / "checkpoints"
-
-
-def _snapshot_finetune_status():
- with _finetune_lock:
- return {
- "running": _finetune_status["running"],
- "step": _finetune_status["step"],
- "log": list(_finetune_status["log"]),
- "checkpoint": _finetune_status["checkpoint"],
- "validation": _finetune_status["validation"],
- "eval_results": _finetune_status["eval_results"],
- "base_eval": _finetune_status["base_eval"],
- }
-
-
-def _set_finetune_status(**updates):
- with _finetune_lock:
- _finetune_status.update(updates)
-
-
-_MAX_FINETUNE_LOG_LINES = 2000
-
-
-def _append_finetune_log(msg):
- with _finetune_lock:
- log = _finetune_status["log"]
- log.append(msg)
- if len(log) > _MAX_FINETUNE_LOG_LINES:
- del log[: len(log) - _MAX_FINETUNE_LOG_LINES]
- print(f"[finetune] {msg}", file=sys.stderr)
-
-
-
-def _read_request_body(handler, max_bytes):
- try:
- length = int(handler.headers.get("Content-Length", "0"))
- except ValueError:
- raise ValueError("invalid Content-Length")
- if length <= 0:
- raise ValueError("empty request body")
- if length > max_bytes:
- raise ValueError(f"request body too large (max {max_bytes} bytes)")
- return handler.rfile.read(length)
-
-
-def _read_json_request(handler, max_bytes=_MAX_JSON_BYTES):
- try:
- raw = _read_request_body(handler, max_bytes)
- body = _json.loads(raw)
- except _json.JSONDecodeError as exc:
- raise ValueError("invalid JSON") from exc
- if not isinstance(body, dict):
- raise ValueError("JSON body must be an object")
- return body
-
-
-def _parse_bool(value, default):
- if value is None:
- return default
- if isinstance(value, bool):
- return value
- if isinstance(value, str):
- lowered = value.strip().lower()
- if lowered in {"1", "true", "yes", "on"}:
- return True
- if lowered in {"0", "false", "no", "off"}:
- return False
- raise ValueError("boolean field is invalid")
-
-
-def _clamp_int(value, default, minimum, maximum, field_name):
- if value in (None, ""):
- return default
- try:
- parsed = int(value)
- except (TypeError, ValueError) as exc:
- raise ValueError(f"{field_name} must be an integer") from exc
- return max(minimum, min(parsed, maximum))
-
-
-def _normalize_tools_json(tools):
- if tools is None or tools == "":
- return "[]"
- if isinstance(tools, str):
- try:
- parsed = _json.loads(tools)
- except _json.JSONDecodeError as exc:
- raise ValueError("tools must be valid JSON") from exc
- elif isinstance(tools, list):
- parsed = tools
- else:
- raise ValueError("tools must be a JSON string or array")
- if not isinstance(parsed, list):
- raise ValueError("tools must be a JSON array")
- return _json.dumps(parsed, separators=(",", ":"))
-
-
-def _parse_generate_request(body):
- query = body.get("query")
- if not isinstance(query, str) or not query.strip():
- raise ValueError("query is required")
- tools = _normalize_tools_json(body.get("tools", "[]"))
- seed = _clamp_int(body.get("seed", 0), 0, 0, 2**31 - 1, "seed")
- max_gen_len = _clamp_int(
- body.get("max_gen_len", _DEFAULT_MAX_GEN_LEN),
- _DEFAULT_MAX_GEN_LEN,
- 1,
- _MAX_MAX_GEN_LEN,
- "max_gen_len",
- )
- constrained = _parse_bool(body.get("constrained", True), True)
- return query.strip(), tools, seed, max_gen_len, constrained
-
-
-def _parse_finetune_request(body):
- tools = _normalize_tools_json(body.get("tools", "[]"))
- parsed_tools = _json.loads(tools)
- if not parsed_tools:
- raise ValueError("tools must be a non-empty JSON array")
- for i, t in enumerate(parsed_tools):
- if not isinstance(t, dict):
- raise ValueError(f"tool[{i}] must be an object")
- if "name" not in t:
- if "function" in t:
- raise ValueError(
- 'Wrong format: tools use OpenAI format ({"type":"function","function":{...}}). '
- 'Needle requires [{"name":"...","description":"...","parameters":{...}}]'
- )
- raise ValueError(f"tool[{i}] missing 'name' field")
- if "description" not in t:
- raise ValueError(f"tool[{i}] missing 'description' field")
- api_key = body.get("api_key")
- if not isinstance(api_key, str) or not api_key.strip():
- raise ValueError("Gemini API key is required")
- return tools, api_key.strip()
-
-
-def _stream_upload_to_file(handler, max_bytes, target_dir):
- """Parse multipart upload and stream the file part directly to disk.
-
- Returns (filename, target_path) on success. Raises ValueError on bad input.
- """
- content_type = handler.headers.get("Content-Type", "")
- if "multipart/form-data" not in content_type:
- raise ValueError("expected multipart/form-data")
- if "boundary=" not in content_type:
- raise ValueError("missing multipart boundary")
- try:
- length = int(handler.headers.get("Content-Length", "0"))
- except ValueError:
- raise ValueError("invalid Content-Length")
- if length <= 0:
- raise ValueError("empty request body")
- if length > max_bytes:
- raise ValueError(f"request body too large (max {max_bytes} bytes)")
-
- body = handler.rfile.read(length)
- header = f"Content-Type: {content_type}\r\nMIME-Version: 1.0\r\n\r\n".encode("utf-8")
- message = BytesParser(policy=_email_policy).parsebytes(header + body)
- del body # free raw body (~512 MB)
- if not message.is_multipart():
- raise ValueError("expected multipart/form-data")
-
- filename = None
- payload = None
- for part in message.iter_parts():
- if part.get_param("name", header="content-disposition") != "file":
- continue
- filename = part.get_filename()
- payload = part.get_payload(decode=True)
- break
- del message # free parsed MIME tree
- if not filename or payload is None:
- raise ValueError("expected a .pkl file")
- if not filename.endswith(".pkl"):
- raise ValueError("expected a .pkl file")
- target_dir.mkdir(parents=True, exist_ok=True)
- safe_name = re.sub(r"[^A-Za-z0-9._-]", "_", Path(filename).name)
- target = target_dir / safe_name
- with open(target, "wb") as f:
- f.write(payload)
- del payload # free file data
- return filename, target
-
-
-_datagen_lock = threading.Lock()
-
-
-def _generate_custom_data(tools_json, api_key, num_samples, data_file):
- import json
- from ..dataset import generate as gd
-
- tools = json.loads(tools_json)
-
- def _pick_tools(_rng, force_empty=False, few_tools=False):
- if force_empty:
- return []
- return [dict(t) for t in tools]
-
- def _synthesize_tools(*args, **kwargs):
- return None
-
- with _datagen_lock:
- old_api_key = os.environ.get("GEMINI_API_KEY")
- old_pick_tools = gd._pick_tools
- old_synthesize_tools = gd._synthesize_tools
- old_overlap_pairs = gd._OVERLAP_PAIRS
- old_languages = gd.LANGUAGES
- try:
- os.environ["GEMINI_API_KEY"] = api_key
- gd._pick_tools = _pick_tools
- gd._synthesize_tools = _synthesize_tools
- gd._OVERLAP_PAIRS = []
- gd.LANGUAGES = ["English"]
- client_pool = gd.ClientPool(gd.make_clients())
- examples = gd.generate_all(
- num_samples,
- workers=4,
- batch_size=25,
- model=gd.MODEL,
- client_pool=client_pool,
- )
- with open(data_file, "w") as f:
- for ex in examples:
- f.write(json.dumps(ex) + "\n")
- return len(examples)
- finally:
- gd._pick_tools = old_pick_tools
- gd._synthesize_tools = old_synthesize_tools
- gd._OVERLAP_PAIRS = old_overlap_pairs
- gd.LANGUAGES = old_languages
- if old_api_key is None:
- os.environ.pop("GEMINI_API_KEY", None)
- else:
- os.environ["GEMINI_API_KEY"] = old_api_key
-
-
-class _Handler(BaseHTTPRequestHandler):
- def do_GET(self):
- path = self.path.split("?")[0]
- if path == "/":
- path = "/static/index.html"
- if path == "/model-info":
- self._json_response(200, {"name": _current_model})
- return
- if path == "/finetune/status":
- if not _is_local_request(self.client_address[0]):
- self._json_response(403, {"error": "finetune status is only available from localhost"})
- return
- self._json_response(200, _snapshot_finetune_status())
- return
- if path.startswith("/download/"):
- if not _is_local_request(self.client_address[0]):
- self._json_response(403, {"error": "checkpoint download is only allowed from localhost"})
- return
- name = urllib.parse.unquote(path[len("/download/"):])
- if "/" in name or "\\" in name or name.startswith("."):
- self.send_error(400)
- return
- fpath = (_checkpoints_dir() / name).resolve()
- ckpt_dir = _checkpoints_dir().resolve()
- if fpath.is_file() and fpath.is_relative_to(ckpt_dir) and fpath.suffix in (".pkl", ".zip"):
- ctype = "application/zip" if fpath.suffix == ".zip" else "application/octet-stream"
- self.send_response(200)
- self.send_header("Content-Type", ctype)
- self.send_header("Content-Disposition", f'attachment; filename="{name}"')
- self.send_header("Content-Length", str(fpath.stat().st_size))
- self.end_headers()
- with open(fpath, "rb") as f:
- while chunk := f.read(65536):
- self.wfile.write(chunk)
- return
- if path.startswith("/static/"):
- name = path[len("/static/"):]
- fpath = (_STATIC_DIR / name).resolve()
- if fpath.is_file() and fpath.is_relative_to(_STATIC_DIR.resolve()):
- self.send_response(200)
- self.send_header("Content-Type", _CONTENT_TYPES.get(fpath.suffix, "application/octet-stream"))
- self.end_headers()
- self.wfile.write(fpath.read_bytes())
- return
- self.send_error(404)
-
- def do_POST(self):
- if not _is_same_origin(self):
- self._json_response(403, {"error": "cross-origin requests are not allowed"})
- return
- if self.path == "/generate":
- self._handle_generate()
- elif self.path == "/finetune":
- self._handle_finetune()
- elif self.path == "/load-model":
- self._handle_load_model()
- else:
- self.send_error(404)
-
- def _handle_generate(self):
- try:
- body = _read_json_request(self)
- query, tools, seed, max_gen_len, constrained = _parse_generate_request(body)
- except ValueError as exc:
- self._json_response(400, {"error": str(exc)})
- return
-
- result, error = _run_generate(query, tools, seed, max_gen_len, constrained)
- if error:
- self._json_response(500, {"error": error})
- else:
- self._json_response(200, {"result": result})
-
- def _handle_finetune(self):
- if not _is_local_request(self.client_address[0]):
- self._json_response(403, {"error": "finetune is only allowed from localhost"})
- return
- try:
- body = _read_json_request(self)
- tools, api_key = _parse_finetune_request(body)
- except ValueError as exc:
- self._json_response(400, {"error": str(exc)})
- return
- if not _current_model_path:
- self._json_response(400, {"error": "Load a model before finetuning"})
- return
- if not _start_finetune(tools, api_key):
- self._json_response(409, {"error": "finetune already running"})
- return
- self._json_response(200, {"status": "started"})
-
- def _handle_load_model(self):
- if not _is_local_request(self.client_address[0]):
- self._json_response(403, {"error": "model upload is only allowed from localhost"})
- return
- upload_dir = _checkpoints_dir() / "_uploaded"
- try:
- filename, target = _stream_upload_to_file(self, _MAX_UPLOAD_BYTES, upload_dir)
- except ValueError as exc:
- self._json_response(400, {"error": str(exc)})
- return
- try:
- _load_checkpoint(str(target), display_name=filename)
- self._json_response(200, {"name": filename})
- except Exception as exc:
- target.unlink(missing_ok=True)
- self._json_response(500, {"error": "Failed to load checkpoint"})
-
- def _json_response(self, code, data):
- payload = _json.dumps(data).encode()
- self.send_response(code)
- self.send_header("Content-Type", "application/json; charset=utf-8")
- self.send_header("Content-Length", str(len(payload)))
- self.end_headers()
- self.wfile.write(payload)
-
- def log_message(self, fmt, *args):
- pass
-
-
-def _run_generate(query, tools, seed, max_gen_len, constrained):
- from ..model.run import generate
-
- with _lock:
- if _model is None or _params is None or _tokenizer is None:
- return None, "model is not loaded"
- try:
- result = generate(
- _model,
- _params,
- _tokenizer,
- query,
- tools=tools,
- max_gen_len=max_gen_len,
- seed=seed,
- stream=False,
- constrained=constrained,
- )
- return result, None
- except Exception as exc:
- print(f"[generate] {exc}", file=sys.stderr)
- return None, "Generation failed"
-
-
-def _load_checkpoint(path, display_name=None):
- global _model, _params, _tokenizer, _current_model, _current_model_path
- from ..dataset.dataset import get_tokenizer
- from ..model.architecture import SimpleAttentionNetwork
- from ..model.run import load_checkpoint
-
- with _lock:
- _params, config = load_checkpoint(path)
- _model = SimpleAttentionNetwork(config)
- _tokenizer = get_tokenizer()
- _current_model = display_name or Path(path).name
- _current_model_path = path
- print(f"Loaded: {_current_model}", file=sys.stderr)
-
-
-def _validate_training_data(data_file_path):
- """Validate JSONL training data and return a report dict."""
- import collections
- examples = []
- warnings = []
- with open(data_file_path) as f:
- for i, line in enumerate(f):
- if not line.strip():
- continue
- try:
- ex = _json.loads(line)
- except _json.JSONDecodeError:
- warnings.append(f"Line {i+1}: invalid JSON")
- continue
- for key in ("query", "tools", "answers"):
- if key not in ex:
- warnings.append(f"Line {i+1}: missing '{key}'")
- examples.append(ex)
-
- tool_counts = collections.Counter()
- for ex in examples:
- try:
- answers = _json.loads(ex.get("answers", "[]"))
- for a in answers:
- if isinstance(a, dict) and "name" in a:
- tool_counts[a["name"]] += 1
- except (_json.JSONDecodeError, TypeError):
- pass
-
- seen = set()
- dup_count = 0
- for ex in examples:
- key = (ex.get("query", ""), ex.get("answers", ""))
- if key in seen:
- dup_count += 1
- seen.add(key)
-
- counts = list(tool_counts.values()) or [0]
- return {
- "total": len(examples),
- "per_tool": dict(tool_counts),
- "duplicates": dup_count,
- "min_per_tool": min(counts),
- "max_per_tool": max(counts),
- "warnings": warnings[:20],
- }
-
-
-_SAMPLES_PER_TOOL = 120 # 100 train + 10 val + 10 test
-_EPOCHS = 1
-
-
-def _start_finetune(tools_json, api_key):
- with _finetune_lock:
- if _finetune_status["running"]:
- return False
- _finetune_status["running"] = True
- _finetune_status["step"] = "starting"
- _finetune_status["log"] = []
- _finetune_status["checkpoint"] = None
- _finetune_status["validation"] = None
- _finetune_status["eval_results"] = None
- _finetune_status["base_eval"] = None
-
- def _run():
- python = sys.executable
- work_dir = Path(tempfile.mkdtemp(prefix="needle-ui-finetune-", dir=str(_project_root())))
- data_file = work_dir / "data.jsonl"
- cache_dir = work_dir / "cache"
-
- try:
- num_tools = len(_json.loads(tools_json))
- num_samples = _SAMPLES_PER_TOOL * num_tools
-
- _set_finetune_status(step="generating data")
- _append_finetune_log(f"Generating {_SAMPLES_PER_TOOL} samples/tool for {num_tools} tools...")
- generated = _generate_custom_data(tools_json, api_key, num_samples, data_file)
- if generated < 3:
- raise RuntimeError("generated fewer than 3 examples; need more data for train/val/test")
- _append_finetune_log(f"Generated {generated} samples.")
-
- # Data validation
- # Validation runs as part of data generation step
- _append_finetune_log("Validating training data...")
- validation = _validate_training_data(str(data_file))
- _set_finetune_status(validation=validation)
- _append_finetune_log(
- f"Validated: {validation['total']} examples, "
- f"{len(validation['per_tool'])} tools, "
- f"{validation['duplicates']} duplicates"
- )
- if validation["warnings"]:
- for w in validation["warnings"][:5]:
- _append_finetune_log(f" WARNING: {w}")
-
- # Training subprocess
- _set_finetune_status(step="training")
- checkpoint_arg = []
- ckpt_name = "scratch"
- ckpt_dir = _checkpoints_dir()
- existing_pkls = set()
- if ckpt_dir.exists():
- existing_pkls = set(p.name for p in ckpt_dir.glob("*.pkl"))
- if _current_model_path:
- checkpoint_arg = ["--checkpoint", _current_model_path]
- ckpt_name = Path(_current_model_path).name
- batch_size = min(32, generated)
- _append_finetune_log(
- f"Training {_EPOCHS} epochs, batch_size={batch_size}, "
- f"{generated} examples, from {ckpt_name}"
- )
-
- _set_finetune_status(step="evaluating base")
- last_eval = None
- proc = subprocess.Popen(
- [
- python, "-u",
- "-m", "needle.training.finetune",
- str(data_file),
- "--epochs", str(_EPOCHS),
- "--batch-size", str(batch_size),
- "--checkpoint-dir", str(ckpt_dir),
- "--cache-dir", str(cache_dir),
- *checkpoint_arg,
- ],
- cwd=str(_project_root()),
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- text=True,
- )
- try:
- for line in proc.stdout:
- line = line.rstrip("\n")
- if line.startswith("BASE_EVAL:"):
- try:
- _set_finetune_status(base_eval=_json.loads(line[10:]), step="training")
- except _json.JSONDecodeError:
- pass
- elif line.startswith("FINETUNED_EVAL:"):
- try:
- last_eval = _json.loads(line[15:])
- _set_finetune_status(eval_results=last_eval)
- except _json.JSONDecodeError:
- pass
- elif line:
- if "Evaluating finetuned" in line:
- _set_finetune_status(step="evaluating finetuned")
- _append_finetune_log(line)
- proc.wait(timeout=7200)
- except subprocess.TimeoutExpired:
- proc.kill()
- _append_finetune_log("Training timed out (2h limit).")
- _set_finetune_status(step="failed")
- return
-
- if proc.returncode != 0:
- _append_finetune_log("Training failed (see log above).")
- _set_finetune_status(step="failed")
- return
- _append_finetune_log("Training complete.")
-
- # Store final eval results
- if last_eval:
- _set_finetune_status(eval_results=last_eval)
-
- new_files = [p for p in ckpt_dir.glob("*.pkl") if p.name not in existing_pkls]
- if not new_files:
- _append_finetune_log("No new checkpoint produced.")
- _set_finetune_status(step="done")
- return
-
- newest = max(new_files, key=lambda p: p.stat().st_mtime)
- ts = time.strftime("%Y%m%d_%H%M%S")
- final_name = f"needle_finetuned_{ts}.pkl"
- final_path = ckpt_dir / final_name
- newest.replace(final_path)
-
- # Assemble download bundle (same per-tool split as finetune)
- import zipfile
- from ..training.finetune import _per_tool_split
-
- with open(data_file) as _df:
- all_examples = [_json.loads(ln) for ln in _df if ln.strip()]
- train_examples, val_examples, test_examples = _per_tool_split(all_examples)
-
- with _finetune_lock:
- base_eval = _finetune_status.get("base_eval")
-
- per_tool_report = {}
- if base_eval and base_eval.get("per_tool"):
- for t, bm in base_eval["per_tool"].items():
- per_tool_report[t] = {"base": round(bm["correct"] / max(bm["total"], 1), 3)}
- if last_eval and last_eval.get("per_tool"):
- for t, fm in last_eval["per_tool"].items():
- entry = per_tool_report.setdefault(t, {})
- entry["finetuned"] = round(fm["correct"] / max(fm["total"], 1), 3)
-
- eval_report = {
- "model": ckpt_name,
- "finetuned_checkpoint": final_name,
- "training": {"examples": len(train_examples), "tools": num_tools, "epochs": _EPOCHS},
- "validation": {"examples": len(val_examples)},
- "test": {"examples": len(test_examples), "duplicates": validation.get("duplicates", 0)},
- "base": {k: base_eval.get(k) for k in ("call_f1", "name_f1", "exact_match", "parse_rate", "args_acc")} if base_eval else None,
- "finetuned": {k: last_eval.get(k) for k in ("call_f1", "name_f1", "exact_match", "parse_rate", "args_acc")} if last_eval else None,
- "per_tool": per_tool_report,
- }
-
- def _pct(v):
- return f"{v * 100:.1f}%" if v is not None else "n/a"
-
- b = eval_report.get("base") or {}
- f_ = eval_report.get("finetuned") or {}
- readme = (
- f"# Needle Finetuned Model\n\n"
- f"## Training\n"
- f"- Base model: {ckpt_name}\n"
- f"- Epochs: {_EPOCHS}\n"
- f"- Training examples: {len(train_examples)}\n"
- f"- Validation examples: {len(val_examples)} (used for checkpoint selection)\n"
- f"- Test examples: {len(test_examples)} (held out, used for eval)\n"
- f"- Tools: {num_tools}\n\n"
- f"## Evaluation (on held-out test set)\n\n"
- f"| Metric | Base | Finetuned |\n"
- f"|--------|------|----------|\n"
- f"| Call F1 | {_pct(b.get('call_f1'))} | {_pct(f_.get('call_f1'))} |\n"
- f"| Name F1 | {_pct(b.get('name_f1'))} | {_pct(f_.get('name_f1'))} |\n"
- f"| Exact Match | {_pct(b.get('exact_match'))} | {_pct(f_.get('exact_match'))} |\n"
- f"| Parse Rate | {_pct(b.get('parse_rate'))} | {_pct(f_.get('parse_rate'))} |\n"
- f"| Args Accuracy | {_pct(b.get('args_acc'))} | {_pct(f_.get('args_acc'))} |\n\n"
- f"## Contents\n"
- f"- `checkpoint.pkl` — Finetuned model weights\n"
- f"- `tools.json` — Tool definitions\n"
- f"- `train.jsonl` — Training data ({len(train_examples)} examples)\n"
- f"- `val.jsonl` — Validation data ({len(val_examples)} examples)\n"
- f"- `test.jsonl` — Test data ({len(test_examples)} examples)\n"
- f"- `eval_report.json` — Machine-readable evaluation\n"
- )
-
- bundle_name = f"needle_finetuned_{ts}.zip"
- bundle_path = ckpt_dir / bundle_name
- with zipfile.ZipFile(str(bundle_path), "w", zipfile.ZIP_DEFLATED) as zf:
- zf.write(str(final_path), "checkpoint.pkl")
- zf.writestr("tools.json", tools_json)
- zf.writestr("train.jsonl", "\n".join(_json.dumps(ex) for ex in train_examples) + "\n")
- zf.writestr("val.jsonl", "\n".join(_json.dumps(ex) for ex in val_examples) + "\n")
- zf.writestr("test.jsonl", "\n".join(_json.dumps(ex) for ex in test_examples) + "\n")
- zf.writestr("eval_report.json", _json.dumps(eval_report, indent=2))
- zf.writestr("README.md", readme)
-
- _set_finetune_status(checkpoint=bundle_name)
- _append_finetune_log(f"Bundle ready: {bundle_name}")
- _set_finetune_status(step="done")
- except Exception as exc:
- _append_finetune_log(f"Error: {exc}")
- _set_finetune_status(step="failed")
- finally:
- _set_finetune_status(running=False)
- shutil.rmtree(work_dir, ignore_errors=True)
-
- threading.Thread(target=_run, daemon=True).start()
- return True
-
-
-_HF_MODEL_REPO = "Cactus-Compute/needle"
-_HF_MODEL_FILE = "needle.pkl"
-
-
-def _resolve_checkpoint(checkpoint_arg):
- """Resolve checkpoint path: always download from HuggingFace to ensure freshness."""
- from huggingface_hub import hf_hub_download
- local_dir = "checkpoints"
- os.makedirs(local_dir, exist_ok=True)
- filename = os.path.basename(checkpoint_arg) if checkpoint_arg else _HF_MODEL_FILE
- repo = _HF_MODEL_REPO
- print(f"Downloading {filename} from {repo}...", file=sys.stderr)
- path = hf_hub_download(
- repo_id=repo,
- filename=filename,
- repo_type="model",
- local_dir=local_dir,
- force_download=True,
- )
- print(f"Downloaded to {path}", file=sys.stderr)
- return path
-
-
-def main(args):
- global _model, _params, _tokenizer, _current_model, _current_model_path
- from ..dataset.dataset import get_tokenizer
- from ..model.architecture import SimpleAttentionNetwork
- from ..model.run import load_checkpoint
-
- checkpoint_path = _resolve_checkpoint(args.checkpoint)
- print(f"Loading checkpoint: {checkpoint_path}", file=sys.stderr)
- _params, config = load_checkpoint(checkpoint_path)
- _model = SimpleAttentionNetwork(config)
- _tokenizer = get_tokenizer()
- _current_model = Path(checkpoint_path).name
- _current_model_path = checkpoint_path
-
- param_count = sum(x.size for x in __import__("jax").tree.leaves(_params))
- print(f"Model parameters: {param_count:,}", file=sys.stderr)
-
- server = ThreadingHTTPServer((args.host, args.port), _Handler)
- server.timeout = _SOCKET_TIMEOUT
- server.request_queue_size = 16
- print(f"Needle UI: http://{args.host}:{args.port}", file=sys.stderr)
- server.serve_forever()
diff --git a/needle/ui/static/app.js b/needle/ui/static/app.js
deleted file mode 100644
index cb06b58..0000000
--- a/needle/ui/static/app.js
+++ /dev/null
@@ -1,354 +0,0 @@
-var _toastTimer = null;
-
-function showError(msg) {
- if (_toastTimer) clearTimeout(_toastTimer);
- document.getElementById("toastMsg").textContent = msg;
- document.getElementById("toast").classList.add("visible");
- _toastTimer = setTimeout(dismissToast, 8000);
-}
-
-function dismissToast() {
- document.getElementById("toast").classList.remove("visible");
-}
-
-function loadToolsFile(input) {
- var file = input.files[0];
- if (!file) return;
- var reader = new FileReader();
- reader.onload = function () {
- try {
- JSON.parse(reader.result);
- document.getElementById("tools").value = reader.result;
- } catch (e) {
- showError("Invalid JSON file");
- }
- };
- reader.readAsText(file);
- input.value = "";
-}
-
-function fetchModelName() {
- fetch("/model-info")
- .then(function (r) {
- if (!r.ok) throw new Error(r.status);
- return r.json();
- })
- .then(function (data) {
- document.getElementById("modelName").textContent = data.name || "";
- })
- .catch(function () {});
-}
-
-fetchModelName();
-
-function loadModelFile(input) {
- var file = input.files[0];
- if (!file) return;
- var formData = new FormData();
- formData.append("file", file);
- document.getElementById("result").textContent = "Loading model...";
- fetch("/load-model", { method: "POST", body: formData })
- .then(function (r) {
- if (!r.ok && r.headers.get("content-type") !== "application/json; charset=utf-8") throw new Error("Server error " + r.status);
- return r.json();
- })
- .then(function (data) {
- if (data.error) {
- showError(data.error);
- document.getElementById("result").textContent = "";
- } else {
- document.getElementById("result").className = "result-box has-result";
- document.getElementById("result").textContent = "Model loaded: " + data.name;
- document.getElementById("modelName").textContent = data.name;
- }
- })
- .catch(function (e) { showError("Upload failed: " + e.message); });
- input.value = "";
-}
-
-async function send() {
- var input = document.getElementById("query");
- var btn = document.getElementById("sendBtn");
- var result = document.getElementById("result");
- var query = input.value.trim();
- if (!query) return;
-
- input.disabled = true;
- btn.disabled = true;
- result.className = "result-box";
- result.textContent = "Running...";
-
- try {
- var r = await fetch("/generate", {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({
- query: query,
- tools: document.getElementById("tools").value.trim() || "[]",
- seed: 0,
- max_gen_len: 512,
- constrained: true,
- }),
- });
- if (!r.ok && !(r.headers.get("content-type") || "").includes("json")) throw new Error("Server error " + r.status);
- var data = await r.json();
- if (data.error) {
- result.textContent = "";
- showError(data.error);
- } else if (data.result === null || data.result === undefined || data.result === "") {
- result.textContent = "";
- showError("Empty response from model");
- } else {
- result.className = "result-box has-result";
- result.textContent = data.result;
- }
- } catch (e) {
- result.textContent = "";
- showError("Request failed: " + e.message);
- } finally {
- input.disabled = false;
- btn.disabled = false;
- input.focus();
- }
-}
-
-function togglePanel() {
- var sidebar = document.getElementById("sidebar");
- sidebar.classList.toggle("open");
- var label = document.querySelector(".tools-toggle span");
- label.textContent = sidebar.classList.contains("open") ? "Query" : "Tools";
-}
-
-var _pollTimer = null;
-var _ftRunning = false;
-
-function openFinetuneModal() {
- var tools = document.getElementById("tools").value.trim() || "[]";
- try {
- JSON.parse(tools);
- } catch (e) {
- showError("Invalid tools JSON");
- return;
- }
- _resetModal();
- document.getElementById("modalOverlay").classList.add("visible");
- document.getElementById("ftApiKey").focus();
-}
-
-function closeModal(e) {
- if (e && e.target && e.target !== document.getElementById("modalOverlay")) return;
- if (_ftRunning) return;
- document.getElementById("modalOverlay").classList.remove("visible");
-}
-
-function _resetModal() {
- document.getElementById("ftSteps").classList.remove("visible");
- document.getElementById("ftStartBtn").disabled = false;
- document.getElementById("ftStartBtn").textContent = "Start Finetune";
- document.getElementById("ftStartBtn").style.display = "";
- document.getElementById("modalCloseBtn").style.display = "";
- var dl = document.getElementById("ftDownload");
- if (dl) dl.remove();
- document.getElementById("ftEvalResults").innerHTML = "";
- var steps = document.querySelectorAll(".modal-step");
- for (var i = 0; i < steps.length; i++) {
- steps[i].classList.remove("active", "done");
- }
-}
-
-async function startFinetune() {
- var apiKey = document.getElementById("ftApiKey").value.trim();
- if (!apiKey) {
- showError("Gemini API key is required");
- return;
- }
-
- var tools = document.getElementById("tools").value.trim() || "[]";
- try {
- JSON.parse(tools);
- } catch (e) {
- showError("Invalid tools JSON");
- return;
- }
-
- var btn = document.getElementById("ftStartBtn");
- btn.disabled = true;
- btn.textContent = "Starting...";
- document.getElementById("modalCloseBtn").style.display = "none";
- document.getElementById("ftSteps").classList.add("visible");
- var steps = document.querySelectorAll(".modal-step");
- for (var i = 0; i < steps.length; i++) steps[i].classList.remove("active", "done");
- _ftRunning = true;
-
- try {
- var r = await fetch("/finetune", {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({
- tools: tools,
- api_key: apiKey,
- }),
- });
- if (!r.ok && !(r.headers.get("content-type") || "").includes("json")) throw new Error("Server error " + r.status);
- var data = await r.json();
- if (data.error) {
- showError(data.error);
- _ftRunning = false;
- _resetModal();
- return;
- }
- _pollTimer = setInterval(pollFinetune, 2000);
- } catch (e) {
- showError("Request failed: " + e.message);
- _ftRunning = false;
- _resetModal();
- }
-}
-
-function _updateSteps(currentStep) {
- var steps = document.querySelectorAll(".modal-step");
- var found = false;
- for (var i = 0; i < steps.length; i++) {
- if (steps[i].getAttribute("data-step") === currentStep) found = true;
- }
- if (!found) return;
-
- var past = true;
- for (var i = 0; i < steps.length; i++) {
- var stepName = steps[i].getAttribute("data-step");
- steps[i].classList.remove("active", "done");
- if (stepName === currentStep) {
- steps[i].classList.add("active");
- past = false;
- } else if (past) {
- steps[i].classList.add("done");
- }
- }
-}
-
-function _renderEvalResults(ft, base) {
- var el = document.getElementById("ftEvalResults");
- if (!ft && !base) { el.innerHTML = ""; return; }
- var metrics = ["call_f1", "name_f1", "exact_match", "parse_rate", "args_acc"];
- var labels = {"call_f1": "Call F1", "name_f1": "Name F1", "exact_match": "Exact Match", "parse_rate": "Parse Rate", "args_acc": "Args Accuracy"};
- var hasBase = base && base.call_f1 != null;
- var rows = metrics.map(function (k) {
- var bv = base && base[k] != null ? (base[k] * 100).toFixed(1) + '%' : '-';
- var fv = ft && ft[k] != null ? (ft[k] * 100).toFixed(1) + '%' : '-';
- if (hasBase) {
- return '<tr><td>' + labels[k] + '</td><td>' + bv + '</td><td>' + fv + '</td></tr>';
- }
- return '<tr><td>' + labels[k] + '</td><td>' + fv + '</td></tr>';
- }).join("");
- var header = hasBase
- ? '<tr><th>Metric</th><th>Base</th><th>Finetuned</th></tr>'
- : '<tr><th>Metric</th><th>Score</th></tr>';
- el.innerHTML =
- '<div class="ft-section-title">Evaluation Results</div>' +
- '<table class="ft-table"><thead>' + header + '</thead><tbody>' + rows + '</tbody></table>';
-}
-
-async function pollFinetune() {
- try {
- var r = await fetch("/finetune/status");
- if (!r.ok) throw new Error("Server error " + r.status);
- var data = await r.json();
-
- _updateSteps(data.step);
-
- var btn = document.getElementById("ftStartBtn");
- var labels = {
- "starting": "Starting...",
- "generating data": "Generating data...",
- "evaluating base": "Training...",
- "training": "Training...",
- "evaluating finetuned": "Training...",
- };
- var label = labels[data.step] || data.step;
- btn.textContent = label;
-
- _renderEvalResults(data.eval_results, data.base_eval);
-
- if (!data.running) {
- clearInterval(_pollTimer);
- _pollTimer = null;
- _ftRunning = false;
- document.getElementById("modalCloseBtn").style.display = "";
-
- if (data.step === "done") {
- var steps = document.querySelectorAll(".modal-step");
- for (var i = 0; i < steps.length; i++) {
- steps[i].classList.remove("active");
- steps[i].classList.add("done");
- }
- btn.style.display = "none";
- if (data.checkpoint) {
- var footer = document.getElementById("ftFooter");
- var dl = document.createElement("a");
- dl.id = "ftDownload";
- dl.className = "modal-download";
- dl.href = "/download/" + data.checkpoint;
- dl.download = data.checkpoint;
- dl.textContent = "Download " + data.checkpoint;
- footer.appendChild(dl);
- }
- } else if (data.step === "failed") {
- var last = data.log.length > 0 ? data.log[data.log.length - 1] : "Unknown error";
- showError("Finetune failed — " + last);
- btn.textContent = "Retry";
- btn.disabled = false;
- }
- }
- } catch (e) {
- clearInterval(_pollTimer);
- _pollTimer = null;
- _ftRunning = false;
- document.getElementById("modalCloseBtn").style.display = "";
- showError("Lost connection to server");
- document.getElementById("ftStartBtn").textContent = "Retry";
- document.getElementById("ftStartBtn").disabled = false;
- }
-}
-
-document.getElementById("query").addEventListener("keydown", function (e) {
- if (e.key === "Enter" && !e.shiftKey) {
- e.preventDefault();
- send();
- }
-});
-
-document.addEventListener("keydown", function (e) {
- if (e.key === "Escape") closeModal();
-});
-
-(function () {
- var handle = document.getElementById("resizeHandle");
- var sidebar = document.getElementById("sidebar");
- var dragging = false;
-
- handle.addEventListener("mousedown", function (e) {
- if (window.innerWidth <= 768) return;
- e.preventDefault();
- dragging = true;
- handle.classList.add("active");
- document.body.style.cursor = "col-resize";
- document.body.style.userSelect = "none";
- });
-
- window.addEventListener("mousemove", function (e) {
- if (!dragging) return;
- sidebar.style.width = Math.min(Math.max(e.clientX, 200), window.innerWidth * 0.6) + "px";
- });
-
- window.addEventListener("mouseup", function () {
- if (!dragging) return;
- dragging = false;
- handle.classList.remove("active");
- document.body.style.cursor = "";
- document.body.style.userSelect = "";
- });
-
- window.addEventListener("resize", function () {
- if (window.innerWidth <= 768) sidebar.style.width = "";
- });
-})();
diff --git a/needle/ui/static/index.html b/needle/ui/static/index.html
deleted file mode 100644
index a06db16..0000000
--- a/needle/ui/static/index.html
+++ /dev/null
@@ -1,101 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" class="dark">
-<head>
- <meta charset="utf-8">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- <title>Needle Playground | Cactus</title>
- <link rel="preconnect" href="https://fonts.googleapis.com">
- <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
- <link href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600;700&family=Geist+Mono:wght@400;500&display=swap" rel="stylesheet">
- <link rel="stylesheet" href="/static/style.css">
-</head>
-<body>
- <header>
- <div class="header-border">
- <div class="header-inner">
- <a class="header-logo" href="https://cactuscompute.com">
- <img src="https://cactuscompute.com/assets/cactus_white.png" alt="Cactus" width="32" height="32">
- <span>Cactus</span>
- </a>
- <nav class="header-nav">
- <span class="nav-current">Needle Playground</span>
- </nav>
- <div class="header-spacer"></div>
- <button class="tools-toggle" onclick="togglePanel()" aria-label="Toggle tools panel">
- <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"/><circle cx="12" cy="12" r="3"/></svg>
- <span>Tools</span>
- </button>
- </div>
- </div>
- </header>
- <main>
- <aside id="sidebar">
- <div class="panel-section panel-tools">
- <div class="panel-header">
- <label class="panel-label">Tools JSON</label>
- <label class="file-btn">
- Upload
- <input type="file" id="toolsFile" accept=".json" onchange="loadToolsFile(this)">
- </label>
- </div>
- <textarea id="tools" spellcheck="false" placeholder="Paste tool definitions here...">[
- {"name": "set_timer", "description": "Set a timer for the specified duration or end time.", "parameters": {"time_human": {"type": "string", "description": "The duration or target end time in human readable format e.g. '1 hour and 30 minutes', '45 minutes', 'for 1:50pm', 'at 13:30'.", "required": true}}},
- {"name": "send_email", "description": "Send an email to a recipient.", "parameters": {"to": {"type": "string", "description": "The recipient's email address.", "required": true}, "subject": {"type": "string", "description": "The email subject line.", "required": true}, "body": {"type": "string", "description": "The email body text.", "required": true}}},
- {"name": "create_note", "description": "Create a new note with the given text.", "parameters": {"text": {"type": "string", "description": "The text of the note.", "required": true}, "title": {"type": "string", "description": "Optional title for the note.", "required": false}}}
-]</textarea>
- <button class="btn-finetune" onclick="openFinetuneModal()">Finetune on these tools</button>
- </div>
- </aside>
- <div class="resize-handle" id="resizeHandle"></div>
- <section class="query-area">
- <div class="query-section">
- <div class="panel-header">
- <label class="panel-label">Query</label>
- <span class="model-name" id="modelName"></span>
- <label class="file-btn">
- Load model
- <input type="file" id="modelFile" accept=".pkl" onchange="loadModelFile(this)">
- </label>
- </div>
- <div class="input-row">
- <input id="query" type="text" placeholder="e.g. Set a 25 minute timer" autofocus>
- <button class="btn-primary" id="sendBtn" onclick="send()">Run</button>
- </div>
- </div>
- <div class="result-section">
- <label class="panel-label">Result</label>
- <pre class="result-box" id="result"></pre>
- </div>
- </section>
- </main>
-
- <div class="modal-overlay" id="modalOverlay">
- <div class="modal" onclick="event.stopPropagation()">
- <div class="modal-header">
- <span class="modal-title">Finetune</span>
- <button class="modal-close" id="modalCloseBtn" onclick="closeModal()">&times;</button>
- </div>
- <div class="modal-body">
- <div class="modal-field">
- <label class="modal-label">Gemini API Key</label>
- <input type="password" id="ftApiKey" class="modal-input" placeholder="Paste your API key" autocomplete="off" spellcheck="false">
- </div>
- <div class="modal-steps" id="ftSteps">
- <div class="modal-step" data-step="generating data"><span class="step-dot"></span>Generating data</div>
- <div class="modal-step" data-step="training"><span class="step-dot"></span>Training</div>
- </div>
- <div class="ft-eval" id="ftEvalResults"></div>
- </div>
- <div class="modal-footer" id="ftFooter">
- <button class="btn-primary modal-start" id="ftStartBtn" onclick="startFinetune()">Start Finetune</button>
- </div>
- </div>
- </div>
-
- <div class="toast" id="toast">
- <span class="toast-msg" id="toastMsg"></span>
- <button class="toast-close" onclick="dismissToast()">&times;</button>
- </div>
- <script src="/static/app.js"></script>
-</body>
-</html>
diff --git a/needle/ui/static/style.css b/needle/ui/static/style.css
deleted file mode 100644
index a9b680c..0000000
--- a/needle/ui/static/style.css
+++ /dev/null
@@ -1,653 +0,0 @@
-:root {
- --background: #000000;
- --foreground: #ffffff;
- --card: #0a0a0a;
- --primary: #ffffff;
- --primary-foreground: #000000;
- --muted-foreground: #a3a3a3;
- --accent: #262626;
- --accent-foreground: #ffffff;
- --destructive: #FF3B30;
- --border: #262626;
- --input: #171717;
- --ring: #262626;
- --radius: 0.5rem;
-}
-
-*, *::before, *::after {
- box-sizing: border-box;
- margin: 0;
- padding: 0;
- border-color: var(--border);
-}
-
-body {
- background: var(--background);
- color: var(--foreground);
- font-family: 'Geist', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
- font-size: 14px;
- line-height: 1.5;
- height: 100vh;
- display: flex;
- flex-direction: column;
- -webkit-font-smoothing: antialiased;
- -moz-osx-font-smoothing: grayscale;
-}
-
-header {
- position: sticky;
- top: 0;
- z-index: 50;
- background: rgba(0, 0, 0, 0.8);
- backdrop-filter: blur(12px);
- -webkit-backdrop-filter: blur(12px);
- flex-shrink: 0;
-}
-
-.header-border {
- border-bottom: 1px solid var(--border);
-}
-
-.header-inner {
- width: 100%;
- height: 64px;
- padding: 0 24px;
- display: flex;
- align-items: center;
-}
-
-.header-logo {
- display: flex;
- align-items: center;
- gap: 8px;
- text-decoration: none;
- color: var(--foreground);
-}
-
-.header-logo img {
- height: 32px;
- width: auto;
- object-fit: contain;
-}
-
-.header-logo span {
- font-size: 18px;
- font-weight: 600;
- letter-spacing: -0.02em;
-}
-
-.header-nav {
- display: flex;
- align-items: center;
- margin-left: 12px;
- padding-left: 12px;
- border-left: 1px solid var(--border);
-}
-
-.nav-current {
- color: var(--muted-foreground);
- font-size: 14px;
-}
-
-.header-spacer {
- flex: 1;
-}
-
-.tools-toggle {
- display: none;
- align-items: center;
- gap: 6px;
- height: 32px;
- padding: 0 12px;
- background: transparent;
- color: var(--muted-foreground);
- border: 1px solid var(--border);
- border-radius: calc(var(--radius) - 2px);
- font-family: inherit;
- font-size: 13px;
- font-weight: 500;
- cursor: pointer;
- transition: all 150ms;
-}
-
-.tools-toggle:hover {
- color: var(--foreground);
- background: var(--accent);
-}
-
-main {
- flex: 1;
- display: flex;
- overflow: hidden;
- width: 100%;
-}
-
-aside {
- width: 340px;
- min-width: 200px;
- max-width: 60vw;
- flex-shrink: 0;
- padding: 20px;
- display: flex;
- flex-direction: column;
- gap: 16px;
- overflow-y: auto;
-}
-
-.resize-handle {
- width: 5px;
- flex-shrink: 0;
- cursor: col-resize;
- background: transparent;
- border-left: 1px solid var(--border);
- transition: background 150ms;
-}
-
-.resize-handle:hover,
-.resize-handle.active {
- background: var(--accent);
-}
-
-.panel-section {
- display: flex;
- flex-direction: column;
- gap: 8px;
-}
-
-.panel-tools {
- flex: 1;
- min-height: 0;
-}
-
-.panel-header {
- display: flex;
- align-items: center;
- justify-content: space-between;
-}
-
-.panel-label {
- font-size: 13px;
- font-weight: 600;
- letter-spacing: -0.01em;
-}
-
-.file-btn {
- font-size: 12px;
- font-weight: 500;
- color: var(--muted-foreground);
- cursor: pointer;
- transition: color 150ms;
-}
-
-.file-btn:hover {
- color: var(--foreground);
-}
-
-.file-btn input {
- display: none;
-}
-
-.model-name {
- font-size: 12px;
- font-family: 'Geist Mono', monospace;
- color: var(--muted-foreground);
-}
-
-.btn-finetune {
- flex-shrink: 0;
- height: 36px;
- padding: 0 14px;
- background: var(--primary);
- color: var(--primary-foreground);
- border: none;
- border-radius: var(--radius);
- font-family: inherit;
- font-size: 13px;
- font-weight: 500;
- cursor: pointer;
- transition: background 150ms;
- width: 100%;
-}
-
-.btn-finetune:hover {
- background: rgba(255, 255, 255, 0.9);
-}
-
-.btn-finetune:disabled {
- opacity: 0.5;
- pointer-events: none;
-}
-
-textarea {
- flex: 1;
- font-family: 'Geist Mono', 'SFMono-Regular', 'Menlo', 'Monaco', 'Consolas', monospace;
- font-size: 13px;
- line-height: 1.6;
- resize: none;
- padding: 12px;
- background: var(--card);
- color: var(--foreground);
- border: 1px solid var(--border);
- border-radius: var(--radius);
- outline: none;
- transition: border-color 150ms;
-}
-
-textarea:focus {
- border-color: var(--ring);
- box-shadow: 0 0 0 3px rgba(38, 38, 38, 0.5);
-}
-
-textarea::placeholder {
- color: var(--muted-foreground);
-}
-
-.query-area {
- flex: 1;
- display: flex;
- flex-direction: column;
- padding: 20px;
- gap: 20px;
- min-width: 0;
-}
-
-.query-section {
- display: flex;
- flex-direction: column;
- gap: 8px;
- flex-shrink: 0;
-}
-
-.input-row {
- display: flex;
- gap: 8px;
-}
-
-.input-row input {
- flex: 1;
- height: 40px;
- padding: 0 14px;
- background: var(--input);
- color: var(--foreground);
- border: 1px solid var(--border);
- border-radius: var(--radius);
- font-family: inherit;
- font-size: 14px;
- outline: none;
- transition: border-color 150ms;
-}
-
-.input-row input:focus {
- border-color: var(--ring);
- box-shadow: 0 0 0 3px rgba(38, 38, 38, 0.5);
-}
-
-.input-row input::placeholder {
- color: var(--muted-foreground);
-}
-
-.btn-primary {
- height: 40px;
- padding: 0 20px;
- background: var(--primary);
- color: var(--primary-foreground);
- border: none;
- border-radius: var(--radius);
- font-family: inherit;
- font-size: 14px;
- font-weight: 500;
- cursor: pointer;
- transition: background 150ms;
- white-space: nowrap;
-}
-
-.btn-primary:hover {
- background: rgba(255, 255, 255, 0.9);
-}
-
-.btn-primary:disabled {
- opacity: 0.5;
- pointer-events: none;
-}
-
-.result-section {
- flex: 1;
- min-height: 0;
- display: flex;
- flex-direction: column;
- gap: 8px;
-}
-
-.result-box {
- flex: 1;
- margin: 0;
- padding: 14px;
- background: var(--card);
- border: 1px solid var(--border);
- border-radius: var(--radius);
- font-family: 'Geist Mono', 'SFMono-Regular', 'Menlo', 'Monaco', 'Consolas', monospace;
- font-size: 13px;
- line-height: 1.6;
- color: var(--muted-foreground);
- white-space: pre-wrap;
- word-break: break-word;
- overflow-y: auto;
-}
-
-.result-box.has-result {
- color: var(--foreground);
-}
-
-.modal-overlay {
- position: fixed;
- inset: 0;
- z-index: 60;
- display: none;
- align-items: center;
- justify-content: center;
- background: rgba(0, 0, 0, 0.6);
- padding: 16px;
-}
-
-.modal-overlay.visible {
- display: flex;
-}
-
-.modal {
- width: 100%;
- max-width: 520px;
- max-height: 85vh;
- background: var(--card);
- border: 1px solid var(--border);
- border-radius: 12px;
- display: flex;
- flex-direction: column;
- overflow: hidden;
-}
-
-.modal-header {
- display: flex;
- align-items: center;
- justify-content: space-between;
- padding: 16px 20px;
- border-bottom: 1px solid var(--border);
- flex-shrink: 0;
-}
-
-.modal-title {
- font-size: 15px;
- font-weight: 600;
-}
-
-.modal-close {
- width: 28px;
- height: 28px;
- display: flex;
- align-items: center;
- justify-content: center;
- background: none;
- border: none;
- color: var(--muted-foreground);
- font-size: 20px;
- cursor: pointer;
- border-radius: calc(var(--radius) - 2px);
- transition: color 150ms, background 150ms;
- padding: 0;
- line-height: 1;
-}
-
-.modal-close:hover {
- color: var(--foreground);
- background: var(--accent);
-}
-
-.modal-body {
- padding: 20px;
- overflow-y: auto;
- display: flex;
- flex-direction: column;
- gap: 16px;
-}
-
-.modal-field {
- display: flex;
- flex-direction: column;
- gap: 6px;
- flex: 1;
-}
-
-.modal-label {
- font-size: 13px;
- font-weight: 500;
- color: var(--muted-foreground);
-}
-
-.modal-input {
- height: 36px;
- padding: 0 12px;
- background: var(--input);
- color: var(--foreground);
- border: 1px solid var(--border);
- border-radius: var(--radius);
- font-family: inherit;
- font-size: 13px;
- outline: none;
- transition: border-color 150ms;
- width: 100%;
-}
-
-.modal-input:focus {
- border-color: var(--ring);
- box-shadow: 0 0 0 3px rgba(38, 38, 38, 0.5);
-}
-
-.modal-input::placeholder {
- color: var(--muted-foreground);
-}
-
-.modal-input-sm {
- font-family: 'Geist Mono', monospace;
-}
-
-.modal-row {
- display: flex;
- gap: 12px;
-}
-
-.modal-steps {
- display: none;
- gap: 4px;
-}
-
-.modal-steps.visible {
- display: flex;
- flex-direction: column;
-}
-
-.modal-step {
- display: flex;
- align-items: center;
- gap: 8px;
- font-size: 13px;
- color: var(--muted-foreground);
- padding: 4px 0;
-}
-
-.modal-step.active {
- color: var(--foreground);
-}
-
-.modal-step.done {
- color: #4ade80;
-}
-
-.step-dot {
- width: 8px;
- height: 8px;
- border-radius: 50%;
- background: var(--accent);
- flex-shrink: 0;
- transition: background 200ms;
-}
-
-.modal-step.active .step-dot {
- background: var(--foreground);
-}
-
-.modal-step.done .step-dot {
- background: #4ade80;
-}
-
-.ft-progress {
- min-height: 20px;
- margin-top: 4px;
- color: var(--foreground);
- font-family: 'Geist Mono', monospace;
- font-size: 13px;
-}
-
-.modal-footer {
- padding: 16px 20px;
- border-top: 1px solid var(--border);
- flex-shrink: 0;
-}
-
-.modal-start {
- width: 100%;
-}
-
-.modal-download {
- display: block;
- width: 100%;
- height: 40px;
- line-height: 40px;
- text-align: center;
- background: var(--primary);
- color: var(--primary-foreground);
- border-radius: var(--radius);
- font-family: inherit;
- font-size: 14px;
- font-weight: 500;
- text-decoration: none;
- transition: background 150ms;
-}
-
-.modal-download:hover {
- background: rgba(255, 255, 255, 0.9);
-}
-
-.toast {
- position: fixed;
- top: 80px;
- right: 24px;
- display: flex;
- align-items: flex-start;
- gap: 10px;
- max-width: 420px;
- padding: 12px 14px;
- background: #1a0000;
- border: 1px solid rgba(255, 59, 48, 0.3);
- border-radius: var(--radius);
- color: #ff6b6b;
- font-size: 13px;
- line-height: 1.5;
- z-index: 100;
- opacity: 0;
- transform: translateX(20px);
- pointer-events: none;
- transition: opacity 200ms, transform 200ms;
-}
-
-.toast.visible {
- opacity: 1;
- transform: translateX(0);
- pointer-events: auto;
-}
-
-.toast-msg {
- flex: 1;
- word-break: break-word;
-}
-
-.toast-close {
- flex-shrink: 0;
- width: 20px;
- height: 20px;
- display: flex;
- align-items: center;
- justify-content: center;
- background: none;
- border: none;
- color: #ff6b6b;
- font-size: 16px;
- cursor: pointer;
- opacity: 0.6;
- transition: opacity 150ms;
- padding: 0;
- line-height: 1;
-}
-
-.toast-close:hover {
- opacity: 1;
-}
-
-.ft-validation, .ft-eval { margin-top: 12px; }
-.ft-section-title { font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted-foreground); margin-bottom: 8px; }
-.ft-stat-row { display: flex; gap: 16px; font-size: 13px; color: var(--foreground); margin-bottom: 6px; flex-wrap: wrap; }
-.ft-stat-row span { white-space: nowrap; }
-.ft-warn { color: var(--destructive); font-size: 12px; margin-bottom: 6px; }
-.ft-table { width: 100%; border-collapse: collapse; font-size: 12px; margin-top: 6px; }
-.ft-table th { text-align: left; color: var(--muted-foreground); font-weight: 500; padding: 4px 8px; border-bottom: 1px solid var(--border); }
-.ft-table td { padding: 3px 8px; border-bottom: 1px solid var(--border); }
-.ft-table tbody tr:last-child td { border-bottom: none; }
-
-::-webkit-scrollbar { width: 6px; }
-::-webkit-scrollbar-track { background: transparent; }
-::-webkit-scrollbar-thumb { background: var(--accent); border-radius: 3px; }
-
-@media (max-width: 768px) {
- .header-inner { padding: 0 16px; height: 56px; }
- .header-nav { display: none; }
- .tools-toggle { display: flex; }
-
- main { flex-direction: column; min-height: 0; }
-
- .resize-handle { display: none !important; }
-
- aside {
- width: 100% !important;
- min-width: 0 !important;
- max-width: none !important;
- flex-shrink: 0;
- border-right: none;
- display: none !important;
- padding: 16px;
- overflow-y: auto;
- }
-
- aside.open {
- display: flex !important;
- flex: 1;
- max-height: none;
- }
-
- aside.open .panel-tools { flex: 1; min-height: 0; }
- aside.open textarea { flex: 1; min-height: 0; }
-
- aside.open ~ .resize-handle,
- aside.open ~ .query-area { display: none !important; }
-
- .query-area { flex: 1; min-height: 0; padding: 16px; gap: 16px; }
- .result-section { flex: 1; min-height: 0; }
-
- .modal { max-width: none; max-height: none; height: 100%; border-radius: 0; }
- .modal-overlay { padding: 0; }
-}
-
-@media (max-width: 480px) {
- .header-inner { padding: 0 12px; }
- .header-logo span { display: none; }
- aside.open { padding: 12px; }
- .query-area { padding: 12px; }
- .btn-primary { padding: 0 14px; }
-}
diff --git a/needle/utils/__init__.py b/needle/utils/__init__.py
deleted file mode 100644
index 031bbde..0000000
--- a/needle/utils/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from .distributed import ( # noqa: F401
- _replicate, _unreplicate, shard_batch, _upload_checkpoint,
- load_pretrained_params, partial_load_params,
-)
diff --git a/needle/utils/distributed.py b/needle/utils/distributed.py
deleted file mode 100644
index 29bce05..0000000
--- a/needle/utils/distributed.py
+++ /dev/null
@@ -1,166 +0,0 @@
-import os
-import pickle
-import threading
-import jax
-import jax.numpy as jnp
-import numpy as np
-
-_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"
-
-
-def _download_checkpoint(name_or_path):
- """Return a local path to a checkpoint, downloading from HF if needed.
-
- If `name_or_path` already exists on disk, it's returned unchanged.
- Otherwise treats the basename as a file in `Cactus-Compute/checkpoints`.
- """
- if os.path.exists(name_or_path):
- return name_or_path
- from huggingface_hub import hf_hub_download
- local_dir = os.path.dirname(name_or_path) or "checkpoints"
- os.makedirs(local_dir, exist_ok=True)
- print(f" Downloading {os.path.basename(name_or_path)} from {_HF_CHECKPOINT_REPO}...", flush=True)
- local_path = hf_hub_download(
- repo_id=_HF_CHECKPOINT_REPO,
- filename=os.path.basename(name_or_path),
- repo_type="model",
- local_dir=local_dir,
- )
- print(f" Downloaded to {local_path}", flush=True)
- return local_path
-
-
-def _flatten_params(tree, prefix=()):
- """Flatten a nested dict of params into {tuple-path: leaf}."""
- out = {}
- if isinstance(tree, dict):
- for k, v in tree.items():
- out.update(_flatten_params(v, prefix + (k,)))
- else:
- out[prefix] = tree
- return out
-
-
-def _unflatten_params(flat):
- """Inverse of _flatten_params: rebuild a nested dict."""
- root = {}
- for path, leaf in flat.items():
- node = root
- for k in path[:-1]:
- node = node.setdefault(k, {})
- node[path[-1]] = leaf
- return root
-
-
-def partial_load_params(init_params, loaded_params):
- """Merge `loaded_params` into `init_params` by matching path + shape.
-
- For every leaf in `init_params`:
- - if the same path exists in `loaded_params` and shapes agree,
- replace it with the loaded value (cast to the init leaf's dtype);
- - otherwise keep the init leaf (random init).
-
- Extra leaves in `loaded_params` (not present in `init_params`) are ignored.
-
- Returns (merged_params, stats_dict).
- """
- init_flat = _flatten_params(init_params)
- loaded_flat = _flatten_params(loaded_params)
-
- merged = {}
- n_loaded = 0
- n_random = 0
- shape_mismatches = []
- missing_in_ckpt = []
- for path, init_leaf in init_flat.items():
- loaded_leaf = loaded_flat.get(path)
- if loaded_leaf is None:
- merged[path] = init_leaf
- n_random += 1
- missing_in_ckpt.append(path)
- continue
- if tuple(loaded_leaf.shape) != tuple(init_leaf.shape):
- merged[path] = init_leaf
- n_random += 1
- shape_mismatches.append((path, tuple(loaded_leaf.shape), tuple(init_leaf.shape)))
- continue
- merged[path] = jnp.asarray(loaded_leaf, dtype=init_leaf.dtype)
- n_loaded += 1
-
- extra_in_ckpt = [p for p in loaded_flat.keys() if p not in init_flat]
-
- stats = {
- "loaded": n_loaded,
- "random_init": n_random,
- "shape_mismatches": shape_mismatches,
- "missing_in_ckpt": missing_in_ckpt,
- "extra_in_ckpt": extra_in_ckpt,
- }
- return _unflatten_params(merged), stats
-
-
-def load_pretrained_params(path_or_name):
- """Download (if needed) + pickle.load a checkpoint, return its params dict."""
- local = _download_checkpoint(path_or_name)
- with open(local, "rb") as f:
- data = pickle.load(f)
- return data["params"], data.get("config", None), local
-
-
-def _replicate(tree):
- """Replicate a pytree across all local devices (multi-host safe)."""
- devices = np.array(jax.local_devices())
- n = len(devices)
- mesh = jax.sharding.Mesh(devices, ("d",))
- sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec("d"))
-
- def _rep(x):
- x = jnp.asarray(x)
- return jax.device_put(jnp.broadcast_to(x, (n,) + x.shape), sharding)
-
- return jax.tree.map(_rep, tree)
-
-
-def _unreplicate(tree):
- """Get a single copy from a replicated pytree (multi-host safe).
-
- Uses addressable_shards instead of x[0] indexing, which fails
- in multi-host pmap because the array spans devices on other hosts.
- Each shard has shape (1, ...) — we index [0] to strip the leading dim.
- """
- def _get_first(x):
- if hasattr(x, 'addressable_shards') and x.addressable_shards:
- shard = x.addressable_shards[0].data
- # Shard has shape (1, ...) from the pmap partition; strip it
- return jax.device_get(shard[0]) if shard.ndim > 0 else jax.device_get(shard)
- return x
- return jax.tree.map(_get_first, tree)
-
-
-def shard_batch(batch, num_devices):
- """Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""
- return batch.reshape(num_devices, -1, *batch.shape[1:])
-
-
-def _upload_checkpoint(ckpt_path):
- """Upload a checkpoint file to HuggingFace Hub in a background thread."""
- import threading
-
- def _upload():
- try:
- from huggingface_hub import HfApi
- api = HfApi()
- api.create_repo(_HF_CHECKPOINT_REPO, repo_type="model", private=True, exist_ok=True)
- filename = os.path.basename(ckpt_path)
- print(f"[hf] Uploading {filename} to {_HF_CHECKPOINT_REPO} ...")
- api.upload_file(
- path_or_fileobj=ckpt_path,
- path_in_repo=filename,
- repo_id=_HF_CHECKPOINT_REPO,
- repo_type="model",
- )
- print(f"[hf] Checkpoint uploaded: {_HF_CHECKPOINT_REPO}/{filename}")
- except Exception as e:
- print(f"[hf] Warning: checkpoint upload failed: {e}")
-
- threading.Thread(target=_upload, daemon=True).start()
diff --git a/needle/utils/gcs.py b/needle/utils/gcs.py
deleted file mode 100644
index 0323ac3..0000000
--- a/needle/utils/gcs.py
+++ /dev/null
@@ -1,97 +0,0 @@
-"""GCS sync utilities for persisting datasets to gs://cactus-dataset.
-
-Uploads and downloads three artifact types:
- - synth_tool_calls/ synthesized tool-calling dataset (.arrow files)
- - tokenized_data/ pre-tokenized .npy arrays + metadata JSONs
- - tokenizer/ SentencePiece model + vocab
-
-Uses `gcloud storage` CLI (authenticated via `gcloud auth login`).
-"""
-
-import os
-import subprocess
-
-BUCKET = "cactus-dataset"
-
-_RAW_DATA_PREFIX = "raw_data"
-_SYNTH_DATA_PREFIX = "tool_calls"
-_TOKENIZED_DATA_PREFIX = "tokenized_data"
-_TOKENIZER_PREFIX = "tokenizer"
-
-
-def upload_directory(local_dir, gcs_prefix):
- """Upload all files in local_dir to gs://BUCKET/gcs_prefix/.
-
- Clears the remote prefix first to avoid stale files from previous uploads.
- """
- if not os.path.isdir(local_dir):
- print(f"[GCS] Skipping upload: {local_dir} does not exist")
- return
- dest = f"gs://{BUCKET}/{gcs_prefix}/"
- print(f"[GCS] Clearing {dest} ...")
- subprocess.run(
- ["gcloud", "storage", "rm", "-r", dest],
- capture_output=True,
- )
- print(f"[GCS] Uploading {local_dir} -> {dest}")
- subprocess.run(
- ["gcloud", "storage", "cp", "-r", f"{local_dir}/*", dest],
- check=True,
- )
- print(f"[GCS] Upload complete: {dest}")
-
-
-def download_directory(gcs_prefix, local_dir):
- """Download gs://BUCKET/gcs_prefix/ to local_dir. Returns True if successful."""
- src = f"gs://{BUCKET}/{gcs_prefix}/"
- os.makedirs(local_dir, exist_ok=True)
- print(f"[GCS] Downloading {src} -> {local_dir}")
- result = subprocess.run(
- ["gcloud", "storage", "cp", "-r", f"{src}*", local_dir],
- capture_output=True, text=True,
- )
- if result.returncode != 0:
- if "not found" in result.stderr.lower() or "CommandException" in result.stderr:
- print(f"[GCS] Nothing found at {src}")
- return False
- raise RuntimeError(f"gcloud storage download failed: {result.stderr}")
- print(f"[GCS] Download complete: {local_dir}")
- return True
-
-
-# ── Convenience wrappers ──
-
-
-def upload_raw_data(local_dir):
- """Upload raw tool-calling dataset to GCS."""
- upload_directory(local_dir, _RAW_DATA_PREFIX)
-
-
-def download_raw_data(local_dir):
- """Download raw tool-calling dataset from GCS. Returns True if successful."""
- return download_directory(_RAW_DATA_PREFIX, local_dir)
-
-
-def download_synth_data(local_dir):
- """Download synthesized tool-calling dataset from GCS. Returns True if successful."""
- return download_directory(_SYNTH_DATA_PREFIX, local_dir)
-
-
-def upload_tokenized_data(cache_dir):
- """Upload tokenized .npy files and metadata to GCS."""
- upload_directory(cache_dir, _TOKENIZED_DATA_PREFIX)
-
-
-def download_tokenized_data(cache_dir):
- """Download tokenized data from GCS. Returns True if successful."""
- return download_directory(_TOKENIZED_DATA_PREFIX, cache_dir)
-
-
-def upload_tokenizer(tokenizer_dir):
- """Upload tokenizer model and vocab to GCS."""
- upload_directory(tokenizer_dir, _TOKENIZER_PREFIX)
-
-
-def download_tokenizer(tokenizer_dir):
- """Download tokenizer from GCS. Returns True if successful."""
- return download_directory(_TOKENIZER_PREFIX, tokenizer_dir)
diff --git a/needle/utils/tpu.py b/needle/utils/tpu.py
deleted file mode 100644
index 5b7ecf7..0000000
--- a/needle/utils/tpu.py
+++ /dev/null
@@ -1,665 +0,0 @@
-import getpass
-import os
-import re
-import subprocess
-import sys
-
-def _get_project():
- """Get GCP project from gcloud config or NEEDLE_GCP_PROJECT env var."""
- proj = os.environ.get("NEEDLE_GCP_PROJECT")
- if proj:
- return proj
- result = subprocess.run(
- ["gcloud", "config", "get-value", "project"],
- capture_output=True, text=True,
- )
- proj = result.stdout.strip()
- if proj and proj != "(unset)":
- return proj
- print(
- "[tpu] ERROR: no GCP project set. Either run 'gcloud config set project <PROJECT>' "
- "or set NEEDLE_GCP_PROJECT environment variable.",
- file=sys.stderr,
- )
- sys.exit(1)
-
-
-PROJECT = None # resolved lazily
-
-VERSION_FOR_TYPE = {
- "v6e": "v2-alpha-tpuv6e",
-}
-
-
-def _resolve_version(accel_type, explicit_version):
- """Pick the right software version for the accelerator type.
-
- If the user explicitly passed --version, use that.
- Otherwise, match on the accelerator type prefix (e.g. 'v6e-4' → 'v6e').
- """
- if explicit_version is not None:
- return explicit_version
- for prefix, version in sorted(VERSION_FOR_TYPE.items(), key=lambda kv: -len(kv[0])):
- if accel_type.startswith(prefix):
- return version
-
- print(
- f"[tpu] WARNING: unknown accelerator type '{accel_type}', "
- f"defaulting to tpu-ubuntu2204-base. Pass --version to override.",
- file=sys.stderr,
- )
- return "tpu-ubuntu2204-base"
-
-
-# Trillium (v6e) zones with quota, ordered by on-demand price (cheapest first)
-# us-east/us-central/us-south/us-west: $2.70/chip/hr
-# europe-west4: $2.97/chip/hr
-# asia: $3.24/chip/hr
-ZONES = [
- "us-east1-d",
- "us-east5-a", "us-east5-b",
- "us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f",
- "us-east4-c",
- "us-south1-a", "us-south1-b", "us-south1-c",
- "us-west1-a", "us-west1-b", "us-west1-c",
- "europe-west4-a", "europe-west4-b", "europe-west4-c",
- "asia-east1-a", "asia-east1-b", "asia-east1-c",
- "asia-northeast1-a", "asia-northeast1-b", "asia-northeast1-c",
- "asia-south1-a", "asia-south1-b", "asia-south1-c",
- "asia-southeast1-a", "asia-southeast1-b", "asia-southeast1-c",
-]
-
-TPU_HELP = """
- tpu commands:
- needle tpu create NAME [--type TYPE] [--version VER]
- needle tpu connect NAME [--zone ZONE]
- needle tpu setup NAME [--zone ZONE] setup all workers (multi-host)
- needle tpu sync NAME [--zone ZONE] sync code only (fast, no venv rebuild)
- needle tpu train NAME [--zone ZONE] launch training on all workers
- needle tpu pretrain NAME [--zone ZONE] launch pretraining on all workers
- needle tpu claude NAME [--zone ZONE]
- needle tpu stop NAME [--zone ZONE]
- needle tpu start NAME [--zone ZONE]
- needle tpu delete NAME [--zone ZONE]
- needle tpu list
-"""
-
-
-def _run(cmd, check=True, capture=False, quiet=False):
- if not quiet:
- print(f"[tpu] $ {' '.join(cmd)}")
- try:
- result = subprocess.run(cmd, capture_output=capture, text=True)
- except FileNotFoundError:
- print(
- "[tpu] ERROR: 'gcloud' not found. "
- "Install: https://cloud.google.com/sdk/docs/install",
- file=sys.stderr,
- )
- sys.exit(1)
- if check and result.returncode != 0:
- if capture and result.stderr:
- print(result.stderr.strip(), file=sys.stderr)
- sys.exit(result.returncode)
- return result
-
-
-def _detect_zone(name):
- print(f"[tpu] Searching for '{name}'...")
- for zone in ZONES:
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "describe", name,
- "--zone", zone, "--project", PROJECT,
- "--format", "value(name)"],
- check=False, capture=True, quiet=True,
- )
- if result.returncode == 0 and result.stdout.strip():
- print(f"[tpu] Found '{name}' in {zone}")
- return zone
- print(
- f"[tpu] ERROR: instance '{name}' not found. "
- "Run 'needle tpu list' to see instances.",
- file=sys.stderr,
- )
- sys.exit(1)
-
-
-def _update_ssh_config(path, host_name, new_block):
- os.makedirs(os.path.dirname(path), exist_ok=True)
- if os.path.exists(path):
- with open(path, "r") as f:
- content = f.read()
- pattern = rf"\n?Host {re.escape(host_name)}\n(?: .*\n)*"
- content = re.sub(pattern, "", content)
- else:
- content = ""
- with open(path, "w") as f:
- f.write(content.rstrip("\n") + "\n" if content.strip() else "")
- f.write(new_block)
-
-
-def _is_multihost(accel_type):
- """Check if a TPU accelerator type requires multi-host setup.
-
- v6e-1/4/8 are single-host. v6e-16+ are multi-host.
- """
- m = re.search(r"(\d+)$", accel_type)
- if not m:
- return False
- num_chips = int(m.group(1))
- return num_chips > 8
-
-
-def _get_num_workers(name, zone):
- """Get the number of workers for a TPU VM."""
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "describe", name,
- "--zone", zone, "--project", PROJECT,
- "--format", "json(networkEndpoints)"],
- check=False, capture=True, quiet=True,
- )
- import json
- try:
- data = json.loads(result.stdout)
- return len(data.get("networkEndpoints", []))
- except (json.JSONDecodeError, TypeError):
- return 1
-
-
-def _run_all_workers(name, zone, command):
- """Run a command on all workers of a TPU VM."""
- return _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", name,
- "--zone", zone, "--project", PROJECT,
- "--worker=all",
- "--command", command],
- check=False,
- )
-
-
-def _check_tpu_health(name, zone):
- """SSH into the VM and verify /dev/accel* devices exist."""
- print(f"[tpu] Verifying TPU devices on '{name}'...")
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", name,
- "--zone", zone, "--project", PROJECT,
- "--command", "ls /dev/accel* 2>/dev/null && echo TPU_OK || echo TPU_MISSING"],
- check=False, capture=True, quiet=True,
- )
- output = result.stdout.strip()
- if "TPU_OK" in output:
- devices = [l for l in output.splitlines() if l.startswith("/dev/accel")]
- print(f"[tpu] TPU healthy: {len(devices)} device(s) found")
- return True
- else:
- print(
- f"[tpu] WARNING: no /dev/accel* devices found. "
- f"The software version may not match the accelerator type.",
- file=sys.stderr,
- )
- return False
-
-
-def _collect_git_config():
- """Read git user.name and user.email from local git config."""
- name = ""
- email = ""
- try:
- name = subprocess.run(
- ["git", "config", "user.name"], capture_output=True, text=True
- ).stdout.strip()
- email = subprocess.run(
- ["git", "config", "user.email"], capture_output=True, text=True
- ).stdout.strip()
- except FileNotFoundError:
- pass
-
- if not name or not email:
- print("[tpu] Skipping git config (user.name or user.email not set locally).")
- return None, None
- print(f"[tpu] Using git config: {name} <{email}>")
- return name, email
-
-
-def _setup_git_on_instance(name, zone, git_name, git_email):
- """SSH into the VM and configure git user.name/email."""
- if not git_name or not git_email:
- return
- cmd = (
- f'git config --global user.name "{git_name}" && '
- f'git config --global user.email "{git_email}"'
- )
- print(f"[tpu] Configuring git as '{git_name} <{git_email}>'...")
- _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", name,
- "--zone", zone, "--project", PROJECT,
- "--command", cmd],
- check=False, capture=True, quiet=True,
- )
-
-
-def tpu_create(args):
- version = _resolve_version(args.accel_type, args.version)
- preemptible = getattr(args, "preemptible", False)
- print(f"[tpu] Accelerator: {args.accel_type}, software version: {version}"
- f"{', preemptible' if preemptible else ''}")
-
- git_name, git_email = _collect_git_config()
-
- for zone in ZONES:
- print(f"[tpu] Trying {zone}...")
- cmd = ["gcloud", "compute", "tpus", "tpu-vm", "create", args.name,
- "--zone", zone,
- "--accelerator-type", args.accel_type,
- "--version", version,
- "--project", PROJECT]
- if preemptible:
- cmd.append("--preemptible")
- result = _run(cmd,
- check=False, capture=True,
- )
- if result.returncode == 0:
- print(f"[tpu] SUCCESS: created '{args.name}' in {zone}")
- args.zone = zone
- _check_tpu_health(args.name, zone)
- if _is_multihost(args.accel_type):
- print(f"[tpu] Multi-host TPU detected ({args.accel_type})")
- _setup_git_on_instance(args.name, zone, git_name, git_email)
- tpu_setup(args)
- tpu_claude(args)
- else:
- _setup_git_on_instance(args.name, zone, git_name, git_email)
- tpu_claude(args)
- tpu_connect(args)
- return
- stderr = result.stderr.strip()
- if not stderr:
- stderr = result.stdout.strip()
-
- if stderr and any(phrase in stderr.lower() for phrase in [
- "active account selected",
- "gcloud auth login",
- "could not load the default credentials",
- "reauth",
- ]):
- print(f"[tpu] AUTH ERROR:\n{stderr}", file=sys.stderr)
- sys.exit(1)
- # extract "message" from JSON error, or fall back to first ERROR line
- msg = "unknown error"
- if stderr:
- m = re.search(r'"message":\s*"(.+?)(?<!\\)"', stderr)
- if m:
- msg = m.group(1).replace('\\"', '"')
- else:
- for line in stderr.splitlines():
- if "ERROR" in line:
- msg = line
- break
- else:
- msg = stderr.splitlines()[-1]
- print(f"[tpu] {zone}: {msg}")
- print(
- f"[tpu] ERROR: could not create '{args.name}' in any zone.",
- file=sys.stderr,
- )
- print(
- f"[tpu] Check quota: "
- f"https://console.cloud.google.com/iam-admin/quotas?project={PROJECT}",
- file=sys.stderr,
- )
- sys.exit(1)
-
-
-def tpu_connect(args):
- zone = args.zone or _detect_zone(args.name)
-
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
- "--zone", zone, "--project", PROJECT, "--dry-run"],
- capture=True,
- )
-
- ip_match = re.search(
- r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})",
- result.stdout + result.stderr,
- )
- if not ip_match:
- print("[tpu] ERROR: could not parse IP from dry-run output.", file=sys.stderr)
- print(f"[tpu] stdout: {result.stdout}", file=sys.stderr)
- sys.exit(1)
- ip = ip_match.group(1)
- print(f"[tpu] Detected IP: {ip}")
-
- ssh_config_path = os.path.expanduser("~/.ssh/config")
- user = getpass.getuser()
- block = (
- f"\nHost {args.name}\n"
- f" HostName {ip}\n"
- f" User {user}\n"
- f" IdentityFile ~/.ssh/google_compute_engine\n"
- f" CheckHostIP no\n"
- f" StrictHostKeyChecking no\n"
- )
- _update_ssh_config(ssh_config_path, args.name, block)
- print(f"[tpu] Updated {ssh_config_path} with host '{args.name}'")
-
- print(f"[tpu] Connecting to {args.name} (this propagates SSH keys)...")
- subprocess.run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
- "--zone", zone, "--project", PROJECT],
- )
-
-
-def tpu_stop(args):
- zone = args.zone or _detect_zone(args.name)
- _run(["gcloud", "compute", "tpus", "tpu-vm", "stop", args.name,
- "--zone", zone, "--project", PROJECT])
- print(f"[tpu] Stopped '{args.name}'")
-
-
-def _update_ssh_config_for(args):
- """Fetch the current external IP and update ~/.ssh/config."""
- zone = args.zone
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "describe", args.name,
- "--zone", zone, "--project", PROJECT,
- "--format", "get(networkEndpoints[0].accessConfig.externalIp)"],
- capture=True,
- )
- ip = result.stdout.strip()
- if not ip:
- print("[tpu] WARNING: could not get external IP, SSH config not updated.",
- file=sys.stderr)
- return
- print(f"[tpu] New IP: {ip}")
- ssh_config_path = os.path.expanduser("~/.ssh/config")
- user = getpass.getuser()
- block = (
- f"\nHost {args.name}\n"
- f" HostName {ip}\n"
- f" User {user}\n"
- f" IdentityFile ~/.ssh/google_compute_engine\n"
- f" CheckHostIP no\n"
- f" StrictHostKeyChecking no\n"
- )
- _update_ssh_config(ssh_config_path, args.name, block)
- print(f"[tpu] Updated SSH config for '{args.name}'")
-
-
-def tpu_start(args):
- zone = args.zone or _detect_zone(args.name)
- _run(["gcloud", "compute", "tpus", "tpu-vm", "start", args.name,
- "--zone", zone, "--project", PROJECT])
- print(f"[tpu] Started '{args.name}'")
- args.zone = zone
- _update_ssh_config_for(args)
-
-
-def tpu_delete(args):
- zone = args.zone or _detect_zone(args.name)
- answer = input(f"[tpu] Delete '{args.name}' in {zone}? This is permanent. [y/N] ")
- if answer.lower() != "y":
- print("[tpu] Aborted.")
- return
- _run(["gcloud", "compute", "tpus", "tpu-vm", "delete", args.name,
- "--zone", zone, "--project", PROJECT, "--quiet"])
- print(f"[tpu] Deleted '{args.name}'")
-
-
-def tpu_claude(args):
- zone = args.zone or _detect_zone(args.name)
- setup_script = (
- "curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - && "
- "sudo apt-get install -y nodejs && "
- "sudo npm install -g @anthropic-ai/claude-code && "
- "echo '[tpu] Claude Code installed. Run: claude'"
- )
- print(f"[tpu] Installing Claude Code on '{args.name}'...")
- _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
- "--zone", zone, "--project", PROJECT,
- "--command", setup_script],
- )
-
-
-def tpu_list(args):
- found = False
- for zone in ZONES:
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "list",
- "--zone", zone, "--project", PROJECT],
- check=False, capture=True, quiet=True,
- )
- if result.returncode == 0 and result.stdout.strip():
- if not found:
- header = result.stdout.strip().splitlines()[0]
- print(f"{header} ZONE")
- found = True
- for line in result.stdout.strip().splitlines()[1:]:
- print(f"{line} {zone}")
- if not found:
- print("[tpu] No instances found.")
-
-
-def _sync_code_to_workers(name, zone, num_workers):
- """Tarball source code and SCP to all workers. Returns True on success.
-
- Preserves .venv, .data_cache, and checkpoints on workers if they exist.
- """
- import tempfile
- project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
- tarball = os.path.join(tempfile.gettempdir(), "needle_sync.tar.gz")
- print(f"[tpu] Creating tarball (excluding .venv, .data_cache, .git)...")
- _run(
- ["tar", "-czf", tarball,
- "--exclude=.venv", "--exclude=.data_cache",
- "--exclude=__pycache__", "--exclude=.git",
- "--exclude=*.pyc", "--exclude=checkpoints",
- "-C", os.path.dirname(project_root),
- os.path.basename(project_root)],
- )
- tar_size = os.path.getsize(tarball)
- print(f"[tpu] Tarball: {tar_size / 1024:.0f} KB")
-
- print(f"[tpu] Copying to all {num_workers} workers...")
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "scp",
- tarball, f"{name}:/tmp/needle_sync.tar.gz",
- "--zone", zone, "--project", PROJECT,
- "--worker=all"],
- check=False,
- )
- os.remove(tarball)
- if result.returncode != 0:
- print("[tpu] ERROR: failed to copy tarball to workers.", file=sys.stderr)
- return False
-
- print(f"[tpu] Extracting on all {num_workers} workers...")
- extract_cmd = (
- "set -e; "
- "cd ~; "
- # Save heavy dirs if they exist
- "for d in .venv .data_cache checkpoints; do "
- " if [ -d ~/needle/$d ]; then mv ~/needle/$d /tmp/needle_save_$d; fi; "
- "done; "
- # Remove old code and extract new
- "rm -rf ~/needle; "
- "tar -xzf /tmp/needle_sync.tar.gz -C ~; "
- "rm -f /tmp/needle_sync.tar.gz; "
- # Restore heavy dirs
- "for d in .venv .data_cache checkpoints; do "
- " if [ -d /tmp/needle_save_$d ]; then mv /tmp/needle_save_$d ~/needle/$d; fi; "
- "done"
- )
- result = _run_all_workers(name, zone, extract_cmd)
- if result.returncode != 0:
- print("[tpu] ERROR: failed to extract on workers.", file=sys.stderr)
- return False
- return True
-
-
-def tpu_setup(args):
- """Set up all workers of a multi-host TPU: sync code, install deps.
-
- Creates a lightweight tarball (excludes .venv, .data_cache, __pycache__,
- .git), copies it to all workers via SCP, then runs `source ./setup`
- on each worker to create a fresh venv with jax[tpu].
- """
- zone = args.zone or _detect_zone(args.name)
- num_workers = _get_num_workers(args.name, zone)
-
- if num_workers <= 1:
- print(f"[tpu] Single-host TPU ({num_workers} worker). Use 'needle tpu connect' instead.")
- return
-
- print(f"[tpu] Multi-host TPU: {num_workers} workers")
-
- if not _sync_code_to_workers(args.name, zone, num_workers):
- sys.exit(1)
-
- # Full setup: recreate venv + install jax[tpu]
- print(f"[tpu] Running full setup on all {num_workers} workers...")
- result = _run_all_workers(args.name, zone,
- "cd ~/needle && rm -rf .venv && source ./setup")
- if result.returncode != 0:
- print("[tpu] WARNING: setup failed on some workers.", file=sys.stderr)
-
- # Set up git config on all workers
- git_name, git_email = _collect_git_config()
- if git_name and git_email:
- _run_all_workers(args.name, zone,
- f'git config --global user.name "{git_name}" && '
- f'git config --global user.email "{git_email}"')
-
- print(f"[tpu] All {num_workers} workers ready.")
-
-
-def tpu_sync(args):
- """Sync code to all workers without recreating venv.
-
- Fast alternative to `tpu setup` — just copies source code and runs
- `pip install -e .` to pick up changes. Use when the venv already exists.
- """
- zone = args.zone or _detect_zone(args.name)
- num_workers = _get_num_workers(args.name, zone)
-
- if num_workers <= 1:
- print(f"[tpu] Single-host TPU ({num_workers} worker). Use 'needle tpu connect' instead.")
- return
-
- print(f"[tpu] Multi-host TPU: {num_workers} workers (sync only)")
-
- if not _sync_code_to_workers(args.name, zone, num_workers):
- sys.exit(1)
-
- # Just reinstall the package to pick up code changes
- print(f"[tpu] Installing package on all {num_workers} workers...")
- result = _run_all_workers(args.name, zone,
- "cd ~/needle && .venv/bin/pip install -e . -q")
- if result.returncode != 0:
- print("[tpu] WARNING: pip install failed on some workers.", file=sys.stderr)
-
- print(f"[tpu] All {num_workers} workers synced.")
-
-
-def _collect_env_exports():
- """Collect env vars to forward to remote workers (HF_TOKEN, WANDB_API_KEY, etc)."""
- exports = []
- for var in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "WANDB_API_KEY"):
- val = os.environ.get(var)
- if val:
- # Quote value to handle special characters in tokens
- safe_val = val.replace("'", "'\\''")
- exports.append(f"export {var}='{safe_val}'")
- return " && ".join(exports)
-
-
-def _tpu_run_command(args, needle_cmd="train"):
- """Launch a needle command on all workers of a multi-host TPU."""
- zone = args.zone or _detect_zone(args.name)
- num_workers = _get_num_workers(args.name, zone)
-
- if num_workers > 1:
- print(f"[tpu] Killing stale processes on all workers...")
- _run_all_workers(args.name, zone,
- "sudo pkill -9 -f [n]eedle || true")
-
- extra = getattr(args, "train_args", [])
- if extra and extra[0] == "--":
- extra = extra[1:]
-
- train_cmd = f"cd ~/needle && PYTHONUNBUFFERED=1 .venv/bin/needle {needle_cmd}"
- if extra:
- train_cmd += " " + " ".join(extra)
-
- # Forward auth env vars to workers
- env_exports = _collect_env_exports()
- if env_exports:
- env_count = len(env_exports.split(" && "))
- print(f"[tpu] Forwarding {env_count} env var(s) to workers")
- train_cmd = env_exports + " && " + train_cmd
-
- # Wrap with nohup so training survives SSH disconnects
- log_file = f"/tmp/needle_{needle_cmd}.log"
- train_cmd = (
- f"nohup bash -c '{train_cmd}' > {log_file} 2>&1 &"
- f" echo $!; sleep 1; head -5 {log_file}"
- )
-
- if num_workers <= 1:
- print(f"[tpu] Single-host TPU. Running in background...")
- _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
- "--zone", zone, "--project", PROJECT,
- "--command", train_cmd],
- quiet=True,
- )
- print(f"[tpu] Running in background. Logs: ssh into worker and 'tail -f {log_file}'")
- return
-
- print(f"[tpu] Launching 'needle {needle_cmd}' on {num_workers} workers (background)...")
- result = _run(
- ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
- "--zone", zone, "--project", PROJECT,
- "--worker=all",
- "--command", train_cmd],
- check=False, quiet=True,
- )
- print(f"[tpu] Processes launched. Monitor with:")
- print(f" gcloud compute tpus tpu-vm ssh large --zone=asia-northeast1-b --worker=0 --command='tail -f {log_file}'")
- if result.returncode != 0:
- print(f"[tpu] WARNING: 'needle {needle_cmd}' exited with errors on some workers.",
- file=sys.stderr)
-
-
-def tpu_train(args):
- """Launch 'needle train' on all workers."""
- _tpu_run_command(args, needle_cmd="train")
-
-
-def tpu_pretrain(args):
- """Launch 'needle pretrain' on all workers."""
- _tpu_run_command(args, needle_cmd="pretrain")
-
-
-def tpu_dispatch(args):
- global PROJECT
- PROJECT = _get_project()
- actions = {
- "create": tpu_create,
- "connect": tpu_connect,
- "setup": tpu_setup,
- "sync": tpu_sync,
- "train": tpu_train,
- "pretrain": tpu_pretrain,
- "claude": tpu_claude,
- "stop": tpu_stop,
- "start": tpu_start,
- "delete": tpu_delete,
- "list": tpu_list,
- }
- if not args.tpu_action or args.tpu_action not in actions:
- print(TPU_HELP)
- sys.exit(0 if not args.tpu_action else 1)
- actions[args.tpu_action](args)
diff --git a/pyproject.toml b/pyproject.toml
index 08b0472..600698a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,25 +1,21 @@
[project]
-name = "needle"
-version = "0.1.0"
-requires-python = ">=3.11"
+name = "cactus-needle"
+version = "2.0.0"
+description = "A 14MB foundation tool-calling model for tiny devices: inference, LoRA finetuning, and build."
+requires-python = ">=3.9"
+readme = "README.md"
+license = { text = "Apache-2.0" }
dependencies = [
+ "huggingface_hub",
+ "numpy",
"jax",
"jaxlib",
"flax",
"optax",
- "datasets",
- "huggingface_hub",
- "transformers",
- "wandb",
"sentencepiece",
- "scipy",
- "gcsfs",
- "tqdm",
- "google-genai",
]
[project.optional-dependencies]
-tpu = ["jax[tpu]"]
gpu = ["jax[cuda12]"]
[project.scripts]
@@ -28,6 +24,10 @@ needle = "needle.cli:main"
[tool.setuptools.packages.find]
include = ["needle*"]
+[tool.setuptools.package-data]
+"needle.model" = ["*.model", "*.vocab"]
+"needle.agent" = ["index.html"]
+
[build-system]
requires = ["setuptools>=68.0"]
build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
index eaf2d02..558faaf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,7 @@
+huggingface_hub
+numpy
jax
jaxlib
flax
optax
-datasets
-huggingface_hub
-gcsfs
-transformers
-wandb
-pyyaml
sentencepiece
-google-genai
\ No newline at end of file
diff --git a/setup b/setup
index f1f5e62..27752b5 100755
--- a/setup
+++ b/setup
@@ -1,129 +1,30 @@
#!/usr/bin/env bash
+set -e
VENV_DIR=".venv"
-# On Ubuntu/Debian, ensure Python >= 3.11 and venv are available
-if [ -f /etc/os-release ] && grep -qi 'ubuntu\|debian' /etc/os-release; then
- PYTHON_OK=$(python3 -c "import sys; print(int(sys.version_info >= (3, 11)))" 2>/dev/null || echo "0")
- if [ "$PYTHON_OK" = "0" ]; then
- echo "Python >= 3.11 required. Installing..."
- sudo apt-get update -qq
- sudo apt-get install -y -qq software-properties-common
- sudo add-apt-repository -y ppa:deadsnakes/ppa
- sudo apt-get update -qq
- sudo apt-get install -y -qq python3.11 python3.11-venv python3.11-dev
- PYTHON=python3.11
- else
- PYTHON=python3
- # Ensure venv package is installed
- if ! $PYTHON -m venv --help &>/dev/null; then
- echo "Installing python3-venv..."
- sudo apt-get update -qq
- sudo apt-get install -y -qq python3-venv
- fi
- fi
-else
- # On macOS or other systems, find Python >= 3.11
- PYTHON=""
- for candidate in python3.14 python3.13 python3.12 python3.11 python3; do
- if command -v "$candidate" &>/dev/null; then
- PY_OK=$("$candidate" -c "import sys; print(int(sys.version_info >= (3, 11)))" 2>/dev/null || echo "0")
- if [ "$PY_OK" = "1" ]; then
- PYTHON="$candidate"
- break
- fi
- fi
- done
- if [ -z "$PYTHON" ]; then
- echo "Error: Python >= 3.11 is required. Install it with: brew install [email protected]"
- return 1 2>/dev/null || exit 1
- fi
-fi
-
-# Recreate venv if it doesn't exist or was built with the wrong Python
-RECREATE=0
-if [ ! -d "$VENV_DIR" ]; then
- RECREATE=1
-elif [ ! -f "$VENV_DIR/bin/activate" ]; then
- RECREATE=1
-elif [ -f "$VENV_DIR/bin/python3" ]; then
- VENV_PY_OK=$("$VENV_DIR/bin/python3" -c "import sys; print(int(sys.version_info >= (3, 11)))" 2>/dev/null || echo "0")
- if [ "$VENV_PY_OK" = "0" ]; then
- RECREATE=1
+PYTHON=""
+for candidate in python3.12 python3.11 python3.10 python3.9 python3; do
+ if command -v "$candidate" >/dev/null 2>&1 && \
+ "$candidate" -c "import sys; exit(0 if sys.version_info >= (3, 9) else 1)" 2>/dev/null; then
+ PYTHON="$candidate"
+ break
fi
+done
+if [ -z "$PYTHON" ]; then
+ echo "Python >= 3.9 is required." >&2
+ exit 1
fi
-if [ "$RECREATE" = "1" ]; then
- echo "Creating virtual environment with $PYTHON..."
- rm -rf "$VENV_DIR"
- $PYTHON -m venv "$VENV_DIR"
-else
- echo "Virtual environment already exists."
-fi
-
+[ -d "$VENV_DIR" ] || "$PYTHON" -m venv "$VENV_DIR"
source "$VENV_DIR/bin/activate"
-echo "Installing dependencies..."
pip install --upgrade pip -q
pip install -e . -q
-# Detect accelerator and install the matching JAX build
-ACCELERATOR="cpu"
-if command -v nvidia-smi &>/dev/null && nvidia-smi &>/dev/null; then
- ACCELERATOR="gpu"
-elif ls /dev/accel* &>/dev/null 2>&1 || [ -n "$TPU_NAME" ] || [ -d /sys/class/accel ]; then
- ACCELERATOR="tpu"
-fi
-
-case "$ACCELERATOR" in
- gpu)
- echo "Detected NVIDIA GPU. Installing jax[cuda12]..."
- pip install -U "jax[cuda12]"
- # GPU-friendly defaults: silence XLA autotuner spam, cache compiled
- # kernels across runs, and let JAX claim most of HBM.
- export TF_CPP_MIN_LOG_LEVEL=2
- export JAX_COMPILATION_CACHE_DIR="$HOME/.cache/jax"
- export XLA_PYTHON_CLIENT_MEM_FRACTION=0.95
- mkdir -p "$JAX_COMPILATION_CACHE_DIR"
- ;;
- tpu)
- echo "Detected TPU. Installing jax[tpu]..."
- pip install -U "jax[tpu]" -f https://storage.googleapis.com/jax-releases/libtpu_releases.html
- # Load TPU kernel modules if not already loaded
- if ! ls /dev/accel* &>/dev/null 2>&1; then
- echo "Loading TPU kernel modules..."
- for mod in gasket tpu_common tpu_v2 tpu_v3 tpu_v4 tpu_v4_lite; do
- sudo modprobe "$mod" 2>/dev/null || true
- done
- fi
- if ls /dev/accel* &>/dev/null 2>&1; then
- echo "TPU devices: $(ls /dev/accel* 2>/dev/null | wc -w) chip(s)"
- fi
- ;;
- *)
- echo "No GPU or TPU detected. Installing CPU-only jax..."
- pip install -U jax
- ;;
-esac
-
-# Sanity check: report the JAX backend that was actually selected
-python -c "import jax; print('JAX devices:', jax.devices())" 2>/dev/null || \
- echo "Warning: jax import failed; check the install above."
-
-if [ -f /sys/kernel/mm/transparent_hugepage/enabled ]; then
- echo "Enabling transparent hugepages..."
- sudo sh -c "echo always > /sys/kernel/mm/transparent_hugepage/enabled" 2>/dev/null || true
-fi
-
-if [ -z "$WANDB_API_KEY" ]; then
- echo ""
- echo " Enter your W&B API key (https://wandb.ai/authorize)"
- echo " Press Enter to skip."
- printf " > "
- read WANDB_API_KEY || true
- if [ -n "$WANDB_API_KEY" ]; then
- export WANDB_API_KEY
- fi
+if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then
+ echo "NVIDIA GPU detected; installing jax[cuda12]..."
+ pip install -U "jax[cuda12]" -q
fi
needle --help
diff --git a/setup.bat b/setup.bat
index 04d0739..5310c17 100644
--- a/setup.bat
+++ b/setup.bat
@@ -1,60 +1,24 @@
@echo off
set VENV_DIR=.venv
-:: Find Python 3.11+
set PYTHON_CMD=
-py -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)" >nul 2>&1
+py -c "import sys; sys.exit(0 if sys.version_info >= (3, 9) else 1)" >nul 2>&1
if %errorlevel% == 0 (
set PYTHON_CMD=py
) else (
- python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)" >nul 2>&1
- if %errorlevel% == 0 (
- set PYTHON_CMD=python
- )
+ python -c "import sys; sys.exit(0 if sys.version_info >= (3, 9) else 1)" >nul 2>&1
+ if %errorlevel% == 0 set PYTHON_CMD=python
)
if "%PYTHON_CMD%"=="" (
- echo Error: Python ^>= 3.11 is required. Please install Python 3.11+ from https://www.python.org/
+ echo Python ^>= 3.9 is required. Install it from https://www.python.org/
exit /b 1
)
-:: Check if venv exists and is valid
-set RECREATE=0
-if not exist "%VENV_DIR%\Scripts\python.exe" (
- set RECREATE=1
-) else (
- "%VENV_DIR%\Scripts\python.exe" -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)" >nul 2>&1
- if errorlevel 1 set RECREATE=1
-)
-
-if %RECREATE%==1 (
- echo Creating virtual environment with %PYTHON_CMD%...
- if exist "%VENV_DIR%" rmdir /s /q "%VENV_DIR%"
- %PYTHON_CMD% -m venv "%VENV_DIR%"
-) else (
- echo Virtual environment already exists.
-)
-
+if not exist "%VENV_DIR%\Scripts\python.exe" %PYTHON_CMD% -m venv "%VENV_DIR%"
call "%VENV_DIR%\Scripts\activate.bat"
-echo Installing dependencies...
-python -m pip install --upgrade pip
-python -m pip install -e .
-
-:: On native Windows, JAX GPU wheels do not exist.
-:: Only CPU mode is supported natively on Windows.
-echo Installing native Windows JAX (CPU mode)...
-python -m pip install -U jax jaxlib
-
-:: Sanity check
-python -c "import jax; print('JAX devices:', jax.devices())" || echo Warning: jax import failed; check the install above.
-
-:: Prompt for W&B key
-if "%WANDB_API_KEY%"=="" (
- echo.
- echo Enter your W^&B API key ^(https://wandb.ai/authorize^)
- echo Press Enter to skip.
- set /p WANDB_API_KEY=" > "
-)
+python -m pip install --upgrade pip -q
+python -m pip install -e . -q
-needle --help
\ No newline at end of file
+needle --help