Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku88a2b5389b9d
diff --git a/scripts/compare_voices.py b/scripts/compare_voices.pydeleted file mode 100644index 4dfc82a..0000000--- a/scripts/compare_voices.py+++ /dev/null@@ -1,61 +0,0 @@-#!/usr/bin/env python3-"""Generate sample TTS audio with Wavenet vs Neural2 for quality comparison."""--import io-import wave-from google.cloud import texttospeech--client = texttospeech.TextToSpeechClient()--QUERIES = [- "What's the weather like in San Francisco today?",- "Book me a flight from London to Tokyo next Friday",- "Send an email to the team about the meeting at 3pm",-]--VOICES = [- ("wavenet_male_us", "en-US-Wavenet-D", "en-US"),- ("chirp3_us", "en-US-Chirp3-HD-Achernar", "en-US"),- ("journey_male_us", "en-US-Journey-D", "en-US"),- ("wavenet_female_gb", "en-GB-Wavenet-A", "en-GB"),- ("chirp3_gb", "en-GB-Chirp3-HD-Leda", "en-GB"),- ("journey_female_us", "en-US-Journey-F", "en-US"),- ("wavenet_male_au", "en-AU-Wavenet-B", "en-AU"),- ("chirp3_au", "en-AU-Chirp3-HD-Charon", "en-AU"),- ("journey_female_us2", "en-US-Journey-O", "en-US"),-]---def pcm_to_wav(pcm_bytes, sr=16000):- buf = io.BytesIO()- with wave.open(buf, "wb") as wf:- wf.setnchannels(1)- wf.setsampwidth(2)- wf.setframerate(sr)- wf.writeframes(pcm_bytes)- return buf.getvalue()---import os-os.makedirs("voice_samples", exist_ok=True)--for qi, query in enumerate(QUERIES):- for label, voice_name, lang in VOICES:- print(f" {label} / query {qi+1}...", end="", flush=True)- response = client.synthesize_speech(- input=texttospeech.SynthesisInput(text=query),- voice=texttospeech.VoiceSelectionParams(- language_code=lang, name=voice_name,- ),- audio_config=texttospeech.AudioConfig(- audio_encoding=texttospeech.AudioEncoding.LINEAR16,- sample_rate_hertz=16000,- ),- )- wav = pcm_to_wav(response.audio_content)- path = f"voice_samples/q{qi+1}_{label}.wav"- with open(path, "wb") as f:- f.write(wav)- print(f" saved {path}")--print(f"\nDone! {len(QUERIES) * len(VOICES)} files in voice_samples/")diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.pydeleted file mode 100644index 6478efa..0000000--- a/scripts/synthesize_tools_data.py+++ /dev/null@@ -1,516 +0,0 @@-#!/usr/bin/env python3-"""Synthesize tool-calling data from PleIAs/SYNTH via OpenRouter.--Each query is sent to a randomly-selected frontier model to generate-(tools, answer) pairs, then validated and checkpointed as JSONL.--Usage:- python scripts/synthesize_tools_data.py- python scripts/synthesize_tools_data.py --model google/gemini-3.1-flash-lite-preview- python scripts/synthesize_tools_data.py --max-shards 1 --max-samples 100- python scripts/synthesize_tools_data.py --resume # continue from checkpoint-"""--import argparse-import json-import logging-import os-import random-import sys-import time-from concurrent.futures import ThreadPoolExecutor, as_completed-from pathlib import Path--import itertools--import requests--logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")-logger = logging.getLogger(__name__)--_PROJECT_ROOT = Path(__file__).resolve().parent.parent-_OUTPUT_DIR = _PROJECT_ROOT / "data" / "synth_tool_calls"-_CHECKPOINT_PATH = _OUTPUT_DIR / "checkpoint.jsonl"-_PARQUET_CACHE_DIR = _PROJECT_ROOT / ".data_cache"--OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"--MODEL_POOL = [- "google/gemini-3.1-flash-lite-preview",-]-DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"--SYSTEM_PROMPT = """\-You are a synthetic data generator for training a tool-calling AI model.--Given a user query, you must invent a realistic set of tools (APIs/functions) that \-could help answer it, then produce the correct tool call(s).--Vary the domain widely — cover areas like software development, data analysis, \-e-commerce, finance, healthcare, travel, IoT, social media, education, DevOps, legal, \-HR, and more. Do NOT default to simple personal-assistant tasks.--Rules:-1. Invent 0-3 tools as JSON. Each tool MUST have: name, description, parameters \-(dict of param_name -> {type, description, required}). Use realistic parameter types \-including string, number, boolean, array, object. Include nested objects or arrays \-where appropriate.-2. Only ONE or TWO tools should be relevant to the query; the rest are plausible \-distractors from the SAME domain.-3. Tool names should be snake_case, realistic API-style, and DIVERSE — avoid repeating \-common names like search_web or get_weather across samples. Think of specific, \-domain-appropriate APIs (e.g. query_elasticsearch, run_sql_migration, price_option_chain, \-schedule_k8s_cronjob, fetch_patient_labs).-4. Produce the answer as a JSON list of tool calls: [{"name": "...", "arguments": {...}}].-5. For complex queries, use MULTIPLE tool calls (chained or parallel) where realistic.-6. If the query genuinely needs no tool, return an empty answer list [].-7. Argument values MUST be grounded in the query — do not hallucinate values.-8. Respond in the SAME LANGUAGE as the query.-9. Return ONLY valid JSON, no markdown fences, no commentary.--Output format (strict JSON object):-{"tools": [...], "answer": [...]}"""--USER_TEMPLATE = "Query: {query}"---def _discover_shard_urls(token, max_shards=None):- """Discover available parquet shard URLs from the HF API."""- api_url = "https://huggingface.co/api/datasets/PleIAs/SYNTH/parquet"- headers = {}- if token:- headers["Authorization"] = f"Bearer {token}"- resp = requests.get(api_url, headers=headers, timeout=30)- resp.raise_for_status()- data = resp.json() # {"default": {"train": [url, ...]}}- urls = data["default"]["train"]- if max_shards:- urls = urls[:max_shards]- logger.info(f"Discovered {len(urls)} parquet shards")- return urls---def download_shard(url, shard_idx, token):- """Download a single parquet shard, returning the local path."""- cache_path = _PARQUET_CACHE_DIR / f"synth_shard{shard_idx}.parquet"- if cache_path.exists():- logger.info(f"Using cached shard {shard_idx}: {cache_path}")- return cache_path-- _PARQUET_CACHE_DIR.mkdir(parents=True, exist_ok=True)-- headers = {}- if token:- headers["Authorization"] = f"Bearer {token}"-- logger.info(f"Downloading shard {shard_idx}...")- resp = requests.get(url, headers=headers, stream=True, timeout=120)- resp.raise_for_status()-- total = int(resp.headers.get("content-length", 0))- downloaded = 0- with open(cache_path, "wb") as f:- for chunk in resp.iter_content(chunk_size=8 * 1024 * 1024):- f.write(chunk)- downloaded += len(chunk)- if total:- pct = downloaded / total * 100- print(f"\r {downloaded / 1e6:.0f}/{total / 1e6:.0f} MB ({pct:.0f}%)", end="", flush=True)- print()- logger.info(f"Saved shard {shard_idx} to {cache_path}")- return cache_path---def load_queries_from_parquet(path, max_samples=None):- """Load queries from parquet, return list of (sample_id, language, query)."""- import pyarrow.parquet as pq-- logger.info(f"Reading parquet: {path}")- table = pq.read_table(path, columns=["synth_id", "language", "query"])- n = table.num_rows- logger.info(f" {n:,} rows in shard")-- ids = table.column("synth_id").to_pylist()- langs = table.column("language").to_pylist()- queries = table.column("query").to_pylist()-- samples = list(zip(ids, langs, queries))-- rng = random.Random(42)- rng.shuffle(samples)-- if max_samples:- samples = samples[:max_samples]-- logger.info(f" Selected {len(samples):,} samples")- return samples---def load_checkpoint():- """Load completed synth_ids from checkpoint file."""- done = set()- if _CHECKPOINT_PATH.exists():- with open(_CHECKPOINT_PATH) as f:- for line in f:- try:- row = json.loads(line)- done.add(row["synth_id"])- except (json.JSONDecodeError, KeyError):- continue- logger.info(f"Resuming: {len(done):,} already completed")- return done---def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):- """Call OpenRouter API to generate tools + answer for a query."""- headers = {- "Authorization": f"Bearer {api_key}",- "Content-Type": "application/json",- }- payload = {- "model": model,- "messages": [- {"role": "system", "content": SYSTEM_PROMPT},- {"role": "user", "content": USER_TEMPLATE.format(query=query)},- ],- "temperature": 0.7,- "max_tokens": 4096,- }-- for attempt in range(max_retries):- try:- resp = requests.post(- OPENROUTER_URL, headers=headers, json=payload, timeout=60- )- if resp.status_code == 429:- wait = min(2 ** attempt * 2, 30)- logger.warning(f"[{model}] 429 rate-limited, retrying in {wait}s")- time.sleep(wait)- continue- if not resp.ok:- logger.warning(f"[{model}] HTTP {resp.status_code}: {resp.text[:200]}")- if attempt == max_retries - 1:- return None- time.sleep(2 ** attempt)- continue- content = resp.json()["choices"][0]["message"]["content"]- parsed = _parse_response(content)- if parsed is None:- logger.warning(f"[{model}] Failed to parse response: {content[:200]}")- return parsed- except (requests.RequestException, KeyError, IndexError) as e:- logger.warning(f"[{model}] Request error (attempt {attempt+1}/{max_retries}): {e}")- if attempt == max_retries - 1:- return None- time.sleep(2 ** attempt)- return None---def _parse_response(content):- """Parse LLM response into (tools, answer) or None."""- content = content.strip()- if content.startswith("```"):- content = content.split("\n", 1)[-1]- if content.endswith("```"):- content = content[:-3]- content = content.strip()-- try:- obj = json.loads(content)- except json.JSONDecodeError:- return None-- if not isinstance(obj, dict):- return None-- tools = obj.get("tools")- answer = obj.get("answer")-- if not isinstance(tools, list):- return None- if not isinstance(answer, list):- return None-- for t in tools:- if not isinstance(t, dict) or "name" not in t:- return None-- for a in answer:- if not isinstance(a, dict) or "name" not in a:- return None- if "arguments" not in a:- a["arguments"] = {}-- return tools, answer---def _validate_and_format(synth_id, language, query, tools, answer, model, source="synth-pleias"):- """Validate the generated data and return a formatted row or None."""- tool_names = {t["name"] for t in tools}- for a in answer:- if a["name"] not in tool_names:- return None-- tool_params = {}- for t in tools:- params = t.get("parameters") or {}- tool_params[t["name"]] = set(params.keys())-- for a in answer:- args = a.get("arguments", {})- if not isinstance(args, dict):- return None- schema_params = tool_params.get(a["name"], set())- if args and not set(args.keys()).issubset(schema_params):- return None-- return {- "synth_id": synth_id,- "language": language,- "query": query,- "answers": json.dumps(answer),- "tools": json.dumps(tools),- "source": source,- "model": model,- }----class _KeyModelRotator:- """Thread-safe round-robin across (api_key, model) pairs."""-- def __init__(self, api_keys, models):- pairs = [(k, m) for k in api_keys for m in models]- self._cycle = itertools.cycle(pairs)- import threading- self._lock = threading.Lock()-- def next(self):- with self._lock:- return next(self._cycle)---def process_sample(sample, rotator, source_tag="synth-pleias"):- """Process a single sample: call LLM, validate, return row or None."""- synth_id, language, query = sample- api_key, model = rotator.next()- result = call_openrouter(query, api_key, model=model)- if result is None:- return None- tools, answer = result- row = _validate_and_format(synth_id, language, query, tools, answer, model, source_tag)- if row is None:- logger.warning(f"[{model}] Validation failed for synth_id={synth_id}")- return row---def _process_batch(samples, label, batch_idx, rotator, args, tqdm, source_tag,- since_last_upload):- """Process a batch of samples, append to checkpoint. Returns (generated, failed)."""- logger.info(f"\n{'=' * 60}")- logger.info(f"Batch: {label}")- logger.info(f"{'=' * 60}")-- done_ids = load_checkpoint()- pending = [s for s in samples if s[0] not in done_ids]- if not pending:- logger.info("All samples already done, skipping")- return 0, 0- logger.info(f"Processing {len(pending):,} samples ({len(done_ids):,} already done)")-- generated = 0- failed = 0- out_file = open(_CHECKPOINT_PATH, "a")-- try:- with ThreadPoolExecutor(max_workers=args.workers) as executor:- futures = {}- for sample in pending:- fut = executor.submit(- process_sample, sample, rotator, source_tag,- )- futures[fut] = sample[0]-- pbar = tqdm(- as_completed(futures),- total=len(futures),- desc=label,- unit="sample",- dynamic_ncols=True,- ) if tqdm else as_completed(futures)-- for i, fut in enumerate(pbar if tqdm else as_completed(futures), 1):- try:- row = fut.result()- except Exception:- failed += 1- row = None-- if row is not None:- out_file.write(json.dumps(row) + "\n")- generated += 1- since_last_upload[0] += 1- else:- failed += 1-- if tqdm:- rate = generated / max(generated + failed, 1) * 100- pbar.set_postfix(ok=generated, fail=failed, rate=f"{rate:.0f}%")- if i % args.batch_size == 0:- out_file.flush()- else:- if i % args.batch_size == 0:- out_file.flush()- total = generated + failed- rate = generated / max(total, 1) * 100- logger.info(- f" [{total:,}/{len(pending):,}] "- f"generated={generated:,} failed={failed:,} "- f"({rate:.0f}% success)"- )-- if since_last_upload[0] >= args.upload_every:- out_file.flush()- _export_dataset()- since_last_upload[0] = 0-- if tqdm:- pbar.close()- except KeyboardInterrupt:- logger.info("\nInterrupted — progress saved to checkpoint")- finally:- out_file.flush()- out_file.close()-- logger.info(f"{label}: {generated:,} generated, {failed:,} failed")- return generated, failed---def main():- parser = argparse.ArgumentParser(description="Synthesize tool-calling data from PleIAs/SYNTH")- parser.add_argument("--max-samples", type=int, default=None,- help="Max queries to process (default: all in shard)")- parser.add_argument("--models", type=str, nargs="+", default=None,- help=f"OpenRouter models to rotate (default: built-in pool)")- parser.add_argument("--model", type=str, default=None,- help="Use a single model instead of the pool")- parser.add_argument("--workers", type=int, default=128,- help="Concurrent API requests (default: 128)")- parser.add_argument("--max-shards", type=int, default=None,- help="Max parquet shards to process (default: all)")- parser.add_argument("--resume", action="store_true",- help="Resume from checkpoint")- parser.add_argument("--batch-size", type=int, default=500,- help="Samples between progress logs (default: 500)")- parser.add_argument("--upload-every", type=int, default=1000,- help="Upload to GCS every N successful generations (default: 1000)")- args = parser.parse_args()-- raw_keys = os.environ.get("OPENROUTER_API_KEY", "")- api_keys = [k.strip() for k in raw_keys.split(",") if k.strip()]- if not api_keys:- print("Error: set OPENROUTER_API_KEY environment variable (comma-separated for multiple)")- sys.exit(1)- logger.info(f"API keys: {len(api_keys)}")-- if args.model:- models = [args.model]- elif args.models:- models = args.models- else:- models = list(MODEL_POOL)- logger.info(f"Model pool ({len(models)}): {', '.join(models)}")- logger.info(f"Key x Model pairs: {len(api_keys) * len(models)} (each gets its own rate limit)")-- rotator = _KeyModelRotator(api_keys, models)-- _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)-- if not args.resume and _CHECKPOINT_PATH.exists():- _CHECKPOINT_PATH.unlink()- logger.info("Cleared previous checkpoint (use --resume to keep)")-- try:- from tqdm import tqdm- except ImportError:- logger.warning("tqdm not installed, falling back to basic progress logging")- tqdm = None-- total_generated = 0- total_failed = 0- batch_idx = 0- since_last_upload = [0]-- hf_token = os.environ.get("HF_TOKEN")- shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)-- for shard_idx, shard_url in enumerate(shard_urls):- parquet_path = download_shard(shard_url, shard_idx, hf_token)- samples = load_queries_from_parquet(parquet_path, max_samples=args.max_samples)- gen, fail = _process_batch(- samples, f"PleIAs shard {shard_idx}", batch_idx, rotator,- args, tqdm, source_tag="synth-pleias",- since_last_upload=since_last_upload,- )- total_generated += gen- total_failed += fail- batch_idx += 1-- if since_last_upload[0] > 0:- _export_dataset()-- logger.info(f"\nAll done: {total_generated:,} generated, {total_failed:,} failed")---def _export_dataset():- """Convert checkpoint JSONL to the unified format and push to GCS."""- from datasets import Dataset-- rows = []- with open(_CHECKPOINT_PATH) as f:- for line in f:- try:- row = json.loads(line)- rows.append({- "query": row["query"],- "answers": row["answers"],- "tools": row["tools"],- "source": row.get("source", "synth-pleias"),- "model": row.get("model", "unknown"),- })- except (json.JSONDecodeError, KeyError):- continue-- if not rows:- return-- ds = Dataset.from_list(rows)- out_path = _OUTPUT_DIR / "dataset"- ds.save_to_disk(str(out_path))- logger.info(f"Exported {len(rows):,} examples to {out_path}/")-- try:- sys.path.insert(0, str(_PROJECT_ROOT))- from src.gcs import upload_directory- gcs_prefix = "synth_tool_calls"- upload_directory(str(out_path), gcs_prefix)- except Exception as e:- logger.warning(f"GCS upload failed: {e}")-- for i, row in enumerate(rows[:10]):- try:- tools = json.loads(row["tools"])- answers = json.loads(row["answers"])- except (json.JSONDecodeError, TypeError):- tools, answers = [], []- called_names = [a["name"] for a in answers if isinstance(a, dict)]- print(f"\n{'─' * 60}")- print(f"[{i+1}] Query: {row['query'][:200]}")- print(f" Available tools ({len(tools)}): {', '.join(t.get('name','?') for t in tools)}")- print(f" Called tools ({len(called_names)}): {', '.join(called_names) or '(none)'}")- print(f" Answers: {row['answers'][:200]}")- print(f"{'─' * 60}")---if __name__ == "__main__":- main()diff --git a/scripts/tools_data.py b/scripts/tools_data.pydeleted file mode 100755index 8a23e68..0000000--- a/scripts/tools_data.py+++ /dev/null@@ -1,879 +0,0 @@-"""-Unified tool-calling dataset builder.--Loads HuggingFace tool-calling datasets and converts them into a common schema:-- query : str – the user's natural-language request- answers : str – JSON list of {"name": ..., "arguments": {...}} tool calls- tools : str – JSON list of available tool definitions- source : str – which dataset the example came from--Datasets:--1. Salesforce/xlam-function-calling-60k- Already in the target schema (query, answers, tools).--2. glaiveai/glaive-function-calling-v2- Tool definitions embedded in the system prompt; chat is a flat string with- USER:/A: role markers and <functioncall> tags. Examples with empty answers- (answers=[]) are kept so the model learns when NOT to call functions.--3. Agent-Ark/Toucan-1.5M (OSS, Kimi-K2, Qwen3 subsets)- Large-scale tool-calling dataset. Tool calls extracted from 'messages' column- (assistant messages with function_call objects), filtered by 'target_tools'.- Tool schemas converted from OpenAI format to flat format.--4. Team-ACE/ToolACE- Synthetic tool-calling dataset (11.3k) with dual-layer verification.- Tool definitions in system prompt JSON; calls use bracket notation- [func(key=val)]. Grounding filter skipped (pre-verified).-"""--import ast-import json-import multiprocessing as mp-import os-import random-import re-from collections import Counter--from datasets import load_dataset, Dataset, concatenate_datasets--_NUM_WORKERS = min(20, max(1, (os.cpu_count() or 1) // 4))---DATASETS = {- "xlam-function-calling-60k": "Salesforce/xlam-function-calling-60k",- "glaive-function-calling": "glaiveai/glaive-function-calling-v2",- "toucan-oss": ("Agent-Ark/Toucan-1.5M", "OSS"),- "toucan-kimi": ("Agent-Ark/Toucan-1.5M", "Kimi-K2"),- "toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),- "toucan-sft": ("Agent-Ark/Toucan-1.5M", "SFT"),- "toolace": "Team-ACE/ToolACE",- "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),-}---def convert_xlam(dataset):- return dataset.select_columns(["query", "answers", "tools"])---def _convert_glaive_chunk(chunk):- rows = []- for system, chat in chunk:- tools = _extract_json_objects(system)- query, answers = _parse_glaive_chat(chat)- if query:- rows.append({- "query": query,- "answers": json.dumps(answers),- "tools": json.dumps(tools),- })- return rows---def convert_glaive(dataset):- pairs = list(zip(dataset["system"], dataset["chat"]))- chunk_size = max(1, len(pairs) // (_NUM_WORKERS * 4))- chunks = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)]- with mp.Pool(_NUM_WORKERS) as pool:- results = pool.map(_convert_glaive_chunk, chunks)- rows = [r for chunk in results for r in chunk]- return Dataset.from_list(rows)---def _extract_json_objects(text):- """Extract top-level JSON objects with a 'name' field from text."""- objects = []- depth = 0- start = None- for i, ch in enumerate(text):- if ch == '{':- if depth == 0:- start = i- depth += 1- elif ch == '}':- depth -= 1- if depth == 0 and start is not None:- try:- obj = json.loads(text[start:i + 1])- if "name" in obj:- objects.append(obj)- except json.JSONDecodeError:- pass- start = None- return objects---def _parse_glaive_chat(chat):- """Extract first user query and all function calls from glaive chat."""- query = None- answers = []-- user_match = re.search(r'USER:\s*(.*?)(?=\n\s*(?:A:|ASSISTANT:)|$)', chat, re.DOTALL)- if user_match:- query = user_match.group(1).strip()-- for m in re.finditer(r'<functioncall>\s*(\{.+?\})\s*(?:<\|endoftext\|>|$)', chat, re.DOTALL):- raw = m.group(1)- raw = re.sub(r"'(\{.*?\})'", r'"\1"', raw)- try:- fc = json.loads(raw)- except json.JSONDecodeError:- raw2 = re.sub(- r"'(\{.*?\})'",- lambda x: json.dumps(x.group(1)),- m.group(1),- )- try:- fc = json.loads(raw2)- except json.JSONDecodeError:- continue- args = fc.get("arguments", {})- if isinstance(args, str):- try:- args = json.loads(args)- except json.JSONDecodeError:- pass- answers.append({"name": fc["name"], "arguments": args})-- return query, answers---def _convert_openai_tools(tools_json):- """Convert OpenAI-format tools to flat format.-- OpenAI: [{"type": "function", "function": {"name": ..., "parameters": {"type": "object", "properties": {...}}}}]- Flat: [{"name": ..., "description": ..., "parameters": {"param_name": {"type": ..., "description": ...}}}]- """- try:- tools = json.loads(tools_json) if isinstance(tools_json, str) else tools_json- except (json.JSONDecodeError, TypeError):- return []- flat = []- for t in tools:- if isinstance(t, dict) and "function" in t:- fn = t["function"]- elif isinstance(t, dict) and "name" in t:- fn = t- else:- continue- params = fn.get("parameters") or {}- # Unwrap JSON Schema object wrapper → flat param dict- if isinstance(params, dict) and "properties" in params:- params = params["properties"]- flat.append({- "name": fn["name"],- "description": fn.get("description", ""),- "parameters": params,- })- return flat---def _extract_toucan_calls(messages_json, tool_names=None):- """Extract tool calls from Toucan messages.-- Collects all function_call entries from assistant messages.- If tool_names is provided, only keeps calls whose name matches an available tool.- """- try:- messages = json.loads(messages_json) if isinstance(messages_json, str) else messages_json- except (json.JSONDecodeError, TypeError):- return []- calls = []- for msg in messages:- if not isinstance(msg, dict):- continue- if msg.get("role") != "assistant":- continue- fc = msg.get("function_call")- if not fc or not isinstance(fc, dict):- continue- name = fc.get("name", "")- if not name:- continue- if tool_names is not None and name not in tool_names:- continue- args = fc.get("arguments", {})- if isinstance(args, str):- try:- args = json.loads(args)- except (json.JSONDecodeError, TypeError, ValueError):- args = {}- if not isinstance(args, dict):- args = {}- calls.append({"name": name, "arguments": args})- return calls---def _convert_toucan_chunk(chunk):- rows = []- for question, tools_raw, messages in chunk:- query = (question or "").strip()- if not query:- continue- tools = _convert_openai_tools(tools_raw or "[]")- if not tools:- continue- tool_names = {t["name"] for t in tools}- calls = _extract_toucan_calls(messages or "[]", tool_names)- if not calls:- continue- rows.append({- "query": query,- "answers": json.dumps(calls),- "tools": json.dumps(tools),- })- return rows---def convert_toucan(dataset):- questions = dataset["question"]- if "available_tools" in dataset.column_names:- tools_raw = dataset["available_tools"]- elif "tools" in dataset.column_names:- tools_raw = dataset["tools"]- else:- tools_raw = ["[]"] * len(dataset)- messages = dataset["messages"]- triples = list(zip(questions, tools_raw, messages))- chunk_size = max(1, len(triples) // (_NUM_WORKERS * 4))- chunks = [triples[i:i + chunk_size] for i in range(0, len(triples), chunk_size)]- with mp.Pool(_NUM_WORKERS) as pool:- results = pool.map(_convert_toucan_chunk, chunks)- rows = [r for chunk in results for r in chunk]- return Dataset.from_list(rows)---def _extract_toolace_tools(system_text):- """Extract JSON tool definitions array from ToolACE system prompt."""- start = system_text.find('[{')- if start == -1:- return []- depth = 0- for i in range(start, len(system_text)):- if system_text[i] == '[':- depth += 1- elif system_text[i] == ']':- depth -= 1- if depth == 0:- try:- return json.loads(system_text[start:i + 1])- except json.JSONDecodeError:- return []- return []---def _split_bracket_args(s, sep=','):- """Split on separator respecting nested brackets and quoted strings."""- parts = []- depth = 0- in_str = False- str_char = None- current = []- for i, ch in enumerate(s):- if in_str:- current.append(ch)- if ch == str_char and (i == 0 or s[i - 1] != '\\'):- in_str = False- elif ch in ('"', "'"):- in_str = True- str_char = ch- current.append(ch)- elif ch in ('(', '[', '{'):- depth += 1- current.append(ch)- elif ch in (')', ']', '}'):- depth -= 1- current.append(ch)- elif ch == sep and depth == 0:- parts.append(''.join(current))- current = []- else:- current.append(ch)- if current:- parts.append(''.join(current))- return parts---def _parse_bracket_value(s):- """Parse a value string from bracket notation arguments."""- s = s.strip()- if not s:- return s- try:- return json.loads(s)- except (json.JSONDecodeError, ValueError):- pass- if s == 'True':- return True- if s == 'False':- return False- if s in ('None', 'null'):- return None- try:- return ast.literal_eval(s)- except (ValueError, SyntaxError):- pass- return s---def _parse_toolace_calls(text, tool_names):- """Parse ToolACE bracket notation into tool calls using known names."""- text = text.strip()- if not text.startswith('[') or not text.endswith(']'):- return []- inner = text[1:-1].strip()- if not inner:- return []-- sorted_names = sorted(tool_names, key=len, reverse=True)- calls = []- pos = 0-- while pos < len(inner):- while pos < len(inner) and inner[pos] in (' ', ',', '\n', '\t'):- pos += 1- if pos >= len(inner):- break-- matched = None- for name in sorted_names:- if inner[pos:pos + len(name)] == name:- rest = inner[pos + len(name):].lstrip()- if rest and rest[0] == '(':- matched = name- break-- if not matched:- break-- pos += len(matched)- while pos < len(inner) and inner[pos] == ' ':- pos += 1- if pos >= len(inner) or inner[pos] != '(':- break-- depth = 0- in_str = False- str_char = None- end = pos- while end < len(inner):- ch = inner[end]- if in_str:- if ch == str_char and (end == 0 or inner[end - 1] != '\\'):- in_str = False- elif ch in ('"', "'"):- in_str = True- str_char = ch- elif ch == '(':- depth += 1- elif ch == ')':- depth -= 1- if depth == 0:- break- end += 1-- args_str = inner[pos + 1:end].strip()- arguments = {}- if args_str:- for pair in _split_bracket_args(args_str, ','):- pair = pair.strip()- eq = pair.find('=')- if eq < 0:- continue- key = pair[:eq].strip()- val = _parse_bracket_value(pair[eq + 1:])- arguments[key] = val-- calls.append({"name": matched, "arguments": arguments})- pos = end + 1-- return calls---def _convert_toolace_chunk(chunk):- rows = []- for system, convs in chunk:- raw_tools = _extract_toolace_tools(system)- tools = _convert_openai_tools(raw_tools)- if not tools:- continue-- tool_names = {t["name"] for t in tools}-- query = None- assistant_text = None- for msg in convs:- role = msg.get("from", "")- val = (msg.get("value") or "").strip()- if role == "user" and query is None:- query = val- elif role == "assistant" and assistant_text is None:- assistant_text = val-- if not query:- continue-- calls = []- if assistant_text:- calls = _parse_toolace_calls(assistant_text, tool_names)-- rows.append({- "query": query,- "answers": json.dumps(calls),- "tools": json.dumps(tools),- })- return rows---def convert_toolace(dataset):- pairs = list(zip(dataset["system"], dataset["conversations"]))- chunk_size = max(1, len(pairs) // (_NUM_WORKERS * 4))- chunks = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)]- with mp.Pool(_NUM_WORKERS) as pool:- results = pool.map(_convert_toolace_chunk, chunks)- rows = [r for chunk in results for r in chunk]- return Dataset.from_list(rows)---def _extract_nemotron_calls(messages, tool_names=None):- """Extract tool calls from Nemotron messages (OpenAI tool_calls format)."""- calls = []- for msg in messages:- if not isinstance(msg, dict) or msg.get("role") != "assistant":- continue-- # OpenAI tool_calls format (newer)- for tc in (msg.get("tool_calls") or []):- if not isinstance(tc, dict):- continue- fn = tc.get("function", tc)- name = fn.get("name", "")- if not name:- continue- if tool_names and name not in tool_names:- continue- args = fn.get("arguments", {})- if isinstance(args, str):- try:- args = json.loads(args)- except (json.JSONDecodeError, TypeError):- args = {}- if not isinstance(args, dict):- args = {}- calls.append({"name": name, "arguments": args})-- # Fallback: function_call format (older OpenAI)- fc = msg.get("function_call")- if fc and isinstance(fc, dict):- name = fc.get("name", "")- if name and (not tool_names or name in tool_names):- args = fc.get("arguments", {})- if isinstance(args, str):- try:- args = json.loads(args)- except (json.JSONDecodeError, TypeError):- args = {}- if not isinstance(args, dict):- args = {}- calls.append({"name": name, "arguments": args})-- return calls---def _convert_nemotron_chunk(chunk):- rows = []- for tools_raw, messages in chunk:- tools = _convert_openai_tools(tools_raw or [])- if not tools:- continue-- tool_names = {t["name"] for t in tools}-- query = None- for msg in (messages or []):- if isinstance(msg, dict) and msg.get("role") == "user":- query = (msg.get("content") or "").strip()- break-- if not query:- continue-- calls = _extract_nemotron_calls(messages, tool_names)- if not calls:- continue-- rows.append({- "query": query,- "answers": json.dumps(calls),- "tools": json.dumps(tools),- })- return rows---def convert_nemotron(dataset):- pairs = list(zip(dataset["tools"], dataset["messages"]))- chunk_size = max(1, len(pairs) // (_NUM_WORKERS * 4))- chunks = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)]- with mp.Pool(_NUM_WORKERS) as pool:- results = pool.map(_convert_nemotron_chunk, chunks)- rows = [r for chunk in results for r in chunk]- return Dataset.from_list(rows)---CONVERTERS = {- "xlam-function-calling-60k": convert_xlam,- "glaive-function-calling": convert_glaive,- "toucan-oss": convert_toucan,- "toucan-kimi": convert_toucan,- "toucan-qwen": convert_toucan,- "toucan-sft": convert_toucan,- "toolace": convert_toolace,- "nemotron-tc": convert_nemotron,-}--_PLACEHOLDER_VALUES = {- "john_doe", "jane_doe", "john.doe", "jane.doe",- "john_doe_456", "jane_doe_123",- "[email protected]", "[email protected]",- "[email protected]", "[email protected]",- "[email protected]", "[email protected]",- "1234567890", "0987654321",- "123 main st", "123 main street",-}---def _has_placeholder_args(answers_json):- """Check if any argument values are known placeholders."""- try:- answers = json.loads(answers_json)- except (json.JSONDecodeError, TypeError):- return False- if not answers:- return False- for call in answers:- args = call.get("arguments", {})- if isinstance(args, str):- try:- args = json.loads(args)- except (json.JSONDecodeError, TypeError):- continue- if not isinstance(args, dict):- continue- for val in args.values():- if isinstance(val, str) and val.lower().strip() in _PLACEHOLDER_VALUES:- return True- return False----def _answer_calls_valid_tools(ex):- """Check that every tool called in the answer exists in the tools list."""- try:- answers = json.loads(ex["answers"])- tools = json.loads(ex["tools"])- except (json.JSONDecodeError, TypeError):- return True- if not answers:- return True- tool_names = {t["name"] for t in tools}- return all(a["name"] in tool_names for a in answers)---def _answer_uses_valid_params(ex):- """Check that arguments in each call are a subset of the tool's schema params."""- try:- answers = json.loads(ex["answers"])- tools = json.loads(ex["tools"])- except (json.JSONDecodeError, TypeError):- return True- if not answers:- return True- tool_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tools}- for a in answers:- schema_params = tool_map.get(a["name"])- if schema_params is None:- continue # caught by _answer_calls_valid_tools- args = a.get("arguments", {})- if isinstance(args, str):- try:- args = json.loads(args)- except (json.JSONDecodeError, TypeError):- continue- if not isinstance(args, dict):- continue- if args and not set(args.keys()).issubset(schema_params):- return False- return True----def _deduplicate(dataset):- """Remove duplicates by (query, tools) pair, keeping the first occurrence.-- Same query with different tool sets is kept — those are distinct examples.- Same query with same tools but different answers (e.g. from different LLMs)- is deduplicated since the model only needs one answer per (query, tools) pair.- """- queries = dataset["query"]- tools = dataset["tools"]- seen = set()- keep = []- for i in range(len(queries)):- key = (queries[i], tools[i])- if key not in seen:- seen.add(key)- keep.append(i)- if len(keep) == len(dataset):- return dataset- return dataset.select(keep)---def _trim_tools(dataset, max_tools=5, seed=42):- """Trim tool lists to at most max_tools, keeping called tools.-- For samples with >max_tools available tools, keeps all tools referenced- in the answers plus randomly sampled distractors up to max_tools.- Samples where the number of called tools alone exceeds max_tools are dropped.- """- rng = random.Random(seed)- rows = []- dropped = 0- for i in range(len(dataset)):- ex = dataset[i]- try:- tools = json.loads(ex["tools"])- except (json.JSONDecodeError, TypeError):- tools = []- if len(tools) <= max_tools:- rows.append(ex)- continue- try:- answers = json.loads(ex["answers"])- except (json.JSONDecodeError, TypeError):- answers = []- called_names = {a["name"] for a in answers if isinstance(a, dict)}- called = [t for t in tools if t.get("name") in called_names]- if len(called) > max_tools:- dropped += 1- continue- distractors = [t for t in tools if t.get("name") not in called_names]- n_distractors = max_tools - len(called)- sampled = rng.sample(distractors, min(n_distractors, len(distractors)))- trimmed = called + sampled- rng.shuffle(trimmed)- rows.append({- "query": ex["query"],- "answers": ex["answers"],- "tools": json.dumps(trimmed),- "source": ex["source"],- })- trimmed = len(dataset) - len(rows) - dropped- print(f"\nTool trimming (max {max_tools}): "- f"trimmed {trimmed + dropped} examples, dropped {dropped}")- return Dataset.from_list(rows)---def _normalize_tool_names(dataset):- """Normalize all tool names to snake_case in both tools and answers fields."""- from src.data import to_snake_case-- rows = []- for i in range(len(dataset)):- ex = dataset[i]- try:- tools = json.loads(ex["tools"])- except (json.JSONDecodeError, TypeError):- rows.append(ex)- continue- try:- answers = json.loads(ex["answers"])- except (json.JSONDecodeError, TypeError):- answers = []-- for t in tools:- if isinstance(t, dict) and "name" in t:- t["name"] = to_snake_case(t["name"])- for a in answers:- if isinstance(a, dict) and "name" in a:- a["name"] = to_snake_case(a["name"])-- rows.append({- "query": ex["query"],- "answers": json.dumps(answers),- "tools": json.dumps(tools),- "source": ex["source"],- })- return Dataset.from_list(rows)---def _augment_irrelevance(dataset, ratio=0.1, seed=42):- """Create irrelevance samples by swapping tools with unrelated queries.-- For a fraction of examples with tool calls, replace the available tools- with tools from a different example and set the answer to [] (no calls).- Ensures no overlap between the query's needed tools and the donor tools.- """- rng = random.Random(seed)-- all_answers = dataset["answers"]- all_tools = dataset["tools"]- all_queries = dataset["query"]-- with_calls = []- call_names = {}- tool_names_by_idx = {}-- for i, ans_str in enumerate(all_answers):- try:- answers = json.loads(ans_str)- except (json.JSONDecodeError, TypeError):- continue- if not answers:- continue- with_calls.append(i)- call_names[i] = {c["name"] for c in answers if isinstance(c, dict)}- try:- tools = json.loads(all_tools[i])- tool_names_by_idx[i] = {t["name"] for t in tools if isinstance(t, dict)}- except (json.JSONDecodeError, TypeError):- tool_names_by_idx[i] = set()-- n_aug = int(len(with_calls) * ratio)- selected = rng.sample(with_calls, min(n_aug, len(with_calls)))-- rows = []- for idx in selected:- orig_call_names = call_names[idx]- for _ in range(5):- donor = rng.choice(with_calls)- if donor == idx:- continue- # Ensure none of the query's needed tools appear in donor's tool set- if not (orig_call_names & tool_names_by_idx.get(donor, set())):- rows.append({- "query": all_queries[idx],- "answers": "[]",- "tools": all_tools[donor],- "source": "irrelevance-aug",- })- break-- if rows:- print(f"\nIrrelevance augmentation: {len(rows)} samples "- f"({len(rows)/len(dataset)*100:.1f}% of dataset)")- return concatenate_datasets([dataset, Dataset.from_list(rows)])- return dataset---def load_and_combine():- parts = []- for name, hf_id in DATASETS.items():- split_override = None- if isinstance(hf_id, tuple):- if len(hf_id) == 3:- repo, subset, split_override = hf_id- else:- repo, subset = hf_id- label = f"{repo}" + (f" [{subset}]" if subset else "")- else:- repo, subset = hf_id, None- label = repo- print(f"\nLoading {name} ({label})...")- try:- if split_override:- raw = load_dataset(repo, subset, split=split_override) if subset else load_dataset(repo, split=split_override)- else:- ds = load_dataset(repo, subset) if subset else load_dataset(repo)- split = "train" if "train" in ds else list(ds.keys())[0]- raw = ds[split]- except Exception as e:- print(f" ERROR: {e}")- continue-- print(f" Converting to xlam format ({len(raw):,} rows)...")- converted = CONVERTERS[name](raw)- converted = converted.add_column("source", [name] * len(converted))-- before = len(converted)- converted = converted.filter(- lambda ex: (- not _has_placeholder_args(ex["answers"])- and _answer_calls_valid_tools(ex)- and _answer_uses_valid_params(ex)- ),- desc=f"Filtering {name}",- num_proc=_NUM_WORKERS,- )- dropped = before - len(converted)- if dropped:- print(f" Filtered out {dropped} low-quality examples ({dropped*100/before:.1f}%)")-- print(f" -> {len(converted)} examples")- parts.append(converted)-- combined = concatenate_datasets(parts)-- # Drop no-call examples with empty tools (trivial negatives)- before_empty = len(combined)- combined = combined.filter(- lambda ex: json.loads(ex["answers"]) != [] or json.loads(ex["tools"]) != [],- desc="Removing empty-tools no-call examples",- num_proc=_NUM_WORKERS,- )- empty_dropped = before_empty - len(combined)- if empty_dropped:- print(f"\nRemoved {empty_dropped} no-call examples with empty tools")-- before_dedup = len(combined)- combined = _deduplicate(combined)- dedup_dropped = before_dedup - len(combined)- if dedup_dropped:- print(f"\nDeduplicated: removed {dedup_dropped} duplicate queries")-- before_trim = len(combined)- combined = _trim_tools(combined, max_tools=5, seed=42)- print(f" {before_trim} -> {len(combined)} after tool trimming")-- combined = _normalize_tool_names(combined)- print(f" Normalized tool names to snake_case")-- combined = _augment_irrelevance(combined, ratio=0.1, seed=42)-- empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])- print(f"\n{'='*60}")- print(f"Combined dataset: {len(combined)} examples ({empty_count} with empty answers = no-call examples)")- print(f"Columns: {combined.column_names}")-- counts = Counter(combined["source"])- print(f"\nBreakdown:")- for src, cnt in counts.items():- print(f" {src}: {cnt}")-- seen = set()- for i in range(len(combined)):- src = combined[i]["source"]- if src in seen:- continue- seen.add(src)- print(f"\n--- Sample from {src} ---")- for k in ["query", "answers", "tools"]:- v = str(combined[i][k])[:200]- print(f" {k}: {v}")-- return combined---if __name__ == "__main__":- combined = load_and_combine()- out_path = "data/tool_calls"- combined.save_to_disk(out_path)- print(f"\nSaved to {out_path}/")-- from datasets import load_from_disk- reloaded = load_from_disk(out_path)- assert len(reloaded) == len(combined), f"Row count mismatch: {len(reloaded)} != {len(combined)}"- assert reloaded.column_names == combined.column_names, "Column mismatch"- assert reloaded[0] == combined[0], "First row mismatch"- print(f"Verified: {len(reloaded)} rows, columns {reloaded.column_names}")diff --git a/src/data.py b/src/data.pyindex b9aa2fe..8ce413d 100644--- a/src/data.py+++ b/src/data.py@@ -234,19 +234,21 @@ def _load_unified_dataset():continuetry:- from .gcs import download_synth_data+ from datasets import load_dataset as _hf_load+ print("Downloading dataset from HuggingFace (Cactus-Compute/tool-calls)...")+ ds = _hf_load("Cactus-Compute/tool-calls", split="train", token=True)target = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR- if download_synth_data(target):- ds = load_from_disk(target)- print(f"Loaded synth dataset from GCS -> {target} ({len(ds)} rows)")- _unified_dataset_cache = _set_audio_backend(ds)- return _unified_dataset_cache+ os.makedirs(target, exist_ok=True)+ ds.save_to_disk(target)+ print(f"Loaded from HuggingFace -> {target} ({len(ds)} rows)")+ _unified_dataset_cache = _set_audio_backend(ds)+ return _unified_dataset_cacheexcept Exception as e:- print(f"Warning: GCS download failed: {e}")+ print(f"Warning: HuggingFace download failed: {e}")raise FileNotFoundError(f"Dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "- f"Run 'needle tokenize' or 'python scripts/synthesize_tools_data.py' first."+ f"Run 'needle tokenize' or 'python scripts/push_to_hf.py' first.")@@ -371,12 +373,50 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):return model_path+_HF_TOKENIZED_REPO = "Cactus-Compute/tool-calls-tokenized"+++def _download_tokenized_from_hf():+ """Download tokenized .npy files from HuggingFace Hub into CACHE_DIR."""+ from huggingface_hub import snapshot_download++ local = snapshot_download(+ _HF_TOKENIZED_REPO,+ repo_type="dataset",+ allow_patterns="tokenized_data/*",+ token=True,+ )+ src = os.path.join(local, "tokenized_data")+ os.makedirs(CACHE_DIR, exist_ok=True)+ for fname in os.listdir(src):+ dst = os.path.join(CACHE_DIR, fname)+ if not os.path.exists(dst):+ os.symlink(os.path.join(src, fname), dst)+++def _download_tokenizer_from_hf():+ """Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""+ from huggingface_hub import snapshot_download++ local = snapshot_download(+ _HF_TOKENIZED_REPO,+ repo_type="dataset",+ allow_patterns="tokenizer/*",+ token=True,+ )+ src = os.path.join(local, "tokenizer")+ os.makedirs(TOKENIZER_DIR, exist_ok=True)+ for fname in os.listdir(src):+ dst = os.path.join(TOKENIZER_DIR, fname)+ if not os.path.exists(dst):+ os.symlink(os.path.join(src, fname), dst)++def get_tokenizer(max_samples=None):model_path = TOKENIZER_PREFIX + ".model"if not os.path.exists(model_path):try:- from .gcs import download_tokenizer- download_tokenizer(TOKENIZER_DIR)+ _download_tokenizer_from_hf()except Exception:passif not os.path.exists(model_path):@@ -648,7 +688,6 @@ def _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,query_arr[i, :l] = toks[:l]np.save(cache_path + "_query_only.npy", query_arr)- # Individual tool tokens + mapping arraystool_texts_individual = []tool_ex_idx = []tool_is_pos = []@@ -932,8 +971,7 @@ def load_prepared_data(split, mmap=False):meta = _load_cache_metadata(split)if meta is None:try:- from .gcs import download_tokenized_data- download_tokenized_data(CACHE_DIR)+ _download_tokenized_from_hf()meta = _load_cache_metadata(split)except Exception:passdiff --git a/src/export.py b/src/export.pyindex 8b5d00e..edd4cc5 100644--- a/src/export.py+++ b/src/export.py@@ -38,7 +38,7 @@ def export_submodel(checkpoint_path, factor, output_path):def slice_leaf(key_path, leaf):arr = np.asarray(leaf)- if arr.ndim != 2:+ if arr.ndim not in (2, 3):return arrparent_name = None@@ -51,13 +51,22 @@ def export_submodel(checkpoint_path, factor, output_path):if parent_name is None:return arr- rows, cols = arr.shape- if parent_name in ("gate_proj", "up_proj"):- if cols == d_ff:- return arr[:, :d_ff_new]- elif parent_name == "down_proj":- if rows == d_ff:- return arr[:d_ff_new, :]+ if arr.ndim == 2:+ rows, cols = arr.shape+ if parent_name in ("gate_proj", "up_proj"):+ if cols == d_ff:+ return arr[:, :d_ff_new]+ elif parent_name == "down_proj":+ if rows == d_ff:+ return arr[:d_ff_new, :]+ elif arr.ndim == 3:+ _, rows, cols = arr.shape+ if parent_name in ("gate_proj", "up_proj"):+ if cols == d_ff:+ return arr[:, :, :d_ff_new]+ elif parent_name == "down_proj":+ if rows == d_ff:+ return arr[:, :d_ff_new, :]return arrdiff --git a/src/model.py b/src/model.pyindex 5c94aa7..81b3134 100644--- a/src/model.py+++ b/src/model.py@@ -255,6 +255,30 @@ class EncoderBlock(nn.Module):return x+class _EncoderScanBody(nn.Module):+ """Wraps EncoderBlock for nn.scan: carry = (x, mask, rope, ffn_mask)."""+ num_heads: int+ num_kv_heads: int+ d_model: int+ d_ff: int+ num_layers: int+ dtype: jnp.dtype = jnp.bfloat16+ activation: str = "drelu"+ dropout_rate: float = 0.0+ conv_kernel_size: int = 0+ deterministic: bool = True++ @nn.compact+ def __call__(self, carry, _):+ x, mask, rope, ffn_mask = carry+ x = EncoderBlock(+ self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,+ self.num_layers, self.dtype, self.activation, self.dropout_rate,+ self.conv_kernel_size,+ )(x, mask, rope, ffn_mask, self.deterministic)+ return (x, mask, rope, ffn_mask), None++class Encoder(nn.Module):"""Self-attention encoder. Returns (output, mask) tuple."""config: TransformerConfig@@ -265,12 +289,17 @@ class Encoder(nn.Module):dt = cfg.jax_dtypex = x.astype(dt)- for i in range(cfg.num_encoder_layers):- x = nn.remat(EncoderBlock, static_argnums=(5,))(- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,- cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,- cfg.conv_kernel_size, name=f"block_{i}"- )(x, mask, rope, ffn_mask, deterministic)+ ScanBlock = nn.scan(+ nn.remat(_EncoderScanBody),+ variable_axes={"params": 0},+ split_rngs={"params": True, "dropout": True},+ length=cfg.num_encoder_layers,+ )+ (x, _, _, _), _ = ScanBlock(+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,+ cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,+ cfg.conv_kernel_size, deterministic, name="layers",+ )((x, mask, rope, ffn_mask), None)x = ZCRMSNorm(dtype=dt, name="final_norm")(x)return x, mask@@ -311,6 +340,28 @@ class DecoderBlock(nn.Module):+class _DecoderScanBody(nn.Module):+ """Wraps DecoderBlock for nn.scan: carry = (x, encoder_out, self_mask, cross_mask, rope, ffn_mask)."""+ num_heads: int+ num_kv_heads: int+ d_model: int+ d_ff: int+ num_layers: int+ dtype: jnp.dtype = jnp.bfloat16+ activation: str = "drelu"+ dropout_rate: float = 0.0+ deterministic: bool = True++ @nn.compact+ def __call__(self, carry, _):+ x, encoder_out, self_mask, cross_mask, rope, ffn_mask = carry+ x = DecoderBlock(+ self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,+ self.num_layers, self.dtype, self.activation, self.dropout_rate,+ )(x, encoder_out, self_mask, cross_mask, rope, ffn_mask, self.deterministic)+ return (x, encoder_out, self_mask, cross_mask, rope, ffn_mask), None++class Decoder(nn.Module):config: TransformerConfig@@ -320,10 +371,17 @@ class Decoder(nn.Module):dt = cfg.jax_dtypex = x.astype(dt)- for i in range(cfg.num_decoder_layers):- x = nn.remat(DecoderBlock, static_argnums=(7,))(- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, cfg.dropout_rate, name=f"block_{i}"- )(x, encoder_out, self_mask, cross_mask, rope, ffn_mask, deterministic)+ ScanBlock = nn.scan(+ nn.remat(_DecoderScanBody),+ variable_axes={"params": 0},+ split_rngs={"params": True, "dropout": True},+ length=cfg.num_decoder_layers,+ )+ (x, _, _, _, _, _), _ = ScanBlock(+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,+ cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,+ deterministic, name="layers",+ )((x, encoder_out, self_mask, cross_mask, rope, ffn_mask), None)x = ZCRMSNorm(dtype=dt)(x)return xdiff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex f6214ec..cc673ec 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -1,8 +1,8 @@"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.-Downloads the synthesized tool-calling dataset from GCS-(gs://cactus-dataset/synth_tool_calls/), trains the SentencePiece tokenizer,-and tokenizes train + val splits, caching everything locally.+Downloads the tool-calling dataset from HuggingFace (Cactus-Compute/tool-calls),+trains the SentencePiece tokenizer, and tokenizes train + val splits,+caching everything locally.Usage:needle tokenize # full run@@ -29,6 +29,36 @@ from .data import ()+_HF_TOKENIZED_REPO = "Cactus-Compute/tool-calls-tokenized"+++def _push_to_hf(cache_dir, tokenizer_dir):+ """Upload tokenized data and tokenizer to HuggingFace Hub."""+ from huggingface_hub import HfApi++ api = HfApi()+ api.create_repo(_HF_TOKENIZED_REPO, repo_type="dataset", private=True, exist_ok=True)++ print(f"Uploading tokenized data to {_HF_TOKENIZED_REPO} ...")+ api.upload_folder(+ folder_path=cache_dir,+ repo_id=_HF_TOKENIZED_REPO,+ repo_type="dataset",+ path_in_repo="tokenized_data",+ allow_patterns=["*.npy", "*.json"],+ )++ print(f"Uploading tokenizer to {_HF_TOKENIZED_REPO} ...")+ api.upload_folder(+ folder_path=tokenizer_dir,+ repo_id=_HF_TOKENIZED_REPO,+ repo_type="dataset",+ path_in_repo="tokenizer",+ allow_patterns=["*.model", "*.vocab"],+ )+ print("HuggingFace upload complete.")++def _clear_local_caches():"""Remove local .data_cache/, tokenizer/, and stale unified dataset directories."""for d in [CACHE_DIR, TOKENIZER_DIR, _DISK_UNIFIED_DIR, _SHM_UNIFIED_DIR]:@@ -38,23 +68,29 @@ def _clear_local_caches():def _download_synth_dataset():- """Download synthesized tool-calling dataset from GCS."""- from .gcs import download_synth_data+ """Download synthesized tool-calling dataset from HuggingFace."""+ from datasets import load_datasettarget = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR- print(f"Downloading synth dataset to {target} ...")- if not download_synth_data(target):+ print(f"Downloading dataset from HuggingFace (Cactus-Compute/tool-calls)...")+ try:+ ds = load_dataset("Cactus-Compute/tool-calls", split="train", token=True)+ except Exception as e:raise FileNotFoundError(- "Synth dataset not found at gs://cactus-dataset/synth_tool_calls/. "- "Run 'python scripts/synthesize_tools_data.py' first."+ f"Dataset not found on HuggingFace (Cactus-Compute/tool-calls): {e}\n"+ "Run 'python scripts/push_to_hf.py' first.")+ import os+ os.makedirs(target, exist_ok=True)+ ds.save_to_disk(target)+ print(f"Saved {len(ds):,} rows to {target}")def tokenize(args):print("=== Clearing existing caches ===")_clear_local_caches()- print("\n=== Downloading synth dataset from GCS ===")+ print("\n=== Downloading dataset from HuggingFace ===")_download_synth_dataset()print("\n=== Training tokenizer ===")@@ -89,8 +125,6 @@ def tokenize(args):_save_cache_metadata(split, text_cache_id, len(kept_indices),max_enc_len, max_dec_len)- from .gcs import upload_tokenized_data, upload_tokenizer- upload_tokenizer(TOKENIZER_DIR)- upload_tokenized_data(CACHE_DIR)+ _push_to_hf(CACHE_DIR, TOKENIZER_DIR)print("\n=== Tokenization pipeline complete ===")diff --git a/src/train.py b/src/train.pyindex fc88861..f49d9de 100644--- a/src/train.py+++ b/src/train.py@@ -57,6 +57,8 @@ def scale_by_muon(momentum=0.95, ns_steps=5):del paramsdef ortho(g):+ if g.ndim == 3:+ return jax.vmap(_newton_schulz, in_axes=(0,))(g)if g.ndim == 2:return _newton_schulz(g, steps=ns_steps)return g@@ -76,7 +78,7 @@ def _param_labels(params):def _label(path, leaf):name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if name == "kernel" and leaf.ndim == 2:+ if name == "kernel" and leaf.ndim in (2, 3):return "muon"return "adam"@@ -180,17 +182,25 @@ def _make_prune_mask(params, sparsity, group_size):"""Compute block-prune mask entirely on-device (no numpy round-trips).Returns binary mask tree matching param shapes and dtypes.+ Handles both 2D (in, out) and 3D (num_layers, in, out) kernels from nn.scan."""all_scores = []for _, leaf in jax.tree_util.tree_leaves_with_path(params):- if leaf.ndim != 2:- continue- in_feat, out_feat = leaf.shape- gs = min(group_size, in_feat)- pad = (gs - in_feat % gs) % gs- w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf- scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()- all_scores.append(scores)+ if leaf.ndim == 2:+ in_feat, out_feat = leaf.shape+ gs = min(group_size, in_feat)+ pad = (gs - in_feat % gs) % gs+ w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()+ all_scores.append(scores)+ elif leaf.ndim == 3:+ num_layers, in_feat, out_feat = leaf.shape+ gs = min(group_size, in_feat)+ pad = (gs - in_feat % gs) % gs+ for li in range(num_layers):+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()+ all_scores.append(scores)if not all_scores:return jax.tree.map(jnp.ones_like, params)@@ -198,6 +208,18 @@ def _make_prune_mask(params, sparsity, group_size):threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)def _leaf_mask(path, leaf):+ if leaf.ndim == 3:+ num_layers, in_feat, out_feat = leaf.shape+ gs = min(group_size, in_feat)+ pad = (gs - in_feat % gs) % gs+ masks = []+ for li in range(num_layers):+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]+ w_grouped = w.reshape(-1, gs, out_feat)+ block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)+ m = jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat]+ masks.append(m)+ return jnp.stack(masks, axis=0).astype(leaf.dtype)if leaf.ndim != 2:return jnp.ones_like(leaf)in_feat, out_feat = leaf.shape@@ -215,6 +237,8 @@ def _quantize_params(params, group_size=32):"""Fake-quantize all Dense kernels in the param tree."""def _maybe_quantize(path, leaf):name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])+ if name == "kernel" and leaf.ndim == 3:+ return jax.vmap(_fake_quantize_int4, in_axes=(0,))(leaf)if name == "kernel" and leaf.ndim == 2:return _fake_quantize_int4(leaf, group_size=group_size)return leaf@@ -239,9 +263,14 @@ def _clip_contrastive_loss(q_emb, t_emb, log_temp):return (jnp.mean(loss_q) + jnp.mean(loss_t)) / 2.0-def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):+def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, do_quantize):pad_id = 0- q_params = _quantize_params(params, group_size=_GROUP_SIZE)+ q_params = jax.lax.cond(+ do_quantize,+ lambda p: _quantize_params(p, group_size=_GROUP_SIZE),+ lambda p: p,+ params,+ )src_mask = make_padding_mask(src, pad_id)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)logits, slot_div = state.apply_fn(@@ -261,9 +290,14 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rnreturn ce_loss + z_loss-def _contrastive_loss_fn(state, params, query_tokens, tool_tokens, rng):+def _contrastive_loss_fn(state, params, query_tokens, tool_tokens, rng, do_quantize):"""Compute CLIP contrastive loss on query/tool pairs."""- q_params = _quantize_params(params, group_size=_GROUP_SIZE)+ q_params = jax.lax.cond(+ do_quantize,+ lambda p: _quantize_params(p, group_size=_GROUP_SIZE),+ lambda p: p,+ params,+ )q_emb, t_emb, log_temp = state.apply_fn({"params": q_params},query_tokens, tool_tokens,@@ -293,87 +327,36 @@ def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):return jnp.concatenate(rows, axis=0)-def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):- ema_decay = 0.999- loss, grads = jax.value_and_grad(- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)- )(state.params)- grads = jax.lax.pmean(grads, axis_name="batch")- loss = jax.lax.pmean(loss, axis_name="batch")- grad_norm = optax.global_norm(grads)- state = state.apply_gradients(grads=grads)- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)- return state, ema_params, loss, grad_norm---def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask):- """Text training step with fused prune mask application."""+def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng, do_quantize, do_contrastive, do_prune):+ """Unified train step with boolean flags for contrastive loss and pruning."""ema_decay = 0.999- loss, grads = jax.value_and_grad(- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)- )(state.params)- grads = jax.lax.pmean(grads, axis_name="batch")- loss = jax.lax.pmean(loss, axis_name="batch")- grad_norm = optax.global_norm(grads)- state = state.apply_gradients(grads=grads)- masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)- state = state.replace(params=masked_params)- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)- return state, ema_params, loss, grad_norm--def _train_step_fused(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng):- """Fused text + contrastive training step — single backward pass & optimizer step."""- ema_decay = 0.999def combined_loss(p):- text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)- cl_loss = _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng)+ text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, do_quantize)+ cl_loss = jax.lax.cond(+ do_contrastive,+ lambda: _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng, do_quantize),+ lambda: 0.0,+ )return text_loss + _CONTRASTIVE_WEIGHT * cl_loss, text_loss- (loss, text_loss), grads = jax.value_and_grad(combined_loss, has_aux=True)(state.params)- grads = jax.lax.pmean(grads, axis_name="batch")- text_loss = jax.lax.pmean(text_loss, axis_name="batch")- grad_norm = optax.global_norm(grads)- state = state.apply_gradients(grads=grads)- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)- return state, ema_params, text_loss, grad_norm--def _train_step_fused_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng):- """Fused text + contrastive training step with prune mask application."""- ema_decay = 0.999-- def combined_loss(p):- text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)- cl_loss = _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng)- return text_loss + _CONTRASTIVE_WEIGHT * cl_loss, text_loss-(loss, text_loss), grads = jax.value_and_grad(combined_loss, has_aux=True)(state.params)grads = jax.lax.pmean(grads, axis_name="batch")text_loss = jax.lax.pmean(text_loss, axis_name="batch")grad_norm = optax.global_norm(grads)state = state.apply_gradients(grads=grads)- masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)- state = state.replace(params=masked_params)- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)+ new_params = jax.lax.cond(+ do_prune,+ lambda: jax.tree.map(lambda w, m: w * m, state.params, prune_mask),+ lambda: state.params,+ )+ state = state.replace(params=new_params)+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, new_params)return state, ema_params, text_loss, grad_norm--def _make_p_train_step():- return jax.pmap(_train_step_text, axis_name="batch", donate_argnums=(0, 1))---def _make_p_train_step_masked():- return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))---def _make_p_train_step_fused():- return jax.pmap(_train_step_fused, axis_name="batch", donate_argnums=(0, 1))---def _make_p_train_step_fused_masked():- return jax.pmap(_train_step_fused_masked, axis_name="batch", donate_argnums=(0, 1))+ return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0, 1))def _make_val_loss_fn(apply_fn):@@ -521,9 +504,6 @@ def train(args):_MAT_FF_WIDTHS = ()n_widths = 1 + len(_MAT_FF_WIDTHS) if _MAT_FF_WIDTHS else 1p_train_step = _make_p_train_step()- p_train_step_masked = _make_p_train_step_masked()- p_train_step_fused = _make_p_train_step_fused()- p_train_step_fused_masked = _make_p_train_step_fused_masked()np.random.seed(args.seed)rng = jax.random.PRNGKey(args.seed)@@ -598,6 +578,13 @@ def train(args):sparsity_ratio = getattr(args, "sparsity_ratio", 0.0)prune_mask = Nonegradual_sparsify_done = False+ # Dummy all-ones prune mask (replicated) — used when pruning is inactive+ _unrep_params = jax_utils.unreplicate(state).params+ dummy_prune_mask = jax_utils.replicate(jax.tree.map(jnp.ones_like, _unrep_params))+ del _unrep_params+ # Dummy contrastive arrays — used when contrastive is inactive+ dummy_cl_tokens = jnp.zeros((num_devices, args.batch_size, 128), dtype=jnp.int32)+ dummy_cl_rng = jax.random.split(jax.random.PRNGKey(0), num_devices)prune_interval = getattr(args, "prune_interval", 100)prune_start_frac = getattr(args, "prune_start_frac", 0.33)@@ -647,36 +634,30 @@ def train(args):rng, text_rng = jax.random.split(rng)text_rngs = jax.random.split(text_rng, num_devices)+ do_qat = jnp.broadcast_to(jnp.array(global_step % 100 == 0), (num_devices,))+do_contrastive = cl_batch_iter is not None and global_step % 100 == 0+ cl_q_b, cl_t_b, cl_rngs = dummy_cl_tokens, dummy_cl_tokens, dummy_cl_rngif do_contrastive:try:cl_q, cl_t = next(cl_batch_iter)cl_q_b = shard_batch(cl_q, num_devices)cl_t_b = shard_batch(cl_t, num_devices)+ rng, cl_rng = jax.random.split(rng)+ cl_rngs = jax.random.split(cl_rng, num_devices)except StopIteration:cl_batch_iter = Nonedo_contrastive = False- if do_contrastive:- rng, cl_rng = jax.random.split(rng)- cl_rngs = jax.random.split(cl_rng, num_devices)- if prune_mask is not None:- state, ema_params, loss, grad_norm = p_train_step_fused_masked(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b, cl_q_b, cl_t_b, cl_rngs,- )- else:- state, ema_params, loss, grad_norm = p_train_step_fused(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b, cl_q_b, cl_t_b, cl_rngs,- )- else:- if prune_mask is not None:- state, ema_params, loss, grad_norm = p_train_step_masked(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,- )- else:- state, ema_params, loss, grad_norm = p_train_step(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,- )+ do_cl = jnp.broadcast_to(jnp.array(do_contrastive), (num_devices,))+ do_prune = jnp.broadcast_to(jnp.array(prune_mask is not None), (num_devices,))+ cur_prune_mask = prune_mask if prune_mask is not None else dummy_prune_mask++ state, ema_params, loss, grad_norm = p_train_step(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask,+ cur_prune_mask, text_ffn_mask, text_rngs, lm_b,+ cl_q_b, cl_t_b, cl_rngs, do_qat, do_cl, do_prune,+ )text_loss_val = float(loss[0])text_losses.append(text_loss_val)