Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Refactor
HenryNdubuaku5778f470c934
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.pyindex 6fcf697..58d6696 100644--- a/scripts/synthesize_tools_data.py+++ b/scripts/synthesize_tools_data.py@@ -1,13 +1,19 @@#!/usr/bin/env python3-"""Synthesize tool-calling data from PleIAs/SYNTH queries via OpenRouter.+"""Synthesize tool-calling data from multiple query sources via OpenRouter.-Downloads the first parquet shard of PleIAs/SYNTH (~1M diverse multilingual-queries), then prompts an LLM to generate (tools, answer) pairs for each query.+Query sources:+ 1. PleIAs/SYNTH parquet shards (~154k queries per shard, 7 shards)+ 2. GCS raw_data — queries extracted from the existing unified tool-calling dataset++Each query is sent to a randomly-selected frontier model to generate+(tools, answer) pairs, then validated and checkpointed as JSONL.Usage:- python scripts/synthesize_tools_data.py- python scripts/synthesize_tools_data.py --max-samples 10000 --model google/gemini-2.5-flash- python scripts/synthesize_tools_data.py --resume # continue from last checkpoint+ python scripts/synthesize_tools_data.py # all sources+ python scripts/synthesize_tools_data.py --model google/gemini-3.1-flash-lite-preview+ python scripts/synthesize_tools_data.py --max-shards 1 --max-samples 100+ python scripts/synthesize_tools_data.py --no-gcs # skip GCS queries+ python scripts/synthesize_tools_data.py --resume # continue from checkpoint"""import argparse@@ -28,10 +34,17 @@ logger = logging.getLogger(__name__)_PROJECT_ROOT = Path(__file__).resolve().parent.parent_OUTPUT_DIR = _PROJECT_ROOT / "data" / "synth_tool_calls"_CHECKPOINT_PATH = _OUTPUT_DIR / "checkpoint.jsonl"-_PARQUET_CACHE = _PROJECT_ROOT / ".data_cache" / "synth_shard0.parquet"+_PARQUET_CACHE_DIR = _PROJECT_ROOT / ".data_cache"OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"-DEFAULT_MODEL = "google/gemini-2.5-flash"++MODEL_POOL = [+ "google/gemini-3.1-flash-lite-preview",+ # "anthropic/claude-opus-4.6",+ # "x-ai/grok-4.1-fast",+ # "openai/gpt-5.3-chat",+]+DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"SYSTEM_PROMPT = """\You are a synthetic data generator for training a tool-calling AI model.@@ -39,15 +52,27 @@ You are a synthetic data generator for training a tool-calling AI model.Given a user query, you must invent a realistic set of tools (APIs/functions) that \could help answer it, then produce the correct tool call(s).+Vary the domain widely — cover areas like software development, data analysis, \+e-commerce, finance, healthcare, travel, IoT, social media, education, DevOps, legal, \+HR, and more. Do NOT default to simple personal-assistant tasks.+Rules:-1. Invent 3-8 tools as JSON. Each tool has: name, description, parameters (dict of param_name -> {type, description}). \-Only ONE or TWO tools should be relevant; the rest are distractors.-2. Tool names should be snake_case, realistic API-style (e.g. search_web, get_weather, translate_text).-3. Produce the answer as a JSON list of tool calls: [{"name": "...", "arguments": {...}}].-4. If the query genuinely needs no tool, return an empty list [].-5. Argument values MUST be grounded in the query — do not hallucinate values.-6. Respond in the SAME LANGUAGE as the query.-7. Return ONLY valid JSON, no markdown fences, no commentary.+1. Invent 1-5 tools as JSON. Each tool MUST have: name, description, parameters \+(dict of param_name -> {type, description, required}). Use realistic parameter types \+including string, number, boolean, array, object. Include nested objects or arrays \+where appropriate.+2. Only ONE or TWO tools should be relevant to the query; the rest are plausible \+distractors from the SAME domain.+3. Tool names should be snake_case, realistic API-style, and DIVERSE — avoid repeating \+common names like search_web or get_weather across samples. Think of specific, \+domain-appropriate APIs (e.g. query_elasticsearch, run_sql_migration, price_option_chain, \+schedule_k8s_cronjob, fetch_patient_labs).+4. Produce the answer as a JSON list of tool calls: [{"name": "...", "arguments": {...}}].+5. For complex queries, use MULTIPLE tool calls (chained or parallel) where realistic.+6. If the query genuinely needs no tool, return an empty answer list [].+7. Argument values MUST be grounded in the query — do not hallucinate values.+8. Respond in the SAME LANGUAGE as the query.+9. Return ONLY valid JSON, no markdown fences, no commentary.Output format (strict JSON object):{"tools": [...], "answer": [...]}"""@@ -55,29 +80,42 @@ Output format (strict JSON object):USER_TEMPLATE = "Query: {query}"-def download_first_shard(token):- """Download the first parquet shard of PleIAs/SYNTH."""- if _PARQUET_CACHE.exists():- logger.info(f"Using cached shard: {_PARQUET_CACHE}")- return _PARQUET_CACHE+def _discover_shard_urls(token, max_shards=None):+ """Discover available parquet shard URLs from the HF API."""+ api_url = "https://huggingface.co/api/datasets/PleIAs/SYNTH/parquet"+ headers = {}+ if token:+ headers["Authorization"] = f"Bearer {token}"+ resp = requests.get(api_url, headers=headers, timeout=30)+ resp.raise_for_status()+ data = resp.json() # {"default": {"train": [url, ...]}}+ urls = data["default"]["train"]+ if max_shards:+ urls = urls[:max_shards]+ logger.info(f"Discovered {len(urls)} parquet shards")+ return urls+++def download_shard(url, shard_idx, token):+ """Download a single parquet shard, returning the local path."""+ cache_path = _PARQUET_CACHE_DIR / f"synth_shard{shard_idx}.parquet"+ if cache_path.exists():+ logger.info(f"Using cached shard {shard_idx}: {cache_path}")+ return cache_path- _PARQUET_CACHE.parent.mkdir(parents=True, exist_ok=True)+ _PARQUET_CACHE_DIR.mkdir(parents=True, exist_ok=True)- url = (- "https://huggingface.co/datasets/PleIAs/SYNTH"- "/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet"- )headers = {}if token:headers["Authorization"] = f"Bearer {token}"- logger.info(f"Downloading first shard from PleIAs/SYNTH...")+ logger.info(f"Downloading shard {shard_idx}...")resp = requests.get(url, headers=headers, stream=True, timeout=120)resp.raise_for_status()total = int(resp.headers.get("content-length", 0))downloaded = 0- with open(_PARQUET_CACHE, "wb") as f:+ with open(cache_path, "wb") as f:for chunk in resp.iter_content(chunk_size=8 * 1024 * 1024):f.write(chunk)downloaded += len(chunk)@@ -85,12 +123,12 @@ def download_first_shard(token):pct = downloaded / total * 100print(f"\r {downloaded / 1e6:.0f}/{total / 1e6:.0f} MB ({pct:.0f}%)", end="", flush=True)print()- logger.info(f"Saved to {_PARQUET_CACHE}")- return _PARQUET_CACHE+ logger.info(f"Saved shard {shard_idx} to {cache_path}")+ return cache_path-def load_queries(path, max_samples=None):- """Load queries from parquet, return list of (synth_id, language, query)."""+def load_queries_from_parquet(path, max_samples=None):+ """Load queries from parquet, return list of (sample_id, language, query)."""import pyarrow.parquet as pqlogger.info(f"Reading parquet: {path}")@@ -104,7 +142,6 @@ def load_queries(path, max_samples=None):samples = list(zip(ids, langs, queries))- # Shuffle for diversity (don't process in order)rng = random.Random(42)rng.shuffle(samples)@@ -115,6 +152,48 @@ def load_queries(path, max_samples=None):return samples+def load_queries_from_gcs(max_samples=None):+ """Download raw_data from GCS and extract queries for re-synthesis.++ Returns list of (sample_id, language, query) tuples, where sample_id+ is prefixed with 'gcs-' to avoid collisions with PleIAs IDs.+ """+ sys.path.insert(0, str(_PROJECT_ROOT))+ from src.gcs import download_raw_data++ local_dir = _PARQUET_CACHE_DIR / "gcs_raw_data"+ if not local_dir.exists():+ logger.info("Downloading raw_data from GCS...")+ ok = download_raw_data(str(local_dir))+ if not ok:+ logger.warning("No raw_data found in GCS, skipping")+ return []+ else:+ logger.info(f"Using cached GCS raw_data: {local_dir}")++ from datasets import load_from_disk+ ds = load_from_disk(str(local_dir))+ logger.info(f"Loaded {len(ds):,} rows from GCS raw_data")++ samples = []+ for i in range(len(ds)):+ ex = ds[i]+ query = ex.get("query", "").strip()+ if not query:+ continue+ sample_id = f"gcs-{i}"+ samples.append((sample_id, "en", query))++ rng = random.Random(42)+ rng.shuffle(samples)++ if max_samples:+ samples = samples[:max_samples]++ logger.info(f" Selected {len(samples):,} GCS queries")+ return samples++def load_checkpoint():"""Load completed synth_ids from checkpoint file."""done = set()@@ -143,7 +222,7 @@ def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):{"role": "user", "content": USER_TEMPLATE.format(query=query)},],"temperature": 0.7,- "max_tokens": 2048,+ "max_tokens": 4096,}for attempt in range(max_retries):@@ -153,12 +232,22 @@ def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):)if resp.status_code == 429:wait = min(2 ** attempt * 2, 30)+ logger.warning(f"[{model}] 429 rate-limited, retrying in {wait}s")time.sleep(wait)continue- resp.raise_for_status()+ if not resp.ok:+ logger.warning(f"[{model}] HTTP {resp.status_code}: {resp.text[:200]}")+ if attempt == max_retries - 1:+ return None+ time.sleep(2 ** attempt)+ continuecontent = resp.json()["choices"][0]["message"]["content"]- return _parse_response(content)+ parsed = _parse_response(content)+ if parsed is None:+ logger.warning(f"[{model}] Failed to parse response: {content[:200]}")+ return parsedexcept (requests.RequestException, KeyError, IndexError) as e:+ logger.warning(f"[{model}] Request error (attempt {attempt+1}/{max_retries}): {e}")if attempt == max_retries - 1:return Nonetime.sleep(2 ** attempt)@@ -168,7 +257,6 @@ def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):def _parse_response(content):"""Parse LLM response into (tools, answer) or None."""content = content.strip()- # Strip markdown fences if presentif content.startswith("```"):content = content.split("\n", 1)[-1]if content.endswith("```"):@@ -191,12 +279,10 @@ def _parse_response(content):if not isinstance(answer, list):return None- # Validate tool structurefor t in tools:if not isinstance(t, dict) or "name" not in t:return None- # Validate answer structurefor a in answer:if not isinstance(a, dict) or "name" not in a:return None@@ -206,15 +292,13 @@ def _parse_response(content):return tools, answer-def _validate_and_format(synth_id, language, query, tools, answer):+def _validate_and_format(synth_id, language, query, tools, answer, model, source="synth-pleias"):"""Validate the generated data and return a formatted row or None."""- # Check that every called tool exists in the tool listtool_names = {t["name"] for t in tools}for a in answer:if a["name"] not in tool_names:return None- # Check argument keys are validtool_params = {}for t in tools:params = t.get("parameters") or {}@@ -234,64 +318,67 @@ def _validate_and_format(synth_id, language, query, tools, answer):"query": query,"answers": json.dumps(answer),"tools": json.dumps(tools),- "source": "synth-pleias",+ "source": source,+ "model": model,}-def process_sample(sample, api_key, model):+def _pick_model(models, rng):+ """Randomly select a model from the pool."""+ return rng.choice(models)+++def process_sample(sample, api_key, models, rng, source_tag="synth-pleias"):"""Process a single sample: call LLM, validate, return row or None."""synth_id, language, query = sample+ model = _pick_model(models, rng)result = call_openrouter(query, api_key, model=model)if result is None:return Nonetools, answer = result- return _validate_and_format(synth_id, language, query, tools, answer)---def main():- parser = argparse.ArgumentParser(description="Synthesize tool-calling data from PleIAs/SYNTH")- parser.add_argument("--max-samples", type=int, default=None,- help="Max queries to process (default: all in shard)")- parser.add_argument("--model", type=str, default=DEFAULT_MODEL,- help=f"OpenRouter model (default: {DEFAULT_MODEL})")- parser.add_argument("--workers", type=int, default=32,- help="Concurrent API requests (default: 32)")- parser.add_argument("--resume", action="store_true",- help="Resume from checkpoint")- parser.add_argument("--batch-size", type=int, default=500,- help="Samples between progress logs (default: 500)")- args = parser.parse_args()+ row = _validate_and_format(synth_id, language, query, tools, answer, model, source_tag)+ if row is None:+ logger.warning(f"[{model}] Validation failed for synth_id={synth_id}")+ return row- api_key = os.environ.get("OPENROUTER_API_KEY")- if not api_key:- print("Error: set OPENROUTER_API_KEY environment variable")- sys.exit(1)- hf_token = os.environ.get("HF_TOKEN")- parquet_path = download_first_shard(hf_token)- samples = load_queries(parquet_path, max_samples=args.max_samples)-- _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)-- done_ids = load_checkpoint() if args.resume else set()- if not args.resume and _CHECKPOINT_PATH.exists():- _CHECKPOINT_PATH.unlink()+def _process_batch(samples, label, batch_idx, api_key, models, args, tqdm, source_tag):+ """Process a batch of samples, append to checkpoint. Returns (generated, failed)."""+ logger.info(f"\n{'=' * 60}")+ logger.info(f"Batch: {label}")+ logger.info(f"{'=' * 60}")+ # Always load checkpoint to skip already-completed IDs (even across batches)+ done_ids = load_checkpoint()pending = [s for s in samples if s[0] not in done_ids]+ if not pending:+ logger.info("All samples already done, skipping")+ return 0, 0logger.info(f"Processing {len(pending):,} samples ({len(done_ids):,} already done)")generated = 0failed = 0out_file = open(_CHECKPOINT_PATH, "a")+ rng = random.Random(42 + batch_idx)try:with ThreadPoolExecutor(max_workers=args.workers) as executor:futures = {}for sample in pending:- fut = executor.submit(process_sample, sample, api_key, args.model)- futures[fut] = sample[0] # synth_id-- for i, fut in enumerate(as_completed(futures), 1):+ fut = executor.submit(+ process_sample, sample, api_key, models, rng, source_tag,+ )+ futures[fut] = sample[0]++ pbar = tqdm(+ as_completed(futures),+ total=len(futures),+ desc=label,+ unit="sample",+ dynamic_ncols=True,+ ) if tqdm else as_completed(futures)++ for i, fut in enumerate(pbar if tqdm else as_completed(futures), 1):try:row = fut.result()except Exception:@@ -304,31 +391,120 @@ def main():else:failed += 1- if i % args.batch_size == 0:- out_file.flush()- total = generated + failed- rate = generated / max(total, 1) * 100- logger.info(- f" [{total:,}/{len(pending):,}] "- f"generated={generated:,} failed={failed:,} "- f"({rate:.0f}% success)"- )+ if tqdm:+ rate = generated / max(generated + failed, 1) * 100+ pbar.set_postfix(ok=generated, fail=failed, rate=f"{rate:.0f}%")+ if i % args.batch_size == 0:+ out_file.flush()+ else:+ if i % args.batch_size == 0:+ out_file.flush()+ total = generated + failed+ rate = generated / max(total, 1) * 100+ logger.info(+ f" [{total:,}/{len(pending):,}] "+ f"generated={generated:,} failed={failed:,} "+ f"({rate:.0f}% success)"+ )++ if tqdm:+ pbar.close()except KeyboardInterrupt:logger.info("\nInterrupted — progress saved to checkpoint")finally:out_file.flush()out_file.close()- logger.info(f"\nDone: {generated:,} generated, {failed:,} failed")- logger.info(f"Output: {_CHECKPOINT_PATH}")+ logger.info(f"{label}: {generated:,} generated, {failed:,} failed")+ return generated, failed+++def main():+ parser = argparse.ArgumentParser(description="Synthesize tool-calling data from PleIAs/SYNTH")+ parser.add_argument("--max-samples", type=int, default=None,+ help="Max queries to process (default: all in shard)")+ parser.add_argument("--models", type=str, nargs="+", default=None,+ help=f"OpenRouter models to rotate (default: built-in pool)")+ parser.add_argument("--model", type=str, default=None,+ help="Use a single model instead of the pool")+ parser.add_argument("--workers", type=int, default=32,+ help="Concurrent API requests (default: 32)")+ parser.add_argument("--max-shards", type=int, default=None,+ help="Max parquet shards to process (default: all)")+ parser.add_argument("--no-gcs", action="store_true",+ help="Skip GCS raw_data queries")+ parser.add_argument("--resume", action="store_true",+ help="Resume from checkpoint")+ parser.add_argument("--batch-size", type=int, default=500,+ help="Samples between progress logs (default: 500)")+ args = parser.parse_args()++ api_key = os.environ.get("OPENROUTER_API_KEY")+ if not api_key:+ print("Error: set OPENROUTER_API_KEY environment variable")+ sys.exit(1)++ if args.model:+ models = [args.model]+ elif args.models:+ models = args.models+ else:+ models = list(MODEL_POOL)+ logger.info(f"Model pool ({len(models)}): {', '.join(models)}")++ _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)++ # Clear checkpoint for fresh runs; --resume keeps existing data+ if not args.resume and _CHECKPOINT_PATH.exists():+ _CHECKPOINT_PATH.unlink()+ logger.info("Cleared previous checkpoint (use --resume to keep)")++ try:+ from tqdm import tqdm+ except ImportError:+ logger.warning("tqdm not installed, falling back to basic progress logging")+ tqdm = None++ total_generated = 0+ total_failed = 0+ batch_idx = 0++ # ── Source 1: GCS raw_data queries ──+ if not args.no_gcs:+ gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)+ if gcs_samples:+ gen, fail = _process_batch(+ gcs_samples, "GCS raw_data", batch_idx, api_key, models,+ args, tqdm, source_tag="synth-gcs",+ )+ total_generated += gen+ total_failed += fail+ batch_idx += 1+ if gen > 0:+ _export_dataset()++ # ── Source 2: PleIAs/SYNTH parquet shards ──+ hf_token = os.environ.get("HF_TOKEN")+ shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)++ for shard_idx, shard_url in enumerate(shard_urls):+ parquet_path = download_shard(shard_url, shard_idx, hf_token)+ samples = load_queries_from_parquet(parquet_path, max_samples=args.max_samples)+ gen, fail = _process_batch(+ samples, f"PleIAs shard {shard_idx}", batch_idx, api_key, models,+ args, tqdm, source_tag="synth-pleias",+ )+ total_generated += gen+ total_failed += fail+ batch_idx += 1+ if gen > 0:+ _export_dataset()- # Convert checkpoint to HF dataset format- if generated > 0:- _export_dataset()+ logger.info(f"\nAll done: {total_generated:,} generated, {total_failed:,} failed")def _export_dataset():- """Convert checkpoint JSONL to the unified format expected by build_dataset.py."""+ """Convert checkpoint JSONL to the unified format and push to GCS."""from datasets import Datasetrows = []@@ -341,6 +517,7 @@ def _export_dataset():"answers": row["answers"],"tools": row["tools"],"source": row.get("source", "synth-pleias"),+ "model": row.get("model", "unknown"),})except (json.JSONDecodeError, KeyError):continue@@ -352,9 +529,29 @@ def _export_dataset():out_path = _OUTPUT_DIR / "dataset"ds.save_to_disk(str(out_path))logger.info(f"Exported {len(rows):,} examples to {out_path}/")- logger.info(- "To include in training, add to tools_data.py or merge with unified dataset."- )++ # Push to GCS+ try:+ sys.path.insert(0, str(_PROJECT_ROOT))+ from src.gcs import upload_directory+ gcs_prefix = "synth_tool_calls"+ upload_directory(str(out_path), gcs_prefix)+ except Exception as e:+ logger.warning(f"GCS upload failed: {e}")++ for i, row in enumerate(rows[:10]):+ try:+ tools = json.loads(row["tools"])+ answers = json.loads(row["answers"])+ except (json.JSONDecodeError, TypeError):+ tools, answers = [], []+ called_names = [a["name"] for a in answers if isinstance(a, dict)]+ print(f"\n{'─' * 60}")+ print(f"[{i+1}] Query: {row['query'][:200]}")+ print(f" Available tools ({len(tools)}): {', '.join(t.get('name','?') for t in tools)}")+ print(f" Called tools ({len(called_names)}): {', '.join(called_names) or '(none)'}")+ print(f" Answers: {row['answers'][:200]}")+ print(f"{'─' * 60}")if __name__ == "__main__":diff --git a/scripts/tools_data.py b/scripts/tools_data.pyindex 2e8dce9..8a23e68 100755--- a/scripts/tools_data.py+++ b/scripts/tools_data.py@@ -50,7 +50,7 @@ DATASETS = {"toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),"toucan-sft": ("Agent-Ark/Toucan-1.5M", "SFT"),"toolace": "Team-ACE/ToolACE",- # "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),+ "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),}@@ -675,6 +675,39 @@ def _trim_tools(dataset, max_tools=5, seed=42):return Dataset.from_list(rows)+def _normalize_tool_names(dataset):+ """Normalize all tool names to snake_case in both tools and answers fields."""+ from src.data import to_snake_case++ rows = []+ for i in range(len(dataset)):+ ex = dataset[i]+ try:+ tools = json.loads(ex["tools"])+ except (json.JSONDecodeError, TypeError):+ rows.append(ex)+ continue+ try:+ answers = json.loads(ex["answers"])+ except (json.JSONDecodeError, TypeError):+ answers = []++ for t in tools:+ if isinstance(t, dict) and "name" in t:+ t["name"] = to_snake_case(t["name"])+ for a in answers:+ if isinstance(a, dict) and "name" in a:+ a["name"] = to_snake_case(a["name"])++ rows.append({+ "query": ex["query"],+ "answers": json.dumps(answers),+ "tools": json.dumps(tools),+ "source": ex["source"],+ })+ return Dataset.from_list(rows)++def _augment_irrelevance(dataset, ratio=0.1, seed=42):"""Create irrelevance samples by swapping tools with unrelated queries.@@ -803,6 +836,9 @@ def load_and_combine():combined = _trim_tools(combined, max_tools=5, seed=42)print(f" {before_trim} -> {len(combined)} after tool trimming")+ combined = _normalize_tool_names(combined)+ print(f" Normalized tool names to snake_case")+combined = _augment_irrelevance(combined, ratio=0.1, seed=42)empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])diff --git a/src/cli.py b/src/cli.pyindex b3313bc..f17a7a3 100644--- a/src/cli.py+++ b/src/cli.py@@ -19,11 +19,11 @@ def main():p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--d-model", type=int, default=768)p.add_argument("--num-heads", type=int, default=16)p.add_argument("--num-kv-heads", type=int, default=8)- p.add_argument("--num-layers", type=int, default=6)- p.add_argument("--num-dec-layers", type=int, default=6)+ p.add_argument("--num-layers", type=int, default=8)+ p.add_argument("--num-dec-layers", type=int, default=8)p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)p.add_argument("--max-samples", type=int, default=None)@@ -42,8 +42,9 @@ def main():help="Fraction of epoch to train before starting gradual pruning (default: 0.33)")p.add_argument("--prune-end-frac", type=float, default=0.67,help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")- p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])- p.add_argument("--num-memory-slots", type=int, default=128)+ p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])+ p.add_argument("--num-memory-slots", type=int, default=128,+ help="(DEPRECATED — ignored, kept for checkpoint compat)")p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")p.add_argument("--dropout", type=float, default=0.1,diff --git a/src/data.py b/src/data.pyindex fffca9f..8934139 100644--- a/src/data.py+++ b/src/data.py@@ -19,6 +19,18 @@ import sentencepiece as spmimport re as _re+def to_snake_case(name):+ """Convert camelCase, PascalCase, or dot.notation name to snake_case."""+ # Replace any non-alphanumeric/underscore characters with underscores+ s = _re.sub(r'[^a-zA-Z0-9_]+', '_', name)+ # Insert underscore before uppercase letters that follow lowercase/digits+ s = _re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s)+ # Insert underscore between consecutive uppercase and uppercase+lowercase+ s = _re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', s)+ # Collapse multiple underscores and strip edges+ s = _re.sub(r'_+', '_', s)+ return s.lower().strip('_')+_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))_DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")diff --git a/src/eval.py b/src/eval.pyindex 8d35a83..715adf6 100644--- a/src/eval.py+++ b/src/eval.py@@ -57,12 +57,12 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thdecode_fn = _get_decode_fn(model, max_gen_len)# Warmup JIT- encoder_out = model.apply(+ encoder_out, enc_mask = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)dec_buffer = dec_buffer.at[0, 0].set(eos_id)- decode_fn(params, dec_buffer, encoder_out)+ decode_fn(params, dec_buffer, encoder_out, enc_mask)tokens_generated = []latencies = []@@ -73,10 +73,10 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thdec_buffer = dec_buffer.at[0, 0].set(eos_id)start = time.perf_counter()- encoder_out = model.apply(+ encoder_out, enc_mask = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)num_tokens = 0for i in range(max_gen_len - 1):@@ -88,7 +88,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thnum_tokens += 1dec_buffer = dec_buffer.at[0, i + 1].set(next_token)- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)elapsed = time.perf_counter() - starttokens_generated.append(num_tokens)@@ -171,15 +171,14 @@ def compute_wer(hypotheses, references):def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):"""Generate tool-call predictions and compute structured metrics."""import json- from .run import generate_batch- from .data import load_tool_calls+ from .run import generate_batch, normalize_tools, restore_tool_names+ from .data import load_tool_calls, to_snake_caseds = load_tool_calls("validation", max_samples=num_samples)- # Batch-generate all predictions at oncequeries = [ex["query"] for ex in ds]tools_list = [ex["tools"] for ex in ds]- all_preds = generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=max_gen_len, max_enc_len=max_enc_len)+ all_preds = generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=max_gen_len, max_enc_len=max_enc_len, normalize=True)total = 0exact_match = 0@@ -216,6 +215,35 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=except (json.JSONDecodeError, TypeError):ref_calls = []+ # Normalize ref tool names to snake_case for consistent comparison+ for rc in ref_calls:+ if isinstance(rc, dict) and "name" in rc:+ rc["name"] = to_snake_case(rc["name"])++ # Normalize pred tool names to snake_case too (restore_tool_names in+ # generate_batch maps back to originals, but we want snake_case comparison)+ try:+ pred_calls_raw = json.loads(pred_text)+ if isinstance(pred_calls_raw, list):+ for pc in pred_calls_raw:+ if isinstance(pc, dict) and "name" in pc:+ pc["name"] = to_snake_case(pc["name"])+ elif isinstance(pred_calls_raw, dict) and "name" in pred_calls_raw:+ pred_calls_raw["name"] = to_snake_case(pred_calls_raw["name"])+ pred_text = json.dumps(pred_calls_raw)+ except (json.JSONDecodeError, TypeError):+ pass++ # Also normalize tool definitions for param validation+ try:+ tool_defs_raw = json.loads(ex["tools"])+ for td in tool_defs_raw:+ if isinstance(td, dict) and "name" in td:+ td["name"] = to_snake_case(td["name"])+ tools_normalized = json.dumps(tool_defs_raw)+ except (json.JSONDecodeError, TypeError):+ tools_normalized = ex["tools"]+try:pred_calls = json.loads(pred_text)if not isinstance(pred_calls, list):@@ -260,7 +288,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=args_correct += 1try:- tool_defs = json.loads(ex["tools"])+ tool_defs = json.loads(tools_normalized)tool_param_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tool_defs if isinstance(t, dict) and "name" in t}except (json.JSONDecodeError, TypeError):tool_param_map = {}diff --git a/src/model.py b/src/model.pyindex bc0c468..593cddb 100644--- a/src/model.py+++ b/src/model.py@@ -145,69 +145,39 @@ class FeedForward(nn.Module):return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(h)-class MLPMixer(nn.Module):- """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""- num_slots: int- d_model: int- d_ff: int- num_layers: int- dtype: jnp.dtype = jnp.bfloat16- activation: str = "drelu"-- @nn.compact- def __call__(self, s, ffn_mask=None):- residual = s- s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)- s = s.transpose(0, 2, 1)- s = FeedForward(self.num_slots, self.d_ff, self.num_layers, self.dtype, self.activation, name="token_mix")(s, ffn_mask=ffn_mask)- s = s.transpose(0, 2, 1)- s = s + residual-- residual = s- s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)- s = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="channel_mix")(s, ffn_mask=ffn_mask)- s = s + residual-- return s---class MemoryMixerBlock(nn.Module):- """Block 4a: Pack (cross-attn S←X with RoPE on keys) → Mix (MLP-Mixer) → Local Update (MLP)."""+class EncoderBlock(nn.Module):+ """Pre-norm self-attention + pre-norm FFN with residual connections."""num_heads: intnum_kv_heads: intd_model: intd_ff: int- num_slots: intnum_layers: intdtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"dropout_rate: float = 0.0@nn.compact- def __call__(self, x, s, mask=None, rope=None, ffn_mask=None, deterministic=True):- s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)- x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)- pack_out = MultiHeadAttention(- self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype,- rope_keys_only=True, name="pack_attn"- )(s_norm, x_norm, mask=mask, rope=rope)- s = s + nn.Dropout(rate=self.dropout_rate)(pack_out, deterministic=deterministic)-- mix_out = MLPMixer(self.num_slots, self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="mixer")(- ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s), ffn_mask=ffn_mask+ def __call__(self, x, mask=None, rope=None, ffn_mask=None, deterministic=True):+ residual = x+ x = ZCRMSNorm(dtype=self.dtype)(x)+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(+ x, x, mask=mask, rope=rope)- s = s + nn.Dropout(rate=self.dropout_rate)(mix_out, deterministic=deterministic)+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic) + residualresidual = x- x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x, ffn_mask=ffn_mask)+ x = ZCRMSNorm(dtype=self.dtype)(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x, ffn_mask=ffn_mask)x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic) + residual- return x, s+ return x++class Encoder(nn.Module):+ """Self-attention encoder with stride-2 depthwise-separable downsampling.-class MemoryMixerEncoder(nn.Module):- """Encoder using MemoryMixer blocks. Output is the final memory slots S."""+ Returns (output, mask) tuple — variable-length output + adjusted padding mask.+ """config: TransformerConfig@nn.compact@@ -216,16 +186,6 @@ class MemoryMixerEncoder(nn.Module):dt = cfg.jax_dtypex = x.astype(dt)- s_base = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))- if mask is not None:- # mask: (B, 1, 1, T) -> (B, T)- pool_mask = mask[:, 0, 0, :x.shape[1]]- x_pool = jnp.sum(x * pool_mask[:, :, None], axis=1) / jnp.maximum(pool_mask.sum(axis=1, keepdims=True), 1.0)- else:- x_pool = jnp.mean(x, axis=1) # (B, d)- s_bias = nn.Dense(cfg.d_model, dtype=dt, use_bias=False, kernel_init=jinit.zeros, name="slot_init")(x_pool)- s = jnp.broadcast_to(s_base.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model)) + s_bias[:, None, :]-x = nn.Conv(features=cfg.d_model, kernel_size=(4,), strides=(2,),padding='SAME', feature_group_count=cfg.d_model,dtype=dt, use_bias=False, name="downsample_dw")(x)@@ -240,13 +200,13 @@ class MemoryMixerEncoder(nn.Module):mask = mask[..., :T_new]for i in range(cfg.num_encoder_layers):- x, s = nn.remat(MemoryMixerBlock, static_argnums=(6,))(+ x = nn.remat(EncoderBlock, static_argnums=(5,))(cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,- cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, cfg.dropout_rate, name=f"block_{i}"- )(x, s, mask, rope, ffn_mask, deterministic)+ cfg.total_layers, dt, cfg.activation, cfg.dropout_rate, name=f"block_{i}"+ )(x, mask, rope, ffn_mask, deterministic)- s = ZCRMSNorm(dtype=dt, name="final_norm")(s)- return s+ x = ZCRMSNorm(dtype=dt, name="final_norm")(x)+ return x, maskclass DecoderBlock(nn.Module):@@ -347,7 +307,7 @@ class EncoderDecoderTransformer(nn.Module):self.mel_proj = nn.Dense(self.config.d_model, dtype=self.config.jax_dtype,use_bias=True, kernel_init=default_init(), name="mel_proj")self.spec_augment = SpecAugment()- self.encoder = MemoryMixerEncoder(self.config)+ self.encoder = Encoder(self.config)self.decoder = Decoder(self.config)def _rope(self, seq_len):@@ -369,47 +329,39 @@ class EncoderDecoderTransformer(nn.Module):rope = self._rope(x.shape[1])return self.encoder(x, mask=src_mask, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)- def decode(self, tgt, encoder_out, self_mask=None, deterministic=True):- """Decode from encoder memory slots. No cross_mask needed (fixed-size slots)."""+ def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None, deterministic=True):+ """Decode from encoder output with cross_mask for variable-length encoder output."""x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])- x = self.decoder(x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope, deterministic=deterministic)+ x = self.decoder(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)logits = x.astype(jnp.float32) @ self.embedding.embedding.Treturn logitsdef __call__(self, src, tgt, src_mask=None, tgt_mask=None):- encoder_out = self.encode_text(src, src_mask=src_mask)- logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)+ logits = self.decode(tgt, encoder_out, self_mask=tgt_mask, cross_mask=enc_mask)return logits- def _run_decoder(self, encoder_out, tgt, tgt_mask=None, ffn_mask=None, deterministic=True):+ def _run_decoder(self, encoder_out, tgt, tgt_mask=None, cross_mask=None, ffn_mask=None, deterministic=True):"""Run decoder and return float32 hidden states."""x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])- x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)+ x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)return x.astype(jnp.float32)- def _slot_diversity(self, encoder_out):- s = encoder_out.astype(jnp.float32)- gram = jnp.matmul(s, s.transpose(0, 2, 1))- diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)- return (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]-def forward_masked(self, src, tgt, src_mask=None, tgt_mask=None, ffn_mask=None, deterministic=True):"""Single forward with per-batch-item FFN masking. Returns (logits, slot_div)."""- encoder_out = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask, deterministic=deterministic)+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask, ffn_mask=ffn_mask, deterministic=deterministic)logits = x_f32 @ self.embedding.embedding.T- slot_div = self._slot_diversity(encoder_out)- return logits, slot_div+ return logits, 0.0def forward_speech_masked(self, mel, tgt, src_mask=None, tgt_mask=None, ffn_mask=None, deterministic=True):"""Single speech forward with per-batch-item FFN masking. Returns (logits, slot_div)."""- encoder_out = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask, deterministic=deterministic)+ encoder_out, enc_mask = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask, ffn_mask=ffn_mask, deterministic=deterministic)logits = x_f32 @ self.embedding.embedding.T- slot_div = self._slot_diversity(encoder_out)- return logits, slot_div+ return logits, 0.0def _make_eval_ffn_mask(self, ff_width, B, dtype):"""Default prefix FFN mask for eval: first ff_width neurons active."""@@ -424,50 +376,48 @@ class EncoderDecoderTransformer(nn.Module):emb = self.embedding.embeddingB = src.shape[0]- encoder_out = self.encode_text(src, src_mask=src_mask)- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask)logits = x_f32 @ emb.T- slot_div = self._slot_diversity(encoder_out)mat_logits = []if mat_ff_widths is not None:for ff_w in mat_ff_widths:mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)- enc_m = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)- x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)+ enc_m, enc_m_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, cross_mask=enc_m_mask, ffn_mask=mask)mat_logits.append(x_m @ emb.T)- return logits, slot_div, mat_logits+ return logits, 0.0, mat_logitsdef forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None, deterministic=True):"""Eval-only: separate per-width speech forwards for reporting per-width PPL."""emb = self.embedding.embeddingB = mel.shape[0]- encoder_out = self.encode_speech(mel, src_mask=src_mask, deterministic=deterministic)- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)+ encoder_out, enc_mask = self.encode_speech(mel, src_mask=src_mask, deterministic=deterministic)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask)logits = x_f32 @ emb.T- slot_div = self._slot_diversity(encoder_out)mat_logits = []if mat_ff_widths is not None:for ff_w in mat_ff_widths:mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)- enc_m = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask, deterministic=deterministic)- x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)+ enc_m, enc_m_mask = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask, deterministic=deterministic)+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, cross_mask=enc_m_mask, ffn_mask=mask)mat_logits.append(x_m @ emb.T)- return logits, slot_div, mat_logits+ return logits, 0.0, mat_logitsdef init_all(self, src, tgt, mel):"""Dummy forward through both text and speech pathways to initialize all params."""src_mask = make_padding_mask(src, self.config.pad_token_id)tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)- text_out = self.encode_text(src, src_mask=src_mask)+ text_out, text_enc_mask = self.encode_text(src, src_mask=src_mask)mel_mask = make_mel_padding_mask(mel)- speech_out = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)- _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask)- _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask)+ speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)+ _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask, cross_mask=text_enc_mask)+ _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)return jnp.zeros(())diff --git a/src/run.py b/src/run.pyindex f3534c5..f6d46c6 100644--- a/src/run.py+++ b/src/run.py@@ -1,12 +1,14 @@import argparse+import json as _jsonimport pickle+import re as _reimport sysimport jaximport jax.numpy as jnpimport numpy as np-from .data import get_tokenizer, compute_mel_spectrogram, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_GEN_LEN+from .data import get_tokenizer, compute_mel_spectrogram, to_snake_case, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_GEN_LENfrom .model import (EncoderDecoderTransformer,TransformerConfig,@@ -16,6 +18,46 @@ from .model import ()+def normalize_tools(tools_json):+ """Normalize tool names in a tools JSON string to snake_case.++ Returns (normalized_json, name_map) where name_map maps+ snake_name -> original_name for reverse mapping.+ """+ try:+ tools = _json.loads(tools_json)+ except (_json.JSONDecodeError, TypeError):+ return tools_json, {}+ name_map = {}+ for t in tools:+ if isinstance(t, dict) and "name" in t:+ orig = t["name"]+ snake = to_snake_case(orig)+ name_map[snake] = orig+ t["name"] = snake+ return _json.dumps(tools, separators=(",", ":")), name_map+++def restore_tool_names(pred_text, name_map):+ """Replace snake_case tool names in model output back to original names."""+ if not name_map:+ return pred_text+ try:+ calls = _json.loads(pred_text)+ except (_json.JSONDecodeError, TypeError):+ # Fallback: string-level replacement, longest names first+ for snake, orig in sorted(name_map.items(), key=lambda x: len(x[0]), reverse=True):+ pred_text = pred_text.replace(snake, orig)+ return pred_text+ if isinstance(calls, list):+ for c in calls:+ if isinstance(c, dict) and "name" in c:+ c["name"] = name_map.get(c["name"], c["name"])+ elif isinstance(calls, dict) and "name" in calls:+ calls["name"] = name_map.get(calls["name"], calls["name"])+ return _json.dumps(calls, separators=(",", ":"))++_decode_fn_cache = {}@@ -30,10 +72,10 @@ def _get_decode_fn(model, max_gen_len):tgt_mask = make_causal_mask(max_gen_len)@jax.jit- def decode_step(params, dec_buffer, encoder_out):+ def decode_step(params, dec_buffer, encoder_out, cross_mask):return model.apply({"params": params}, dec_buffer, encoder_out,- self_mask=tgt_mask, method="decode",+ self_mask=tgt_mask, cross_mask=cross_mask, method="decode",)_decode_fn_cache[key] = decode_step@@ -61,12 +103,16 @@ def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LEreturn q_toks + [tools_sep_id] + t_toks-def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, seed=0, stream=True, task_token_id=None):+def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, seed=0, stream=True, task_token_id=None, normalize=True):"""Generate tool-call output.Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.Decoder: prefilled with [EOS, <tool_call>], then greedy decode."""+ name_map = {}+ if normalize:+ tools, name_map = normalize_tools(tools)+enc_tokens = _build_encoder_input(tokenizer, query, tools, max_enc_len)enc_input = jnp.array([enc_tokens])@@ -75,7 +121,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAtool_call_id = task_token_id if task_token_id is not None else tokenizer.tool_call_token_idsrc_mask = make_padding_mask(enc_input, pad_id)- encoder_out = model.apply(+ encoder_out, enc_mask = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")@@ -94,7 +140,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAsys.stdout.write(f"\n")sys.stdout.flush()- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)for i in range(prefix_len - 1, max_gen_len - 1):next_logits = logits[0, i]@@ -110,15 +156,18 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAsys.stdout.write(tokenizer.decode([next_token]))sys.stdout.flush()- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)if stream:sys.stdout.write("\n")- return tokenizer.decode(generated_tokens)+ result = tokenizer.decode(generated_tokens)+ if normalize and name_map:+ result = restore_tool_names(result, name_map)+ return result-def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):+def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, normalize=True):"""Batch-generate tool-call outputs for multiple examples at once.Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.@@ -126,6 +175,15 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEReturns a list of decoded strings, one per example."""+ name_maps = []+ if normalize:+ normed_tools = []+ for t in tools_list:+ nt, nm = normalize_tools(t)+ normed_tools.append(nt)+ name_maps.append(nm)+ tools_list = normed_tools+B = len(queries)pad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_id@@ -142,7 +200,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEenc_input = jnp.array(enc_input)src_mask = make_padding_mask(enc_input, pad_id)- encoder_out = model.apply(+ encoder_out, enc_mask = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")@@ -159,7 +217,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEfinished = [False] * Bgen_tokens = [[] for _ in range(B)]- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)for pos in range(prefix_len - 1, max_gen_len - 1):for i in range(B):@@ -175,9 +233,12 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEif all(finished):break- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)- return [tokenizer.decode(toks) for toks in gen_tokens]+ results = [tokenizer.decode(toks) for toks in gen_tokens]+ if normalize and name_maps:+ results = [restore_tool_names(r, nm) for r, nm in zip(results, name_maps)]+ return resultsdef load_audio(path, target_sr=16000):@@ -209,7 +270,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="tool_call_id = tokenizer.tool_call_token_idsrc_mask = make_mel_padding_mask(mel_input)- encoder_out = model.apply(+ encoder_out, enc_mask = model.apply({"params": params}, mel_input, src_mask=src_mask, deterministic=True, method="encode_speech")@@ -229,7 +290,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="sys.stdout.write("\n")sys.stdout.flush()- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)for i in range(prefix_len - 1, max_gen_len - 1):next_logits = logits[0, i]@@ -245,7 +306,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="sys.stdout.write(tokenizer.decode([next_token]))sys.stdout.flush()- logits = decode_fn(params, dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)if stream:sys.stdout.write("\n")diff --git a/src/train.py b/src/train.pyindex bcacfc6..06ba2b5 100644--- a/src/train.py+++ b/src/train.py@@ -244,8 +244,7 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rnoptax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask) / jnp.maximum(jnp.sum(mask), 1.0)z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)- div_loss = 1e-4 * slot_div- return ce_loss + z_loss + div_loss+ return ce_loss + z_loss@@ -351,17 +350,16 @@ def _estimate_mat_params(config, matryoshka_factor):n_enc = config.num_encoder_layersn_dec = config.num_decoder_layerskv_dim = config.num_kv_heads * (d // config.num_heads)- m = config.num_memory_slotsd_ff = config.d_ff // matryoshka_factoremb = v * dattn = d * d + d * kv_dim * 2 + d * dffn = d * d_ff * 3- mixer_token = m * d_ff * 3- mixer_channel = d * d_ff * 3- enc_block = attn + ffn + mixer_token + mixer_channel- dec_block = attn * 2 + ffn- total = emb + n_enc * enc_block + n_dec * dec_block+ # Encoder: self-attn + FFN per block, plus downsample conv params+ downsample = d * 4 + d * d # depthwise conv (kernel=4) + pointwise Dense+ enc_block = attn + ffn+ dec_block = attn * 2 + ffn # self-attn + cross-attn + FFN+ total = emb + downsample + n_enc * enc_block + n_dec * dec_blockreturn int(total)@@ -469,7 +467,6 @@ def train(args):print(f" d_model {config.d_model:>12}")print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")- print(f" Memory slots {config.num_memory_slots:>12}")print(f" Activation {config.activation:>12}")print(f" Dtype {config.dtype:>12}")print(f" ─────────────────────────────────────")