Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Refactor

HenryNdubuaku5778f470c934
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.py
index 6fcf697..58d6696 100644
--- a/scripts/synthesize_tools_data.py
+++ b/scripts/synthesize_tools_data.py
@@ -1,13 +1,19 @@
#!/usr/bin/env python3
-"""Synthesize tool-calling data from PleIAs/SYNTH queries via OpenRouter.
+"""Synthesize tool-calling data from multiple query sources via OpenRouter.
-Downloads the first parquet shard of PleIAs/SYNTH (~1M diverse multilingual
-queries), then prompts an LLM to generate (tools, answer) pairs for each query.
+Query sources:
+ 1. PleIAs/SYNTH parquet shards (~154k queries per shard, 7 shards)
+ 2. GCS raw_data — queries extracted from the existing unified tool-calling dataset
+
+Each query is sent to a randomly-selected frontier model to generate
+(tools, answer) pairs, then validated and checkpointed as JSONL.
Usage:
- python scripts/synthesize_tools_data.py
- python scripts/synthesize_tools_data.py --max-samples 10000 --model google/gemini-2.5-flash
- python scripts/synthesize_tools_data.py --resume # continue from last checkpoint
+ python scripts/synthesize_tools_data.py # all sources
+ python scripts/synthesize_tools_data.py --model google/gemini-3.1-flash-lite-preview
+ python scripts/synthesize_tools_data.py --max-shards 1 --max-samples 100
+ python scripts/synthesize_tools_data.py --no-gcs # skip GCS queries
+ python scripts/synthesize_tools_data.py --resume # continue from checkpoint
"""
import argparse
@@ -28,10 +34,17 @@ logger = logging.getLogger(__name__)
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
_OUTPUT_DIR = _PROJECT_ROOT / "data" / "synth_tool_calls"
_CHECKPOINT_PATH = _OUTPUT_DIR / "checkpoint.jsonl"
-_PARQUET_CACHE = _PROJECT_ROOT / ".data_cache" / "synth_shard0.parquet"
+_PARQUET_CACHE_DIR = _PROJECT_ROOT / ".data_cache"
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
-DEFAULT_MODEL = "google/gemini-2.5-flash"
+
+MODEL_POOL = [
+ "google/gemini-3.1-flash-lite-preview",
+ # "anthropic/claude-opus-4.6",
+ # "x-ai/grok-4.1-fast",
+ # "openai/gpt-5.3-chat",
+]
+DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"
SYSTEM_PROMPT = """\
You are a synthetic data generator for training a tool-calling AI model.
@@ -39,15 +52,27 @@ You are a synthetic data generator for training a tool-calling AI model.
Given a user query, you must invent a realistic set of tools (APIs/functions) that \
could help answer it, then produce the correct tool call(s).
+Vary the domain widely — cover areas like software development, data analysis, \
+e-commerce, finance, healthcare, travel, IoT, social media, education, DevOps, legal, \
+HR, and more. Do NOT default to simple personal-assistant tasks.
+
Rules:
-1. Invent 3-8 tools as JSON. Each tool has: name, description, parameters (dict of param_name -> {type, description}). \
-Only ONE or TWO tools should be relevant; the rest are distractors.
-2. Tool names should be snake_case, realistic API-style (e.g. search_web, get_weather, translate_text).
-3. Produce the answer as a JSON list of tool calls: [{"name": "...", "arguments": {...}}].
-4. If the query genuinely needs no tool, return an empty list [].
-5. Argument values MUST be grounded in the query — do not hallucinate values.
-6. Respond in the SAME LANGUAGE as the query.
-7. Return ONLY valid JSON, no markdown fences, no commentary.
+1. Invent 1-5 tools as JSON. Each tool MUST have: name, description, parameters \
+(dict of param_name -> {type, description, required}). Use realistic parameter types \
+including string, number, boolean, array, object. Include nested objects or arrays \
+where appropriate.
+2. Only ONE or TWO tools should be relevant to the query; the rest are plausible \
+distractors from the SAME domain.
+3. Tool names should be snake_case, realistic API-style, and DIVERSE — avoid repeating \
+common names like search_web or get_weather across samples. Think of specific, \
+domain-appropriate APIs (e.g. query_elasticsearch, run_sql_migration, price_option_chain, \
+schedule_k8s_cronjob, fetch_patient_labs).
+4. Produce the answer as a JSON list of tool calls: [{"name": "...", "arguments": {...}}].
+5. For complex queries, use MULTIPLE tool calls (chained or parallel) where realistic.
+6. If the query genuinely needs no tool, return an empty answer list [].
+7. Argument values MUST be grounded in the query — do not hallucinate values.
+8. Respond in the SAME LANGUAGE as the query.
+9. Return ONLY valid JSON, no markdown fences, no commentary.
Output format (strict JSON object):
{"tools": [...], "answer": [...]}"""
@@ -55,29 +80,42 @@ Output format (strict JSON object):
USER_TEMPLATE = "Query: {query}"
-def download_first_shard(token):
- """Download the first parquet shard of PleIAs/SYNTH."""
- if _PARQUET_CACHE.exists():
- logger.info(f"Using cached shard: {_PARQUET_CACHE}")
- return _PARQUET_CACHE
+def _discover_shard_urls(token, max_shards=None):
+ """Discover available parquet shard URLs from the HF API."""
+ api_url = "https://huggingface.co/api/datasets/PleIAs/SYNTH/parquet"
+ headers = {}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ resp = requests.get(api_url, headers=headers, timeout=30)
+ resp.raise_for_status()
+ data = resp.json() # {"default": {"train": [url, ...]}}
+ urls = data["default"]["train"]
+ if max_shards:
+ urls = urls[:max_shards]
+ logger.info(f"Discovered {len(urls)} parquet shards")
+ return urls
+
+
+def download_shard(url, shard_idx, token):
+ """Download a single parquet shard, returning the local path."""
+ cache_path = _PARQUET_CACHE_DIR / f"synth_shard{shard_idx}.parquet"
+ if cache_path.exists():
+ logger.info(f"Using cached shard {shard_idx}: {cache_path}")
+ return cache_path
- _PARQUET_CACHE.parent.mkdir(parents=True, exist_ok=True)
+ _PARQUET_CACHE_DIR.mkdir(parents=True, exist_ok=True)
- url = (
- "https://huggingface.co/datasets/PleIAs/SYNTH"
- "/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet"
- )
headers = {}
if token:
headers["Authorization"] = f"Bearer {token}"
- logger.info(f"Downloading first shard from PleIAs/SYNTH...")
+ logger.info(f"Downloading shard {shard_idx}...")
resp = requests.get(url, headers=headers, stream=True, timeout=120)
resp.raise_for_status()
total = int(resp.headers.get("content-length", 0))
downloaded = 0
- with open(_PARQUET_CACHE, "wb") as f:
+ with open(cache_path, "wb") as f:
for chunk in resp.iter_content(chunk_size=8 * 1024 * 1024):
f.write(chunk)
downloaded += len(chunk)
@@ -85,12 +123,12 @@ def download_first_shard(token):
pct = downloaded / total * 100
print(f"\r {downloaded / 1e6:.0f}/{total / 1e6:.0f} MB ({pct:.0f}%)", end="", flush=True)
print()
- logger.info(f"Saved to {_PARQUET_CACHE}")
- return _PARQUET_CACHE
+ logger.info(f"Saved shard {shard_idx} to {cache_path}")
+ return cache_path
-def load_queries(path, max_samples=None):
- """Load queries from parquet, return list of (synth_id, language, query)."""
+def load_queries_from_parquet(path, max_samples=None):
+ """Load queries from parquet, return list of (sample_id, language, query)."""
import pyarrow.parquet as pq
logger.info(f"Reading parquet: {path}")
@@ -104,7 +142,6 @@ def load_queries(path, max_samples=None):
samples = list(zip(ids, langs, queries))
- # Shuffle for diversity (don't process in order)
rng = random.Random(42)
rng.shuffle(samples)
@@ -115,6 +152,48 @@ def load_queries(path, max_samples=None):
return samples
+def load_queries_from_gcs(max_samples=None):
+ """Download raw_data from GCS and extract queries for re-synthesis.
+
+ Returns list of (sample_id, language, query) tuples, where sample_id
+ is prefixed with 'gcs-' to avoid collisions with PleIAs IDs.
+ """
+ sys.path.insert(0, str(_PROJECT_ROOT))
+ from src.gcs import download_raw_data
+
+ local_dir = _PARQUET_CACHE_DIR / "gcs_raw_data"
+ if not local_dir.exists():
+ logger.info("Downloading raw_data from GCS...")
+ ok = download_raw_data(str(local_dir))
+ if not ok:
+ logger.warning("No raw_data found in GCS, skipping")
+ return []
+ else:
+ logger.info(f"Using cached GCS raw_data: {local_dir}")
+
+ from datasets import load_from_disk
+ ds = load_from_disk(str(local_dir))
+ logger.info(f"Loaded {len(ds):,} rows from GCS raw_data")
+
+ samples = []
+ for i in range(len(ds)):
+ ex = ds[i]
+ query = ex.get("query", "").strip()
+ if not query:
+ continue
+ sample_id = f"gcs-{i}"
+ samples.append((sample_id, "en", query))
+
+ rng = random.Random(42)
+ rng.shuffle(samples)
+
+ if max_samples:
+ samples = samples[:max_samples]
+
+ logger.info(f" Selected {len(samples):,} GCS queries")
+ return samples
+
+
def load_checkpoint():
"""Load completed synth_ids from checkpoint file."""
done = set()
@@ -143,7 +222,7 @@ def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):
{"role": "user", "content": USER_TEMPLATE.format(query=query)},
],
"temperature": 0.7,
- "max_tokens": 2048,
+ "max_tokens": 4096,
}
for attempt in range(max_retries):
@@ -153,12 +232,22 @@ def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):
)
if resp.status_code == 429:
wait = min(2 ** attempt * 2, 30)
+ logger.warning(f"[{model}] 429 rate-limited, retrying in {wait}s")
time.sleep(wait)
continue
- resp.raise_for_status()
+ if not resp.ok:
+ logger.warning(f"[{model}] HTTP {resp.status_code}: {resp.text[:200]}")
+ if attempt == max_retries - 1:
+ return None
+ time.sleep(2 ** attempt)
+ continue
content = resp.json()["choices"][0]["message"]["content"]
- return _parse_response(content)
+ parsed = _parse_response(content)
+ if parsed is None:
+ logger.warning(f"[{model}] Failed to parse response: {content[:200]}")
+ return parsed
except (requests.RequestException, KeyError, IndexError) as e:
+ logger.warning(f"[{model}] Request error (attempt {attempt+1}/{max_retries}): {e}")
if attempt == max_retries - 1:
return None
time.sleep(2 ** attempt)
@@ -168,7 +257,6 @@ def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):
def _parse_response(content):
"""Parse LLM response into (tools, answer) or None."""
content = content.strip()
- # Strip markdown fences if present
if content.startswith("```"):
content = content.split("\n", 1)[-1]
if content.endswith("```"):
@@ -191,12 +279,10 @@ def _parse_response(content):
if not isinstance(answer, list):
return None
- # Validate tool structure
for t in tools:
if not isinstance(t, dict) or "name" not in t:
return None
- # Validate answer structure
for a in answer:
if not isinstance(a, dict) or "name" not in a:
return None
@@ -206,15 +292,13 @@ def _parse_response(content):
return tools, answer
-def _validate_and_format(synth_id, language, query, tools, answer):
+def _validate_and_format(synth_id, language, query, tools, answer, model, source="synth-pleias"):
"""Validate the generated data and return a formatted row or None."""
- # Check that every called tool exists in the tool list
tool_names = {t["name"] for t in tools}
for a in answer:
if a["name"] not in tool_names:
return None
- # Check argument keys are valid
tool_params = {}
for t in tools:
params = t.get("parameters") or {}
@@ -234,64 +318,67 @@ def _validate_and_format(synth_id, language, query, tools, answer):
"query": query,
"answers": json.dumps(answer),
"tools": json.dumps(tools),
- "source": "synth-pleias",
+ "source": source,
+ "model": model,
}
-def process_sample(sample, api_key, model):
+def _pick_model(models, rng):
+ """Randomly select a model from the pool."""
+ return rng.choice(models)
+
+
+def process_sample(sample, api_key, models, rng, source_tag="synth-pleias"):
"""Process a single sample: call LLM, validate, return row or None."""
synth_id, language, query = sample
+ model = _pick_model(models, rng)
result = call_openrouter(query, api_key, model=model)
if result is None:
return None
tools, answer = result
- return _validate_and_format(synth_id, language, query, tools, answer)
-
-
-def main():
- parser = argparse.ArgumentParser(description="Synthesize tool-calling data from PleIAs/SYNTH")
- parser.add_argument("--max-samples", type=int, default=None,
- help="Max queries to process (default: all in shard)")
- parser.add_argument("--model", type=str, default=DEFAULT_MODEL,
- help=f"OpenRouter model (default: {DEFAULT_MODEL})")
- parser.add_argument("--workers", type=int, default=32,
- help="Concurrent API requests (default: 32)")
- parser.add_argument("--resume", action="store_true",
- help="Resume from checkpoint")
- parser.add_argument("--batch-size", type=int, default=500,
- help="Samples between progress logs (default: 500)")
- args = parser.parse_args()
+ row = _validate_and_format(synth_id, language, query, tools, answer, model, source_tag)
+ if row is None:
+ logger.warning(f"[{model}] Validation failed for synth_id={synth_id}")
+ return row
- api_key = os.environ.get("OPENROUTER_API_KEY")
- if not api_key:
- print("Error: set OPENROUTER_API_KEY environment variable")
- sys.exit(1)
- hf_token = os.environ.get("HF_TOKEN")
- parquet_path = download_first_shard(hf_token)
- samples = load_queries(parquet_path, max_samples=args.max_samples)
-
- _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
- done_ids = load_checkpoint() if args.resume else set()
- if not args.resume and _CHECKPOINT_PATH.exists():
- _CHECKPOINT_PATH.unlink()
+def _process_batch(samples, label, batch_idx, api_key, models, args, tqdm, source_tag):
+ """Process a batch of samples, append to checkpoint. Returns (generated, failed)."""
+ logger.info(f"\n{'=' * 60}")
+ logger.info(f"Batch: {label}")
+ logger.info(f"{'=' * 60}")
+ # Always load checkpoint to skip already-completed IDs (even across batches)
+ done_ids = load_checkpoint()
pending = [s for s in samples if s[0] not in done_ids]
+ if not pending:
+ logger.info("All samples already done, skipping")
+ return 0, 0
logger.info(f"Processing {len(pending):,} samples ({len(done_ids):,} already done)")
generated = 0
failed = 0
out_file = open(_CHECKPOINT_PATH, "a")
+ rng = random.Random(42 + batch_idx)
try:
with ThreadPoolExecutor(max_workers=args.workers) as executor:
futures = {}
for sample in pending:
- fut = executor.submit(process_sample, sample, api_key, args.model)
- futures[fut] = sample[0] # synth_id
-
- for i, fut in enumerate(as_completed(futures), 1):
+ fut = executor.submit(
+ process_sample, sample, api_key, models, rng, source_tag,
+ )
+ futures[fut] = sample[0]
+
+ pbar = tqdm(
+ as_completed(futures),
+ total=len(futures),
+ desc=label,
+ unit="sample",
+ dynamic_ncols=True,
+ ) if tqdm else as_completed(futures)
+
+ for i, fut in enumerate(pbar if tqdm else as_completed(futures), 1):
try:
row = fut.result()
except Exception:
@@ -304,31 +391,120 @@ def main():
else:
failed += 1
- if i % args.batch_size == 0:
- out_file.flush()
- total = generated + failed
- rate = generated / max(total, 1) * 100
- logger.info(
- f" [{total:,}/{len(pending):,}] "
- f"generated={generated:,} failed={failed:,} "
- f"({rate:.0f}% success)"
- )
+ if tqdm:
+ rate = generated / max(generated + failed, 1) * 100
+ pbar.set_postfix(ok=generated, fail=failed, rate=f"{rate:.0f}%")
+ if i % args.batch_size == 0:
+ out_file.flush()
+ else:
+ if i % args.batch_size == 0:
+ out_file.flush()
+ total = generated + failed
+ rate = generated / max(total, 1) * 100
+ logger.info(
+ f" [{total:,}/{len(pending):,}] "
+ f"generated={generated:,} failed={failed:,} "
+ f"({rate:.0f}% success)"
+ )
+
+ if tqdm:
+ pbar.close()
except KeyboardInterrupt:
logger.info("\nInterrupted — progress saved to checkpoint")
finally:
out_file.flush()
out_file.close()
- logger.info(f"\nDone: {generated:,} generated, {failed:,} failed")
- logger.info(f"Output: {_CHECKPOINT_PATH}")
+ logger.info(f"{label}: {generated:,} generated, {failed:,} failed")
+ return generated, failed
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Synthesize tool-calling data from PleIAs/SYNTH")
+ parser.add_argument("--max-samples", type=int, default=None,
+ help="Max queries to process (default: all in shard)")
+ parser.add_argument("--models", type=str, nargs="+", default=None,
+ help=f"OpenRouter models to rotate (default: built-in pool)")
+ parser.add_argument("--model", type=str, default=None,
+ help="Use a single model instead of the pool")
+ parser.add_argument("--workers", type=int, default=32,
+ help="Concurrent API requests (default: 32)")
+ parser.add_argument("--max-shards", type=int, default=None,
+ help="Max parquet shards to process (default: all)")
+ parser.add_argument("--no-gcs", action="store_true",
+ help="Skip GCS raw_data queries")
+ parser.add_argument("--resume", action="store_true",
+ help="Resume from checkpoint")
+ parser.add_argument("--batch-size", type=int, default=500,
+ help="Samples between progress logs (default: 500)")
+ args = parser.parse_args()
+
+ api_key = os.environ.get("OPENROUTER_API_KEY")
+ if not api_key:
+ print("Error: set OPENROUTER_API_KEY environment variable")
+ sys.exit(1)
+
+ if args.model:
+ models = [args.model]
+ elif args.models:
+ models = args.models
+ else:
+ models = list(MODEL_POOL)
+ logger.info(f"Model pool ({len(models)}): {', '.join(models)}")
+
+ _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+ # Clear checkpoint for fresh runs; --resume keeps existing data
+ if not args.resume and _CHECKPOINT_PATH.exists():
+ _CHECKPOINT_PATH.unlink()
+ logger.info("Cleared previous checkpoint (use --resume to keep)")
+
+ try:
+ from tqdm import tqdm
+ except ImportError:
+ logger.warning("tqdm not installed, falling back to basic progress logging")
+ tqdm = None
+
+ total_generated = 0
+ total_failed = 0
+ batch_idx = 0
+
+ # ── Source 1: GCS raw_data queries ──
+ if not args.no_gcs:
+ gcs_samples = load_queries_from_gcs(max_samples=args.max_samples)
+ if gcs_samples:
+ gen, fail = _process_batch(
+ gcs_samples, "GCS raw_data", batch_idx, api_key, models,
+ args, tqdm, source_tag="synth-gcs",
+ )
+ total_generated += gen
+ total_failed += fail
+ batch_idx += 1
+ if gen > 0:
+ _export_dataset()
+
+ # ── Source 2: PleIAs/SYNTH parquet shards ──
+ hf_token = os.environ.get("HF_TOKEN")
+ shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)
+
+ for shard_idx, shard_url in enumerate(shard_urls):
+ parquet_path = download_shard(shard_url, shard_idx, hf_token)
+ samples = load_queries_from_parquet(parquet_path, max_samples=args.max_samples)
+ gen, fail = _process_batch(
+ samples, f"PleIAs shard {shard_idx}", batch_idx, api_key, models,
+ args, tqdm, source_tag="synth-pleias",
+ )
+ total_generated += gen
+ total_failed += fail
+ batch_idx += 1
+ if gen > 0:
+ _export_dataset()
- # Convert checkpoint to HF dataset format
- if generated > 0:
- _export_dataset()
+ logger.info(f"\nAll done: {total_generated:,} generated, {total_failed:,} failed")
def _export_dataset():
- """Convert checkpoint JSONL to the unified format expected by build_dataset.py."""
+ """Convert checkpoint JSONL to the unified format and push to GCS."""
from datasets import Dataset
rows = []
@@ -341,6 +517,7 @@ def _export_dataset():
"answers": row["answers"],
"tools": row["tools"],
"source": row.get("source", "synth-pleias"),
+ "model": row.get("model", "unknown"),
})
except (json.JSONDecodeError, KeyError):
continue
@@ -352,9 +529,29 @@ def _export_dataset():
out_path = _OUTPUT_DIR / "dataset"
ds.save_to_disk(str(out_path))
logger.info(f"Exported {len(rows):,} examples to {out_path}/")
- logger.info(
- "To include in training, add to tools_data.py or merge with unified dataset."
- )
+
+ # Push to GCS
+ try:
+ sys.path.insert(0, str(_PROJECT_ROOT))
+ from src.gcs import upload_directory
+ gcs_prefix = "synth_tool_calls"
+ upload_directory(str(out_path), gcs_prefix)
+ except Exception as e:
+ logger.warning(f"GCS upload failed: {e}")
+
+ for i, row in enumerate(rows[:10]):
+ try:
+ tools = json.loads(row["tools"])
+ answers = json.loads(row["answers"])
+ except (json.JSONDecodeError, TypeError):
+ tools, answers = [], []
+ called_names = [a["name"] for a in answers if isinstance(a, dict)]
+ print(f"\n{'─' * 60}")
+ print(f"[{i+1}] Query: {row['query'][:200]}")
+ print(f" Available tools ({len(tools)}): {', '.join(t.get('name','?') for t in tools)}")
+ print(f" Called tools ({len(called_names)}): {', '.join(called_names) or '(none)'}")
+ print(f" Answers: {row['answers'][:200]}")
+ print(f"{'─' * 60}")
if __name__ == "__main__":
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
index 2e8dce9..8a23e68 100755
--- a/scripts/tools_data.py
+++ b/scripts/tools_data.py
@@ -50,7 +50,7 @@ DATASETS = {
"toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),
"toucan-sft": ("Agent-Ark/Toucan-1.5M", "SFT"),
"toolace": "Team-ACE/ToolACE",
- # "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),
+ "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),
}
@@ -675,6 +675,39 @@ def _trim_tools(dataset, max_tools=5, seed=42):
return Dataset.from_list(rows)
+def _normalize_tool_names(dataset):
+ """Normalize all tool names to snake_case in both tools and answers fields."""
+ from src.data import to_snake_case
+
+ rows = []
+ for i in range(len(dataset)):
+ ex = dataset[i]
+ try:
+ tools = json.loads(ex["tools"])
+ except (json.JSONDecodeError, TypeError):
+ rows.append(ex)
+ continue
+ try:
+ answers = json.loads(ex["answers"])
+ except (json.JSONDecodeError, TypeError):
+ answers = []
+
+ for t in tools:
+ if isinstance(t, dict) and "name" in t:
+ t["name"] = to_snake_case(t["name"])
+ for a in answers:
+ if isinstance(a, dict) and "name" in a:
+ a["name"] = to_snake_case(a["name"])
+
+ rows.append({
+ "query": ex["query"],
+ "answers": json.dumps(answers),
+ "tools": json.dumps(tools),
+ "source": ex["source"],
+ })
+ return Dataset.from_list(rows)
+
+
def _augment_irrelevance(dataset, ratio=0.1, seed=42):
"""Create irrelevance samples by swapping tools with unrelated queries.
@@ -803,6 +836,9 @@ def load_and_combine():
combined = _trim_tools(combined, max_tools=5, seed=42)
print(f" {before_trim} -> {len(combined)} after tool trimming")
+ combined = _normalize_tool_names(combined)
+ print(f" Normalized tool names to snake_case")
+
combined = _augment_irrelevance(combined, ratio=0.1, seed=42)
empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])
diff --git a/src/cli.py b/src/cli.py
index b3313bc..f17a7a3 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -19,11 +19,11 @@ def main():
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=512)
+ p.add_argument("--d-model", type=int, default=768)
p.add_argument("--num-heads", type=int, default=16)
p.add_argument("--num-kv-heads", type=int, default=8)
- p.add_argument("--num-layers", type=int, default=6)
- p.add_argument("--num-dec-layers", type=int, default=6)
+ p.add_argument("--num-layers", type=int, default=8)
+ p.add_argument("--num-dec-layers", type=int, default=8)
p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
p.add_argument("--max-samples", type=int, default=None)
@@ -42,8 +42,9 @@ def main():
help="Fraction of epoch to train before starting gradual pruning (default: 0.33)")
p.add_argument("--prune-end-frac", type=float, default=0.67,
help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")
- p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
- p.add_argument("--num-memory-slots", type=int, default=128)
+ p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])
+ p.add_argument("--num-memory-slots", type=int, default=128,
+ help="(DEPRECATED — ignored, kept for checkpoint compat)")
p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],
help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")
p.add_argument("--dropout", type=float, default=0.1,
diff --git a/src/data.py b/src/data.py
index fffca9f..8934139 100644
--- a/src/data.py
+++ b/src/data.py
@@ -19,6 +19,18 @@ import sentencepiece as spm
import re as _re
+def to_snake_case(name):
+ """Convert camelCase, PascalCase, or dot.notation name to snake_case."""
+ # Replace any non-alphanumeric/underscore characters with underscores
+ s = _re.sub(r'[^a-zA-Z0-9_]+', '_', name)
+ # Insert underscore before uppercase letters that follow lowercase/digits
+ s = _re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s)
+ # Insert underscore between consecutive uppercase and uppercase+lowercase
+ s = _re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', s)
+ # Collapse multiple underscores and strip edges
+ s = _re.sub(r'_+', '_', s)
+ return s.lower().strip('_')
+
_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
_DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")
TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")
diff --git a/src/eval.py b/src/eval.py
index 8d35a83..715adf6 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -57,12 +57,12 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
decode_fn = _get_decode_fn(model, max_gen_len)
# Warmup JIT
- encoder_out = model.apply(
+ encoder_out, enc_mask = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
dec_buffer = dec_buffer.at[0, 0].set(eos_id)
- decode_fn(params, dec_buffer, encoder_out)
+ decode_fn(params, dec_buffer, encoder_out, enc_mask)
tokens_generated = []
latencies = []
@@ -73,10 +73,10 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
dec_buffer = dec_buffer.at[0, 0].set(eos_id)
start = time.perf_counter()
- encoder_out = model.apply(
+ encoder_out, enc_mask = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
num_tokens = 0
for i in range(max_gen_len - 1):
@@ -88,7 +88,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
num_tokens += 1
dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
elapsed = time.perf_counter() - start
tokens_generated.append(num_tokens)
@@ -171,15 +171,14 @@ def compute_wer(hypotheses, references):
def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):
"""Generate tool-call predictions and compute structured metrics."""
import json
- from .run import generate_batch
- from .data import load_tool_calls
+ from .run import generate_batch, normalize_tools, restore_tool_names
+ from .data import load_tool_calls, to_snake_case
ds = load_tool_calls("validation", max_samples=num_samples)
- # Batch-generate all predictions at once
queries = [ex["query"] for ex in ds]
tools_list = [ex["tools"] for ex in ds]
- all_preds = generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=max_gen_len, max_enc_len=max_enc_len)
+ all_preds = generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=max_gen_len, max_enc_len=max_enc_len, normalize=True)
total = 0
exact_match = 0
@@ -216,6 +215,35 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
except (json.JSONDecodeError, TypeError):
ref_calls = []
+ # Normalize ref tool names to snake_case for consistent comparison
+ for rc in ref_calls:
+ if isinstance(rc, dict) and "name" in rc:
+ rc["name"] = to_snake_case(rc["name"])
+
+ # Normalize pred tool names to snake_case too (restore_tool_names in
+ # generate_batch maps back to originals, but we want snake_case comparison)
+ try:
+ pred_calls_raw = json.loads(pred_text)
+ if isinstance(pred_calls_raw, list):
+ for pc in pred_calls_raw:
+ if isinstance(pc, dict) and "name" in pc:
+ pc["name"] = to_snake_case(pc["name"])
+ elif isinstance(pred_calls_raw, dict) and "name" in pred_calls_raw:
+ pred_calls_raw["name"] = to_snake_case(pred_calls_raw["name"])
+ pred_text = json.dumps(pred_calls_raw)
+ except (json.JSONDecodeError, TypeError):
+ pass
+
+ # Also normalize tool definitions for param validation
+ try:
+ tool_defs_raw = json.loads(ex["tools"])
+ for td in tool_defs_raw:
+ if isinstance(td, dict) and "name" in td:
+ td["name"] = to_snake_case(td["name"])
+ tools_normalized = json.dumps(tool_defs_raw)
+ except (json.JSONDecodeError, TypeError):
+ tools_normalized = ex["tools"]
+
try:
pred_calls = json.loads(pred_text)
if not isinstance(pred_calls, list):
@@ -260,7 +288,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
args_correct += 1
try:
- tool_defs = json.loads(ex["tools"])
+ tool_defs = json.loads(tools_normalized)
tool_param_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tool_defs if isinstance(t, dict) and "name" in t}
except (json.JSONDecodeError, TypeError):
tool_param_map = {}
diff --git a/src/model.py b/src/model.py
index bc0c468..593cddb 100644
--- a/src/model.py
+++ b/src/model.py
@@ -145,69 +145,39 @@ class FeedForward(nn.Module):
return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(h)
-class MLPMixer(nn.Module):
- """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""
- num_slots: int
- d_model: int
- d_ff: int
- num_layers: int
- dtype: jnp.dtype = jnp.bfloat16
- activation: str = "drelu"
-
- @nn.compact
- def __call__(self, s, ffn_mask=None):
- residual = s
- s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)
- s = s.transpose(0, 2, 1)
- s = FeedForward(self.num_slots, self.d_ff, self.num_layers, self.dtype, self.activation, name="token_mix")(s, ffn_mask=ffn_mask)
- s = s.transpose(0, 2, 1)
- s = s + residual
-
- residual = s
- s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)
- s = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="channel_mix")(s, ffn_mask=ffn_mask)
- s = s + residual
-
- return s
-
-
-class MemoryMixerBlock(nn.Module):
- """Block 4a: Pack (cross-attn S←X with RoPE on keys) → Mix (MLP-Mixer) → Local Update (MLP)."""
+class EncoderBlock(nn.Module):
+ """Pre-norm self-attention + pre-norm FFN with residual connections."""
num_heads: int
num_kv_heads: int
d_model: int
d_ff: int
- num_slots: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
dropout_rate: float = 0.0
@nn.compact
- def __call__(self, x, s, mask=None, rope=None, ffn_mask=None, deterministic=True):
- s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)
- x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)
- pack_out = MultiHeadAttention(
- self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype,
- rope_keys_only=True, name="pack_attn"
- )(s_norm, x_norm, mask=mask, rope=rope)
- s = s + nn.Dropout(rate=self.dropout_rate)(pack_out, deterministic=deterministic)
-
- mix_out = MLPMixer(self.num_slots, self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="mixer")(
- ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s), ffn_mask=ffn_mask
+ def __call__(self, x, mask=None, rope=None, ffn_mask=None, deterministic=True):
+ residual = x
+ x = ZCRMSNorm(dtype=self.dtype)(x)
+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(
+ x, x, mask=mask, rope=rope
)
- s = s + nn.Dropout(rate=self.dropout_rate)(mix_out, deterministic=deterministic)
+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic) + residual
residual = x
- x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x, ffn_mask=ffn_mask)
+ x = ZCRMSNorm(dtype=self.dtype)(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x, ffn_mask=ffn_mask)
x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic) + residual
- return x, s
+ return x
+
+class Encoder(nn.Module):
+ """Self-attention encoder with stride-2 depthwise-separable downsampling.
-class MemoryMixerEncoder(nn.Module):
- """Encoder using MemoryMixer blocks. Output is the final memory slots S."""
+ Returns (output, mask) tuple — variable-length output + adjusted padding mask.
+ """
config: TransformerConfig
@nn.compact
@@ -216,16 +186,6 @@ class MemoryMixerEncoder(nn.Module):
dt = cfg.jax_dtype
x = x.astype(dt)
- s_base = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))
- if mask is not None:
- # mask: (B, 1, 1, T) -> (B, T)
- pool_mask = mask[:, 0, 0, :x.shape[1]]
- x_pool = jnp.sum(x * pool_mask[:, :, None], axis=1) / jnp.maximum(pool_mask.sum(axis=1, keepdims=True), 1.0)
- else:
- x_pool = jnp.mean(x, axis=1) # (B, d)
- s_bias = nn.Dense(cfg.d_model, dtype=dt, use_bias=False, kernel_init=jinit.zeros, name="slot_init")(x_pool)
- s = jnp.broadcast_to(s_base.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model)) + s_bias[:, None, :]
-
x = nn.Conv(features=cfg.d_model, kernel_size=(4,), strides=(2,),
padding='SAME', feature_group_count=cfg.d_model,
dtype=dt, use_bias=False, name="downsample_dw")(x)
@@ -240,13 +200,13 @@ class MemoryMixerEncoder(nn.Module):
mask = mask[..., :T_new]
for i in range(cfg.num_encoder_layers):
- x, s = nn.remat(MemoryMixerBlock, static_argnums=(6,))(
+ x = nn.remat(EncoderBlock, static_argnums=(5,))(
cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
- cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, cfg.dropout_rate, name=f"block_{i}"
- )(x, s, mask, rope, ffn_mask, deterministic)
+ cfg.total_layers, dt, cfg.activation, cfg.dropout_rate, name=f"block_{i}"
+ )(x, mask, rope, ffn_mask, deterministic)
- s = ZCRMSNorm(dtype=dt, name="final_norm")(s)
- return s
+ x = ZCRMSNorm(dtype=dt, name="final_norm")(x)
+ return x, mask
class DecoderBlock(nn.Module):
@@ -347,7 +307,7 @@ class EncoderDecoderTransformer(nn.Module):
self.mel_proj = nn.Dense(self.config.d_model, dtype=self.config.jax_dtype,
use_bias=True, kernel_init=default_init(), name="mel_proj")
self.spec_augment = SpecAugment()
- self.encoder = MemoryMixerEncoder(self.config)
+ self.encoder = Encoder(self.config)
self.decoder = Decoder(self.config)
def _rope(self, seq_len):
@@ -369,47 +329,39 @@ class EncoderDecoderTransformer(nn.Module):
rope = self._rope(x.shape[1])
return self.encoder(x, mask=src_mask, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)
- def decode(self, tgt, encoder_out, self_mask=None, deterministic=True):
- """Decode from encoder memory slots. No cross_mask needed (fixed-size slots)."""
+ def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None, deterministic=True):
+ """Decode from encoder output with cross_mask for variable-length encoder output."""
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
- x = self.decoder(x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope, deterministic=deterministic)
+ x = self.decoder(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)
logits = x.astype(jnp.float32) @ self.embedding.embedding.T
return logits
def __call__(self, src, tgt, src_mask=None, tgt_mask=None):
- encoder_out = self.encode_text(src, src_mask=src_mask)
- logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)
+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
+ logits = self.decode(tgt, encoder_out, self_mask=tgt_mask, cross_mask=enc_mask)
return logits
- def _run_decoder(self, encoder_out, tgt, tgt_mask=None, ffn_mask=None, deterministic=True):
+ def _run_decoder(self, encoder_out, tgt, tgt_mask=None, cross_mask=None, ffn_mask=None, deterministic=True):
"""Run decoder and return float32 hidden states."""
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
- x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)
+ x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)
return x.astype(jnp.float32)
- def _slot_diversity(self, encoder_out):
- s = encoder_out.astype(jnp.float32)
- gram = jnp.matmul(s, s.transpose(0, 2, 1))
- diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)
- return (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
-
def forward_masked(self, src, tgt, src_mask=None, tgt_mask=None, ffn_mask=None, deterministic=True):
"""Single forward with per-batch-item FFN masking. Returns (logits, slot_div)."""
- encoder_out = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)
- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask, deterministic=deterministic)
+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask, ffn_mask=ffn_mask, deterministic=deterministic)
logits = x_f32 @ self.embedding.embedding.T
- slot_div = self._slot_diversity(encoder_out)
- return logits, slot_div
+ return logits, 0.0
def forward_speech_masked(self, mel, tgt, src_mask=None, tgt_mask=None, ffn_mask=None, deterministic=True):
"""Single speech forward with per-batch-item FFN masking. Returns (logits, slot_div)."""
- encoder_out = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)
- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask, deterministic=deterministic)
+ encoder_out, enc_mask = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask, ffn_mask=ffn_mask, deterministic=deterministic)
logits = x_f32 @ self.embedding.embedding.T
- slot_div = self._slot_diversity(encoder_out)
- return logits, slot_div
+ return logits, 0.0
def _make_eval_ffn_mask(self, ff_width, B, dtype):
"""Default prefix FFN mask for eval: first ff_width neurons active."""
@@ -424,50 +376,48 @@ class EncoderDecoderTransformer(nn.Module):
emb = self.embedding.embedding
B = src.shape[0]
- encoder_out = self.encode_text(src, src_mask=src_mask)
- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)
+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask)
logits = x_f32 @ emb.T
- slot_div = self._slot_diversity(encoder_out)
mat_logits = []
if mat_ff_widths is not None:
for ff_w in mat_ff_widths:
mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)
- enc_m = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)
- x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)
+ enc_m, enc_m_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)
+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, cross_mask=enc_m_mask, ffn_mask=mask)
mat_logits.append(x_m @ emb.T)
- return logits, slot_div, mat_logits
+ return logits, 0.0, mat_logits
def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None, deterministic=True):
"""Eval-only: separate per-width speech forwards for reporting per-width PPL."""
emb = self.embedding.embedding
B = mel.shape[0]
- encoder_out = self.encode_speech(mel, src_mask=src_mask, deterministic=deterministic)
- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)
+ encoder_out, enc_mask = self.encode_speech(mel, src_mask=src_mask, deterministic=deterministic)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask)
logits = x_f32 @ emb.T
- slot_div = self._slot_diversity(encoder_out)
mat_logits = []
if mat_ff_widths is not None:
for ff_w in mat_ff_widths:
mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)
- enc_m = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask, deterministic=deterministic)
- x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)
+ enc_m, enc_m_mask = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask, deterministic=deterministic)
+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, cross_mask=enc_m_mask, ffn_mask=mask)
mat_logits.append(x_m @ emb.T)
- return logits, slot_div, mat_logits
+ return logits, 0.0, mat_logits
def init_all(self, src, tgt, mel):
"""Dummy forward through both text and speech pathways to initialize all params."""
src_mask = make_padding_mask(src, self.config.pad_token_id)
tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)
- text_out = self.encode_text(src, src_mask=src_mask)
+ text_out, text_enc_mask = self.encode_text(src, src_mask=src_mask)
mel_mask = make_mel_padding_mask(mel)
- speech_out = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)
- _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask)
- _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask)
+ speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)
+ _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask, cross_mask=text_enc_mask)
+ _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)
return jnp.zeros(())
diff --git a/src/run.py b/src/run.py
index f3534c5..f6d46c6 100644
--- a/src/run.py
+++ b/src/run.py
@@ -1,12 +1,14 @@
import argparse
+import json as _json
import pickle
+import re as _re
import sys
import jax
import jax.numpy as jnp
import numpy as np
-from .data import get_tokenizer, compute_mel_spectrogram, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_GEN_LEN
+from .data import get_tokenizer, compute_mel_spectrogram, to_snake_case, DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_GEN_LEN
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
@@ -16,6 +18,46 @@ from .model import (
)
+def normalize_tools(tools_json):
+ """Normalize tool names in a tools JSON string to snake_case.
+
+ Returns (normalized_json, name_map) where name_map maps
+ snake_name -> original_name for reverse mapping.
+ """
+ try:
+ tools = _json.loads(tools_json)
+ except (_json.JSONDecodeError, TypeError):
+ return tools_json, {}
+ name_map = {}
+ for t in tools:
+ if isinstance(t, dict) and "name" in t:
+ orig = t["name"]
+ snake = to_snake_case(orig)
+ name_map[snake] = orig
+ t["name"] = snake
+ return _json.dumps(tools, separators=(",", ":")), name_map
+
+
+def restore_tool_names(pred_text, name_map):
+ """Replace snake_case tool names in model output back to original names."""
+ if not name_map:
+ return pred_text
+ try:
+ calls = _json.loads(pred_text)
+ except (_json.JSONDecodeError, TypeError):
+ # Fallback: string-level replacement, longest names first
+ for snake, orig in sorted(name_map.items(), key=lambda x: len(x[0]), reverse=True):
+ pred_text = pred_text.replace(snake, orig)
+ return pred_text
+ if isinstance(calls, list):
+ for c in calls:
+ if isinstance(c, dict) and "name" in c:
+ c["name"] = name_map.get(c["name"], c["name"])
+ elif isinstance(calls, dict) and "name" in calls:
+ calls["name"] = name_map.get(calls["name"], calls["name"])
+ return _json.dumps(calls, separators=(",", ":"))
+
+
_decode_fn_cache = {}
@@ -30,10 +72,10 @@ def _get_decode_fn(model, max_gen_len):
tgt_mask = make_causal_mask(max_gen_len)
@jax.jit
- def decode_step(params, dec_buffer, encoder_out):
+ def decode_step(params, dec_buffer, encoder_out, cross_mask):
return model.apply(
{"params": params}, dec_buffer, encoder_out,
- self_mask=tgt_mask, method="decode",
+ self_mask=tgt_mask, cross_mask=cross_mask, method="decode",
)
_decode_fn_cache[key] = decode_step
@@ -61,12 +103,16 @@ def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LE
return q_toks + [tools_sep_id] + t_toks
-def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, seed=0, stream=True, task_token_id=None):
+def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, seed=0, stream=True, task_token_id=None, normalize=True):
"""Generate tool-call output.
Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
Decoder: prefilled with [EOS, <tool_call>], then greedy decode.
"""
+ name_map = {}
+ if normalize:
+ tools, name_map = normalize_tools(tools)
+
enc_tokens = _build_encoder_input(tokenizer, query, tools, max_enc_len)
enc_input = jnp.array([enc_tokens])
@@ -75,7 +121,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
tool_call_id = task_token_id if task_token_id is not None else tokenizer.tool_call_token_id
src_mask = make_padding_mask(enc_input, pad_id)
- encoder_out = model.apply(
+ encoder_out, enc_mask = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
@@ -94,7 +140,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
sys.stdout.write(f"\n")
sys.stdout.flush()
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
for i in range(prefix_len - 1, max_gen_len - 1):
next_logits = logits[0, i]
@@ -110,15 +156,18 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
sys.stdout.write(tokenizer.decode([next_token]))
sys.stdout.flush()
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
if stream:
sys.stdout.write("\n")
- return tokenizer.decode(generated_tokens)
+ result = tokenizer.decode(generated_tokens)
+ if normalize and name_map:
+ result = restore_tool_names(result, name_map)
+ return result
-def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN):
+def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, normalize=True):
"""Batch-generate tool-call outputs for multiple examples at once.
Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.
@@ -126,6 +175,15 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
Returns a list of decoded strings, one per example.
"""
+ name_maps = []
+ if normalize:
+ normed_tools = []
+ for t in tools_list:
+ nt, nm = normalize_tools(t)
+ normed_tools.append(nt)
+ name_maps.append(nm)
+ tools_list = normed_tools
+
B = len(queries)
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
@@ -142,7 +200,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
enc_input = jnp.array(enc_input)
src_mask = make_padding_mask(enc_input, pad_id)
- encoder_out = model.apply(
+ encoder_out, enc_mask = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
@@ -159,7 +217,7 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
finished = [False] * B
gen_tokens = [[] for _ in range(B)]
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
for pos in range(prefix_len - 1, max_gen_len - 1):
for i in range(B):
@@ -175,9 +233,12 @@ def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DE
if all(finished):
break
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
- return [tokenizer.decode(toks) for toks in gen_tokens]
+ results = [tokenizer.decode(toks) for toks in gen_tokens]
+ if normalize and name_maps:
+ results = [restore_tool_names(r, nm) for r, nm in zip(results, name_maps)]
+ return results
def load_audio(path, target_sr=16000):
@@ -209,7 +270,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="
tool_call_id = tokenizer.tool_call_token_id
src_mask = make_mel_padding_mask(mel_input)
- encoder_out = model.apply(
+ encoder_out, enc_mask = model.apply(
{"params": params}, mel_input, src_mask=src_mask, deterministic=True, method="encode_speech"
)
@@ -229,7 +290,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="
sys.stdout.write("\n")
sys.stdout.flush()
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
for i in range(prefix_len - 1, max_gen_len - 1):
next_logits = logits[0, i]
@@ -245,7 +306,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="
sys.stdout.write(tokenizer.decode([next_token]))
sys.stdout.flush()
- logits = decode_fn(params, dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
if stream:
sys.stdout.write("\n")
diff --git a/src/train.py b/src/train.py
index bcacfc6..06ba2b5 100644
--- a/src/train.py
+++ b/src/train.py
@@ -244,8 +244,7 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rn
optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
) / jnp.maximum(jnp.sum(mask), 1.0)
z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- div_loss = 1e-4 * slot_div
- return ce_loss + z_loss + div_loss
+ return ce_loss + z_loss
@@ -351,17 +350,16 @@ def _estimate_mat_params(config, matryoshka_factor):
n_enc = config.num_encoder_layers
n_dec = config.num_decoder_layers
kv_dim = config.num_kv_heads * (d // config.num_heads)
- m = config.num_memory_slots
d_ff = config.d_ff // matryoshka_factor
emb = v * d
attn = d * d + d * kv_dim * 2 + d * d
ffn = d * d_ff * 3
- mixer_token = m * d_ff * 3
- mixer_channel = d * d_ff * 3
- enc_block = attn + ffn + mixer_token + mixer_channel
- dec_block = attn * 2 + ffn
- total = emb + n_enc * enc_block + n_dec * dec_block
+ # Encoder: self-attn + FFN per block, plus downsample conv params
+ downsample = d * 4 + d * d # depthwise conv (kernel=4) + pointwise Dense
+ enc_block = attn + ffn
+ dec_block = attn * 2 + ffn # self-attn + cross-attn + FFN
+ total = emb + downsample + n_enc * enc_block + n_dec * dec_block
return int(total)
@@ -469,7 +467,6 @@ def train(args):
print(f" d_model {config.d_model:>12}")
print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")
print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")
- print(f" Memory slots {config.num_memory_slots:>12}")
print(f" Activation {config.activation:>12}")
print(f" Dtype {config.dtype:>12}")
print(f" ─────────────────────────────────────")