Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku88a2b5389b9d
diff --git a/scripts/compare_voices.py b/scripts/compare_voices.py
deleted file mode 100644
index 4dfc82a..0000000
--- a/scripts/compare_voices.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-"""Generate sample TTS audio with Wavenet vs Neural2 for quality comparison."""
-
-import io
-import wave
-from google.cloud import texttospeech
-
-client = texttospeech.TextToSpeechClient()
-
-QUERIES = [
- "What's the weather like in San Francisco today?",
- "Book me a flight from London to Tokyo next Friday",
- "Send an email to the team about the meeting at 3pm",
-]
-
-VOICES = [
- ("wavenet_male_us", "en-US-Wavenet-D", "en-US"),
- ("chirp3_us", "en-US-Chirp3-HD-Achernar", "en-US"),
- ("journey_male_us", "en-US-Journey-D", "en-US"),
- ("wavenet_female_gb", "en-GB-Wavenet-A", "en-GB"),
- ("chirp3_gb", "en-GB-Chirp3-HD-Leda", "en-GB"),
- ("journey_female_us", "en-US-Journey-F", "en-US"),
- ("wavenet_male_au", "en-AU-Wavenet-B", "en-AU"),
- ("chirp3_au", "en-AU-Chirp3-HD-Charon", "en-AU"),
- ("journey_female_us2", "en-US-Journey-O", "en-US"),
-]
-
-
-def pcm_to_wav(pcm_bytes, sr=16000):
- buf = io.BytesIO()
- with wave.open(buf, "wb") as wf:
- wf.setnchannels(1)
- wf.setsampwidth(2)
- wf.setframerate(sr)
- wf.writeframes(pcm_bytes)
- return buf.getvalue()
-
-
-import os
-os.makedirs("voice_samples", exist_ok=True)
-
-for qi, query in enumerate(QUERIES):
- for label, voice_name, lang in VOICES:
- print(f" {label} / query {qi+1}...", end="", flush=True)
- response = client.synthesize_speech(
- input=texttospeech.SynthesisInput(text=query),
- voice=texttospeech.VoiceSelectionParams(
- language_code=lang, name=voice_name,
- ),
- audio_config=texttospeech.AudioConfig(
- audio_encoding=texttospeech.AudioEncoding.LINEAR16,
- sample_rate_hertz=16000,
- ),
- )
- wav = pcm_to_wav(response.audio_content)
- path = f"voice_samples/q{qi+1}_{label}.wav"
- with open(path, "wb") as f:
- f.write(wav)
- print(f" saved {path}")
-
-print(f"\nDone! {len(QUERIES) * len(VOICES)} files in voice_samples/")
diff --git a/scripts/synthesize_tools_data.py b/scripts/synthesize_tools_data.py
deleted file mode 100644
index 6478efa..0000000
--- a/scripts/synthesize_tools_data.py
+++ /dev/null
@@ -1,516 +0,0 @@
-#!/usr/bin/env python3
-"""Synthesize tool-calling data from PleIAs/SYNTH via OpenRouter.
-
-Each query is sent to a randomly-selected frontier model to generate
-(tools, answer) pairs, then validated and checkpointed as JSONL.
-
-Usage:
- python scripts/synthesize_tools_data.py
- python scripts/synthesize_tools_data.py --model google/gemini-3.1-flash-lite-preview
- python scripts/synthesize_tools_data.py --max-shards 1 --max-samples 100
- python scripts/synthesize_tools_data.py --resume # continue from checkpoint
-"""
-
-import argparse
-import json
-import logging
-import os
-import random
-import sys
-import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from pathlib import Path
-
-import itertools
-
-import requests
-
-logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
-logger = logging.getLogger(__name__)
-
-_PROJECT_ROOT = Path(__file__).resolve().parent.parent
-_OUTPUT_DIR = _PROJECT_ROOT / "data" / "synth_tool_calls"
-_CHECKPOINT_PATH = _OUTPUT_DIR / "checkpoint.jsonl"
-_PARQUET_CACHE_DIR = _PROJECT_ROOT / ".data_cache"
-
-OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
-
-MODEL_POOL = [
- "google/gemini-3.1-flash-lite-preview",
-]
-DEFAULT_MODEL = "google/gemini-3.1-flash-lite-preview"
-
-SYSTEM_PROMPT = """\
-You are a synthetic data generator for training a tool-calling AI model.
-
-Given a user query, you must invent a realistic set of tools (APIs/functions) that \
-could help answer it, then produce the correct tool call(s).
-
-Vary the domain widely — cover areas like software development, data analysis, \
-e-commerce, finance, healthcare, travel, IoT, social media, education, DevOps, legal, \
-HR, and more. Do NOT default to simple personal-assistant tasks.
-
-Rules:
-1. Invent 0-3 tools as JSON. Each tool MUST have: name, description, parameters \
-(dict of param_name -> {type, description, required}). Use realistic parameter types \
-including string, number, boolean, array, object. Include nested objects or arrays \
-where appropriate.
-2. Only ONE or TWO tools should be relevant to the query; the rest are plausible \
-distractors from the SAME domain.
-3. Tool names should be snake_case, realistic API-style, and DIVERSE — avoid repeating \
-common names like search_web or get_weather across samples. Think of specific, \
-domain-appropriate APIs (e.g. query_elasticsearch, run_sql_migration, price_option_chain, \
-schedule_k8s_cronjob, fetch_patient_labs).
-4. Produce the answer as a JSON list of tool calls: [{"name": "...", "arguments": {...}}].
-5. For complex queries, use MULTIPLE tool calls (chained or parallel) where realistic.
-6. If the query genuinely needs no tool, return an empty answer list [].
-7. Argument values MUST be grounded in the query — do not hallucinate values.
-8. Respond in the SAME LANGUAGE as the query.
-9. Return ONLY valid JSON, no markdown fences, no commentary.
-
-Output format (strict JSON object):
-{"tools": [...], "answer": [...]}"""
-
-USER_TEMPLATE = "Query: {query}"
-
-
-def _discover_shard_urls(token, max_shards=None):
- """Discover available parquet shard URLs from the HF API."""
- api_url = "https://huggingface.co/api/datasets/PleIAs/SYNTH/parquet"
- headers = {}
- if token:
- headers["Authorization"] = f"Bearer {token}"
- resp = requests.get(api_url, headers=headers, timeout=30)
- resp.raise_for_status()
- data = resp.json() # {"default": {"train": [url, ...]}}
- urls = data["default"]["train"]
- if max_shards:
- urls = urls[:max_shards]
- logger.info(f"Discovered {len(urls)} parquet shards")
- return urls
-
-
-def download_shard(url, shard_idx, token):
- """Download a single parquet shard, returning the local path."""
- cache_path = _PARQUET_CACHE_DIR / f"synth_shard{shard_idx}.parquet"
- if cache_path.exists():
- logger.info(f"Using cached shard {shard_idx}: {cache_path}")
- return cache_path
-
- _PARQUET_CACHE_DIR.mkdir(parents=True, exist_ok=True)
-
- headers = {}
- if token:
- headers["Authorization"] = f"Bearer {token}"
-
- logger.info(f"Downloading shard {shard_idx}...")
- resp = requests.get(url, headers=headers, stream=True, timeout=120)
- resp.raise_for_status()
-
- total = int(resp.headers.get("content-length", 0))
- downloaded = 0
- with open(cache_path, "wb") as f:
- for chunk in resp.iter_content(chunk_size=8 * 1024 * 1024):
- f.write(chunk)
- downloaded += len(chunk)
- if total:
- pct = downloaded / total * 100
- print(f"\r {downloaded / 1e6:.0f}/{total / 1e6:.0f} MB ({pct:.0f}%)", end="", flush=True)
- print()
- logger.info(f"Saved shard {shard_idx} to {cache_path}")
- return cache_path
-
-
-def load_queries_from_parquet(path, max_samples=None):
- """Load queries from parquet, return list of (sample_id, language, query)."""
- import pyarrow.parquet as pq
-
- logger.info(f"Reading parquet: {path}")
- table = pq.read_table(path, columns=["synth_id", "language", "query"])
- n = table.num_rows
- logger.info(f" {n:,} rows in shard")
-
- ids = table.column("synth_id").to_pylist()
- langs = table.column("language").to_pylist()
- queries = table.column("query").to_pylist()
-
- samples = list(zip(ids, langs, queries))
-
- rng = random.Random(42)
- rng.shuffle(samples)
-
- if max_samples:
- samples = samples[:max_samples]
-
- logger.info(f" Selected {len(samples):,} samples")
- return samples
-
-
-def load_checkpoint():
- """Load completed synth_ids from checkpoint file."""
- done = set()
- if _CHECKPOINT_PATH.exists():
- with open(_CHECKPOINT_PATH) as f:
- for line in f:
- try:
- row = json.loads(line)
- done.add(row["synth_id"])
- except (json.JSONDecodeError, KeyError):
- continue
- logger.info(f"Resuming: {len(done):,} already completed")
- return done
-
-
-def call_openrouter(query, api_key, model=DEFAULT_MODEL, max_retries=3):
- """Call OpenRouter API to generate tools + answer for a query."""
- headers = {
- "Authorization": f"Bearer {api_key}",
- "Content-Type": "application/json",
- }
- payload = {
- "model": model,
- "messages": [
- {"role": "system", "content": SYSTEM_PROMPT},
- {"role": "user", "content": USER_TEMPLATE.format(query=query)},
- ],
- "temperature": 0.7,
- "max_tokens": 4096,
- }
-
- for attempt in range(max_retries):
- try:
- resp = requests.post(
- OPENROUTER_URL, headers=headers, json=payload, timeout=60
- )
- if resp.status_code == 429:
- wait = min(2 ** attempt * 2, 30)
- logger.warning(f"[{model}] 429 rate-limited, retrying in {wait}s")
- time.sleep(wait)
- continue
- if not resp.ok:
- logger.warning(f"[{model}] HTTP {resp.status_code}: {resp.text[:200]}")
- if attempt == max_retries - 1:
- return None
- time.sleep(2 ** attempt)
- continue
- content = resp.json()["choices"][0]["message"]["content"]
- parsed = _parse_response(content)
- if parsed is None:
- logger.warning(f"[{model}] Failed to parse response: {content[:200]}")
- return parsed
- except (requests.RequestException, KeyError, IndexError) as e:
- logger.warning(f"[{model}] Request error (attempt {attempt+1}/{max_retries}): {e}")
- if attempt == max_retries - 1:
- return None
- time.sleep(2 ** attempt)
- return None
-
-
-def _parse_response(content):
- """Parse LLM response into (tools, answer) or None."""
- content = content.strip()
- if content.startswith("```"):
- content = content.split("\n", 1)[-1]
- if content.endswith("```"):
- content = content[:-3]
- content = content.strip()
-
- try:
- obj = json.loads(content)
- except json.JSONDecodeError:
- return None
-
- if not isinstance(obj, dict):
- return None
-
- tools = obj.get("tools")
- answer = obj.get("answer")
-
- if not isinstance(tools, list):
- return None
- if not isinstance(answer, list):
- return None
-
- for t in tools:
- if not isinstance(t, dict) or "name" not in t:
- return None
-
- for a in answer:
- if not isinstance(a, dict) or "name" not in a:
- return None
- if "arguments" not in a:
- a["arguments"] = {}
-
- return tools, answer
-
-
-def _validate_and_format(synth_id, language, query, tools, answer, model, source="synth-pleias"):
- """Validate the generated data and return a formatted row or None."""
- tool_names = {t["name"] for t in tools}
- for a in answer:
- if a["name"] not in tool_names:
- return None
-
- tool_params = {}
- for t in tools:
- params = t.get("parameters") or {}
- tool_params[t["name"]] = set(params.keys())
-
- for a in answer:
- args = a.get("arguments", {})
- if not isinstance(args, dict):
- return None
- schema_params = tool_params.get(a["name"], set())
- if args and not set(args.keys()).issubset(schema_params):
- return None
-
- return {
- "synth_id": synth_id,
- "language": language,
- "query": query,
- "answers": json.dumps(answer),
- "tools": json.dumps(tools),
- "source": source,
- "model": model,
- }
-
-
-
-class _KeyModelRotator:
- """Thread-safe round-robin across (api_key, model) pairs."""
-
- def __init__(self, api_keys, models):
- pairs = [(k, m) for k in api_keys for m in models]
- self._cycle = itertools.cycle(pairs)
- import threading
- self._lock = threading.Lock()
-
- def next(self):
- with self._lock:
- return next(self._cycle)
-
-
-def process_sample(sample, rotator, source_tag="synth-pleias"):
- """Process a single sample: call LLM, validate, return row or None."""
- synth_id, language, query = sample
- api_key, model = rotator.next()
- result = call_openrouter(query, api_key, model=model)
- if result is None:
- return None
- tools, answer = result
- row = _validate_and_format(synth_id, language, query, tools, answer, model, source_tag)
- if row is None:
- logger.warning(f"[{model}] Validation failed for synth_id={synth_id}")
- return row
-
-
-def _process_batch(samples, label, batch_idx, rotator, args, tqdm, source_tag,
- since_last_upload):
- """Process a batch of samples, append to checkpoint. Returns (generated, failed)."""
- logger.info(f"\n{'=' * 60}")
- logger.info(f"Batch: {label}")
- logger.info(f"{'=' * 60}")
-
- done_ids = load_checkpoint()
- pending = [s for s in samples if s[0] not in done_ids]
- if not pending:
- logger.info("All samples already done, skipping")
- return 0, 0
- logger.info(f"Processing {len(pending):,} samples ({len(done_ids):,} already done)")
-
- generated = 0
- failed = 0
- out_file = open(_CHECKPOINT_PATH, "a")
-
- try:
- with ThreadPoolExecutor(max_workers=args.workers) as executor:
- futures = {}
- for sample in pending:
- fut = executor.submit(
- process_sample, sample, rotator, source_tag,
- )
- futures[fut] = sample[0]
-
- pbar = tqdm(
- as_completed(futures),
- total=len(futures),
- desc=label,
- unit="sample",
- dynamic_ncols=True,
- ) if tqdm else as_completed(futures)
-
- for i, fut in enumerate(pbar if tqdm else as_completed(futures), 1):
- try:
- row = fut.result()
- except Exception:
- failed += 1
- row = None
-
- if row is not None:
- out_file.write(json.dumps(row) + "\n")
- generated += 1
- since_last_upload[0] += 1
- else:
- failed += 1
-
- if tqdm:
- rate = generated / max(generated + failed, 1) * 100
- pbar.set_postfix(ok=generated, fail=failed, rate=f"{rate:.0f}%")
- if i % args.batch_size == 0:
- out_file.flush()
- else:
- if i % args.batch_size == 0:
- out_file.flush()
- total = generated + failed
- rate = generated / max(total, 1) * 100
- logger.info(
- f" [{total:,}/{len(pending):,}] "
- f"generated={generated:,} failed={failed:,} "
- f"({rate:.0f}% success)"
- )
-
- if since_last_upload[0] >= args.upload_every:
- out_file.flush()
- _export_dataset()
- since_last_upload[0] = 0
-
- if tqdm:
- pbar.close()
- except KeyboardInterrupt:
- logger.info("\nInterrupted — progress saved to checkpoint")
- finally:
- out_file.flush()
- out_file.close()
-
- logger.info(f"{label}: {generated:,} generated, {failed:,} failed")
- return generated, failed
-
-
-def main():
- parser = argparse.ArgumentParser(description="Synthesize tool-calling data from PleIAs/SYNTH")
- parser.add_argument("--max-samples", type=int, default=None,
- help="Max queries to process (default: all in shard)")
- parser.add_argument("--models", type=str, nargs="+", default=None,
- help=f"OpenRouter models to rotate (default: built-in pool)")
- parser.add_argument("--model", type=str, default=None,
- help="Use a single model instead of the pool")
- parser.add_argument("--workers", type=int, default=128,
- help="Concurrent API requests (default: 128)")
- parser.add_argument("--max-shards", type=int, default=None,
- help="Max parquet shards to process (default: all)")
- parser.add_argument("--resume", action="store_true",
- help="Resume from checkpoint")
- parser.add_argument("--batch-size", type=int, default=500,
- help="Samples between progress logs (default: 500)")
- parser.add_argument("--upload-every", type=int, default=1000,
- help="Upload to GCS every N successful generations (default: 1000)")
- args = parser.parse_args()
-
- raw_keys = os.environ.get("OPENROUTER_API_KEY", "")
- api_keys = [k.strip() for k in raw_keys.split(",") if k.strip()]
- if not api_keys:
- print("Error: set OPENROUTER_API_KEY environment variable (comma-separated for multiple)")
- sys.exit(1)
- logger.info(f"API keys: {len(api_keys)}")
-
- if args.model:
- models = [args.model]
- elif args.models:
- models = args.models
- else:
- models = list(MODEL_POOL)
- logger.info(f"Model pool ({len(models)}): {', '.join(models)}")
- logger.info(f"Key x Model pairs: {len(api_keys) * len(models)} (each gets its own rate limit)")
-
- rotator = _KeyModelRotator(api_keys, models)
-
- _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
- if not args.resume and _CHECKPOINT_PATH.exists():
- _CHECKPOINT_PATH.unlink()
- logger.info("Cleared previous checkpoint (use --resume to keep)")
-
- try:
- from tqdm import tqdm
- except ImportError:
- logger.warning("tqdm not installed, falling back to basic progress logging")
- tqdm = None
-
- total_generated = 0
- total_failed = 0
- batch_idx = 0
- since_last_upload = [0]
-
- hf_token = os.environ.get("HF_TOKEN")
- shard_urls = _discover_shard_urls(hf_token, max_shards=args.max_shards)
-
- for shard_idx, shard_url in enumerate(shard_urls):
- parquet_path = download_shard(shard_url, shard_idx, hf_token)
- samples = load_queries_from_parquet(parquet_path, max_samples=args.max_samples)
- gen, fail = _process_batch(
- samples, f"PleIAs shard {shard_idx}", batch_idx, rotator,
- args, tqdm, source_tag="synth-pleias",
- since_last_upload=since_last_upload,
- )
- total_generated += gen
- total_failed += fail
- batch_idx += 1
-
- if since_last_upload[0] > 0:
- _export_dataset()
-
- logger.info(f"\nAll done: {total_generated:,} generated, {total_failed:,} failed")
-
-
-def _export_dataset():
- """Convert checkpoint JSONL to the unified format and push to GCS."""
- from datasets import Dataset
-
- rows = []
- with open(_CHECKPOINT_PATH) as f:
- for line in f:
- try:
- row = json.loads(line)
- rows.append({
- "query": row["query"],
- "answers": row["answers"],
- "tools": row["tools"],
- "source": row.get("source", "synth-pleias"),
- "model": row.get("model", "unknown"),
- })
- except (json.JSONDecodeError, KeyError):
- continue
-
- if not rows:
- return
-
- ds = Dataset.from_list(rows)
- out_path = _OUTPUT_DIR / "dataset"
- ds.save_to_disk(str(out_path))
- logger.info(f"Exported {len(rows):,} examples to {out_path}/")
-
- try:
- sys.path.insert(0, str(_PROJECT_ROOT))
- from src.gcs import upload_directory
- gcs_prefix = "synth_tool_calls"
- upload_directory(str(out_path), gcs_prefix)
- except Exception as e:
- logger.warning(f"GCS upload failed: {e}")
-
- for i, row in enumerate(rows[:10]):
- try:
- tools = json.loads(row["tools"])
- answers = json.loads(row["answers"])
- except (json.JSONDecodeError, TypeError):
- tools, answers = [], []
- called_names = [a["name"] for a in answers if isinstance(a, dict)]
- print(f"\n{'─' * 60}")
- print(f"[{i+1}] Query: {row['query'][:200]}")
- print(f" Available tools ({len(tools)}): {', '.join(t.get('name','?') for t in tools)}")
- print(f" Called tools ({len(called_names)}): {', '.join(called_names) or '(none)'}")
- print(f" Answers: {row['answers'][:200]}")
- print(f"{'─' * 60}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
deleted file mode 100755
index 8a23e68..0000000
--- a/scripts/tools_data.py
+++ /dev/null
@@ -1,879 +0,0 @@
-"""
-Unified tool-calling dataset builder.
-
-Loads HuggingFace tool-calling datasets and converts them into a common schema:
-
- query : str – the user's natural-language request
- answers : str – JSON list of {"name": ..., "arguments": {...}} tool calls
- tools : str – JSON list of available tool definitions
- source : str – which dataset the example came from
-
-Datasets:
-
-1. Salesforce/xlam-function-calling-60k
- Already in the target schema (query, answers, tools).
-
-2. glaiveai/glaive-function-calling-v2
- Tool definitions embedded in the system prompt; chat is a flat string with
- USER:/A: role markers and <functioncall> tags. Examples with empty answers
- (answers=[]) are kept so the model learns when NOT to call functions.
-
-3. Agent-Ark/Toucan-1.5M (OSS, Kimi-K2, Qwen3 subsets)
- Large-scale tool-calling dataset. Tool calls extracted from 'messages' column
- (assistant messages with function_call objects), filtered by 'target_tools'.
- Tool schemas converted from OpenAI format to flat format.
-
-4. Team-ACE/ToolACE
- Synthetic tool-calling dataset (11.3k) with dual-layer verification.
- Tool definitions in system prompt JSON; calls use bracket notation
- [func(key=val)]. Grounding filter skipped (pre-verified).
-"""
-
-import ast
-import json
-import multiprocessing as mp
-import os
-import random
-import re
-from collections import Counter
-
-from datasets import load_dataset, Dataset, concatenate_datasets
-
-_NUM_WORKERS = min(20, max(1, (os.cpu_count() or 1) // 4))
-
-
-DATASETS = {
- "xlam-function-calling-60k": "Salesforce/xlam-function-calling-60k",
- "glaive-function-calling": "glaiveai/glaive-function-calling-v2",
- "toucan-oss": ("Agent-Ark/Toucan-1.5M", "OSS"),
- "toucan-kimi": ("Agent-Ark/Toucan-1.5M", "Kimi-K2"),
- "toucan-qwen": ("Agent-Ark/Toucan-1.5M", "Qwen3"),
- "toucan-sft": ("Agent-Ark/Toucan-1.5M", "SFT"),
- "toolace": "Team-ACE/ToolACE",
- "nemotron-tc": ("nvidia/Nemotron-Agentic-v1", None, "tool_calling"),
-}
-
-
-def convert_xlam(dataset):
- return dataset.select_columns(["query", "answers", "tools"])
-
-
-def _convert_glaive_chunk(chunk):
- rows = []
- for system, chat in chunk:
- tools = _extract_json_objects(system)
- query, answers = _parse_glaive_chat(chat)
- if query:
- rows.append({
- "query": query,
- "answers": json.dumps(answers),
- "tools": json.dumps(tools),
- })
- return rows
-
-
-def convert_glaive(dataset):
- pairs = list(zip(dataset["system"], dataset["chat"]))
- chunk_size = max(1, len(pairs) // (_NUM_WORKERS * 4))
- chunks = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)]
- with mp.Pool(_NUM_WORKERS) as pool:
- results = pool.map(_convert_glaive_chunk, chunks)
- rows = [r for chunk in results for r in chunk]
- return Dataset.from_list(rows)
-
-
-def _extract_json_objects(text):
- """Extract top-level JSON objects with a 'name' field from text."""
- objects = []
- depth = 0
- start = None
- for i, ch in enumerate(text):
- if ch == '{':
- if depth == 0:
- start = i
- depth += 1
- elif ch == '}':
- depth -= 1
- if depth == 0 and start is not None:
- try:
- obj = json.loads(text[start:i + 1])
- if "name" in obj:
- objects.append(obj)
- except json.JSONDecodeError:
- pass
- start = None
- return objects
-
-
-def _parse_glaive_chat(chat):
- """Extract first user query and all function calls from glaive chat."""
- query = None
- answers = []
-
- user_match = re.search(r'USER:\s*(.*?)(?=\n\s*(?:A:|ASSISTANT:)|$)', chat, re.DOTALL)
- if user_match:
- query = user_match.group(1).strip()
-
- for m in re.finditer(r'<functioncall>\s*(\{.+?\})\s*(?:<\|endoftext\|>|$)', chat, re.DOTALL):
- raw = m.group(1)
- raw = re.sub(r"'(\{.*?\})'", r'"\1"', raw)
- try:
- fc = json.loads(raw)
- except json.JSONDecodeError:
- raw2 = re.sub(
- r"'(\{.*?\})'",
- lambda x: json.dumps(x.group(1)),
- m.group(1),
- )
- try:
- fc = json.loads(raw2)
- except json.JSONDecodeError:
- continue
- args = fc.get("arguments", {})
- if isinstance(args, str):
- try:
- args = json.loads(args)
- except json.JSONDecodeError:
- pass
- answers.append({"name": fc["name"], "arguments": args})
-
- return query, answers
-
-
-def _convert_openai_tools(tools_json):
- """Convert OpenAI-format tools to flat format.
-
- OpenAI: [{"type": "function", "function": {"name": ..., "parameters": {"type": "object", "properties": {...}}}}]
- Flat: [{"name": ..., "description": ..., "parameters": {"param_name": {"type": ..., "description": ...}}}]
- """
- try:
- tools = json.loads(tools_json) if isinstance(tools_json, str) else tools_json
- except (json.JSONDecodeError, TypeError):
- return []
- flat = []
- for t in tools:
- if isinstance(t, dict) and "function" in t:
- fn = t["function"]
- elif isinstance(t, dict) and "name" in t:
- fn = t
- else:
- continue
- params = fn.get("parameters") or {}
- # Unwrap JSON Schema object wrapper → flat param dict
- if isinstance(params, dict) and "properties" in params:
- params = params["properties"]
- flat.append({
- "name": fn["name"],
- "description": fn.get("description", ""),
- "parameters": params,
- })
- return flat
-
-
-def _extract_toucan_calls(messages_json, tool_names=None):
- """Extract tool calls from Toucan messages.
-
- Collects all function_call entries from assistant messages.
- If tool_names is provided, only keeps calls whose name matches an available tool.
- """
- try:
- messages = json.loads(messages_json) if isinstance(messages_json, str) else messages_json
- except (json.JSONDecodeError, TypeError):
- return []
- calls = []
- for msg in messages:
- if not isinstance(msg, dict):
- continue
- if msg.get("role") != "assistant":
- continue
- fc = msg.get("function_call")
- if not fc or not isinstance(fc, dict):
- continue
- name = fc.get("name", "")
- if not name:
- continue
- if tool_names is not None and name not in tool_names:
- continue
- args = fc.get("arguments", {})
- if isinstance(args, str):
- try:
- args = json.loads(args)
- except (json.JSONDecodeError, TypeError, ValueError):
- args = {}
- if not isinstance(args, dict):
- args = {}
- calls.append({"name": name, "arguments": args})
- return calls
-
-
-def _convert_toucan_chunk(chunk):
- rows = []
- for question, tools_raw, messages in chunk:
- query = (question or "").strip()
- if not query:
- continue
- tools = _convert_openai_tools(tools_raw or "[]")
- if not tools:
- continue
- tool_names = {t["name"] for t in tools}
- calls = _extract_toucan_calls(messages or "[]", tool_names)
- if not calls:
- continue
- rows.append({
- "query": query,
- "answers": json.dumps(calls),
- "tools": json.dumps(tools),
- })
- return rows
-
-
-def convert_toucan(dataset):
- questions = dataset["question"]
- if "available_tools" in dataset.column_names:
- tools_raw = dataset["available_tools"]
- elif "tools" in dataset.column_names:
- tools_raw = dataset["tools"]
- else:
- tools_raw = ["[]"] * len(dataset)
- messages = dataset["messages"]
- triples = list(zip(questions, tools_raw, messages))
- chunk_size = max(1, len(triples) // (_NUM_WORKERS * 4))
- chunks = [triples[i:i + chunk_size] for i in range(0, len(triples), chunk_size)]
- with mp.Pool(_NUM_WORKERS) as pool:
- results = pool.map(_convert_toucan_chunk, chunks)
- rows = [r for chunk in results for r in chunk]
- return Dataset.from_list(rows)
-
-
-def _extract_toolace_tools(system_text):
- """Extract JSON tool definitions array from ToolACE system prompt."""
- start = system_text.find('[{')
- if start == -1:
- return []
- depth = 0
- for i in range(start, len(system_text)):
- if system_text[i] == '[':
- depth += 1
- elif system_text[i] == ']':
- depth -= 1
- if depth == 0:
- try:
- return json.loads(system_text[start:i + 1])
- except json.JSONDecodeError:
- return []
- return []
-
-
-def _split_bracket_args(s, sep=','):
- """Split on separator respecting nested brackets and quoted strings."""
- parts = []
- depth = 0
- in_str = False
- str_char = None
- current = []
- for i, ch in enumerate(s):
- if in_str:
- current.append(ch)
- if ch == str_char and (i == 0 or s[i - 1] != '\\'):
- in_str = False
- elif ch in ('"', "'"):
- in_str = True
- str_char = ch
- current.append(ch)
- elif ch in ('(', '[', '{'):
- depth += 1
- current.append(ch)
- elif ch in (')', ']', '}'):
- depth -= 1
- current.append(ch)
- elif ch == sep and depth == 0:
- parts.append(''.join(current))
- current = []
- else:
- current.append(ch)
- if current:
- parts.append(''.join(current))
- return parts
-
-
-def _parse_bracket_value(s):
- """Parse a value string from bracket notation arguments."""
- s = s.strip()
- if not s:
- return s
- try:
- return json.loads(s)
- except (json.JSONDecodeError, ValueError):
- pass
- if s == 'True':
- return True
- if s == 'False':
- return False
- if s in ('None', 'null'):
- return None
- try:
- return ast.literal_eval(s)
- except (ValueError, SyntaxError):
- pass
- return s
-
-
-def _parse_toolace_calls(text, tool_names):
- """Parse ToolACE bracket notation into tool calls using known names."""
- text = text.strip()
- if not text.startswith('[') or not text.endswith(']'):
- return []
- inner = text[1:-1].strip()
- if not inner:
- return []
-
- sorted_names = sorted(tool_names, key=len, reverse=True)
- calls = []
- pos = 0
-
- while pos < len(inner):
- while pos < len(inner) and inner[pos] in (' ', ',', '\n', '\t'):
- pos += 1
- if pos >= len(inner):
- break
-
- matched = None
- for name in sorted_names:
- if inner[pos:pos + len(name)] == name:
- rest = inner[pos + len(name):].lstrip()
- if rest and rest[0] == '(':
- matched = name
- break
-
- if not matched:
- break
-
- pos += len(matched)
- while pos < len(inner) and inner[pos] == ' ':
- pos += 1
- if pos >= len(inner) or inner[pos] != '(':
- break
-
- depth = 0
- in_str = False
- str_char = None
- end = pos
- while end < len(inner):
- ch = inner[end]
- if in_str:
- if ch == str_char and (end == 0 or inner[end - 1] != '\\'):
- in_str = False
- elif ch in ('"', "'"):
- in_str = True
- str_char = ch
- elif ch == '(':
- depth += 1
- elif ch == ')':
- depth -= 1
- if depth == 0:
- break
- end += 1
-
- args_str = inner[pos + 1:end].strip()
- arguments = {}
- if args_str:
- for pair in _split_bracket_args(args_str, ','):
- pair = pair.strip()
- eq = pair.find('=')
- if eq < 0:
- continue
- key = pair[:eq].strip()
- val = _parse_bracket_value(pair[eq + 1:])
- arguments[key] = val
-
- calls.append({"name": matched, "arguments": arguments})
- pos = end + 1
-
- return calls
-
-
-def _convert_toolace_chunk(chunk):
- rows = []
- for system, convs in chunk:
- raw_tools = _extract_toolace_tools(system)
- tools = _convert_openai_tools(raw_tools)
- if not tools:
- continue
-
- tool_names = {t["name"] for t in tools}
-
- query = None
- assistant_text = None
- for msg in convs:
- role = msg.get("from", "")
- val = (msg.get("value") or "").strip()
- if role == "user" and query is None:
- query = val
- elif role == "assistant" and assistant_text is None:
- assistant_text = val
-
- if not query:
- continue
-
- calls = []
- if assistant_text:
- calls = _parse_toolace_calls(assistant_text, tool_names)
-
- rows.append({
- "query": query,
- "answers": json.dumps(calls),
- "tools": json.dumps(tools),
- })
- return rows
-
-
-def convert_toolace(dataset):
- pairs = list(zip(dataset["system"], dataset["conversations"]))
- chunk_size = max(1, len(pairs) // (_NUM_WORKERS * 4))
- chunks = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)]
- with mp.Pool(_NUM_WORKERS) as pool:
- results = pool.map(_convert_toolace_chunk, chunks)
- rows = [r for chunk in results for r in chunk]
- return Dataset.from_list(rows)
-
-
-def _extract_nemotron_calls(messages, tool_names=None):
- """Extract tool calls from Nemotron messages (OpenAI tool_calls format)."""
- calls = []
- for msg in messages:
- if not isinstance(msg, dict) or msg.get("role") != "assistant":
- continue
-
- # OpenAI tool_calls format (newer)
- for tc in (msg.get("tool_calls") or []):
- if not isinstance(tc, dict):
- continue
- fn = tc.get("function", tc)
- name = fn.get("name", "")
- if not name:
- continue
- if tool_names and name not in tool_names:
- continue
- args = fn.get("arguments", {})
- if isinstance(args, str):
- try:
- args = json.loads(args)
- except (json.JSONDecodeError, TypeError):
- args = {}
- if not isinstance(args, dict):
- args = {}
- calls.append({"name": name, "arguments": args})
-
- # Fallback: function_call format (older OpenAI)
- fc = msg.get("function_call")
- if fc and isinstance(fc, dict):
- name = fc.get("name", "")
- if name and (not tool_names or name in tool_names):
- args = fc.get("arguments", {})
- if isinstance(args, str):
- try:
- args = json.loads(args)
- except (json.JSONDecodeError, TypeError):
- args = {}
- if not isinstance(args, dict):
- args = {}
- calls.append({"name": name, "arguments": args})
-
- return calls
-
-
-def _convert_nemotron_chunk(chunk):
- rows = []
- for tools_raw, messages in chunk:
- tools = _convert_openai_tools(tools_raw or [])
- if not tools:
- continue
-
- tool_names = {t["name"] for t in tools}
-
- query = None
- for msg in (messages or []):
- if isinstance(msg, dict) and msg.get("role") == "user":
- query = (msg.get("content") or "").strip()
- break
-
- if not query:
- continue
-
- calls = _extract_nemotron_calls(messages, tool_names)
- if not calls:
- continue
-
- rows.append({
- "query": query,
- "answers": json.dumps(calls),
- "tools": json.dumps(tools),
- })
- return rows
-
-
-def convert_nemotron(dataset):
- pairs = list(zip(dataset["tools"], dataset["messages"]))
- chunk_size = max(1, len(pairs) // (_NUM_WORKERS * 4))
- chunks = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)]
- with mp.Pool(_NUM_WORKERS) as pool:
- results = pool.map(_convert_nemotron_chunk, chunks)
- rows = [r for chunk in results for r in chunk]
- return Dataset.from_list(rows)
-
-
-CONVERTERS = {
- "xlam-function-calling-60k": convert_xlam,
- "glaive-function-calling": convert_glaive,
- "toucan-oss": convert_toucan,
- "toucan-kimi": convert_toucan,
- "toucan-qwen": convert_toucan,
- "toucan-sft": convert_toucan,
- "toolace": convert_toolace,
- "nemotron-tc": convert_nemotron,
-}
-
-_PLACEHOLDER_VALUES = {
- "john_doe", "jane_doe", "john.doe", "jane.doe",
- "john_doe_456", "jane_doe_123",
- "1234567890", "0987654321",
- "123 main st", "123 main street",
-}
-
-
-def _has_placeholder_args(answers_json):
- """Check if any argument values are known placeholders."""
- try:
- answers = json.loads(answers_json)
- except (json.JSONDecodeError, TypeError):
- return False
- if not answers:
- return False
- for call in answers:
- args = call.get("arguments", {})
- if isinstance(args, str):
- try:
- args = json.loads(args)
- except (json.JSONDecodeError, TypeError):
- continue
- if not isinstance(args, dict):
- continue
- for val in args.values():
- if isinstance(val, str) and val.lower().strip() in _PLACEHOLDER_VALUES:
- return True
- return False
-
-
-
-def _answer_calls_valid_tools(ex):
- """Check that every tool called in the answer exists in the tools list."""
- try:
- answers = json.loads(ex["answers"])
- tools = json.loads(ex["tools"])
- except (json.JSONDecodeError, TypeError):
- return True
- if not answers:
- return True
- tool_names = {t["name"] for t in tools}
- return all(a["name"] in tool_names for a in answers)
-
-
-def _answer_uses_valid_params(ex):
- """Check that arguments in each call are a subset of the tool's schema params."""
- try:
- answers = json.loads(ex["answers"])
- tools = json.loads(ex["tools"])
- except (json.JSONDecodeError, TypeError):
- return True
- if not answers:
- return True
- tool_map = {t["name"]: set((t.get("parameters") or {}).keys()) for t in tools}
- for a in answers:
- schema_params = tool_map.get(a["name"])
- if schema_params is None:
- continue # caught by _answer_calls_valid_tools
- args = a.get("arguments", {})
- if isinstance(args, str):
- try:
- args = json.loads(args)
- except (json.JSONDecodeError, TypeError):
- continue
- if not isinstance(args, dict):
- continue
- if args and not set(args.keys()).issubset(schema_params):
- return False
- return True
-
-
-
-def _deduplicate(dataset):
- """Remove duplicates by (query, tools) pair, keeping the first occurrence.
-
- Same query with different tool sets is kept — those are distinct examples.
- Same query with same tools but different answers (e.g. from different LLMs)
- is deduplicated since the model only needs one answer per (query, tools) pair.
- """
- queries = dataset["query"]
- tools = dataset["tools"]
- seen = set()
- keep = []
- for i in range(len(queries)):
- key = (queries[i], tools[i])
- if key not in seen:
- seen.add(key)
- keep.append(i)
- if len(keep) == len(dataset):
- return dataset
- return dataset.select(keep)
-
-
-def _trim_tools(dataset, max_tools=5, seed=42):
- """Trim tool lists to at most max_tools, keeping called tools.
-
- For samples with >max_tools available tools, keeps all tools referenced
- in the answers plus randomly sampled distractors up to max_tools.
- Samples where the number of called tools alone exceeds max_tools are dropped.
- """
- rng = random.Random(seed)
- rows = []
- dropped = 0
- for i in range(len(dataset)):
- ex = dataset[i]
- try:
- tools = json.loads(ex["tools"])
- except (json.JSONDecodeError, TypeError):
- tools = []
- if len(tools) <= max_tools:
- rows.append(ex)
- continue
- try:
- answers = json.loads(ex["answers"])
- except (json.JSONDecodeError, TypeError):
- answers = []
- called_names = {a["name"] for a in answers if isinstance(a, dict)}
- called = [t for t in tools if t.get("name") in called_names]
- if len(called) > max_tools:
- dropped += 1
- continue
- distractors = [t for t in tools if t.get("name") not in called_names]
- n_distractors = max_tools - len(called)
- sampled = rng.sample(distractors, min(n_distractors, len(distractors)))
- trimmed = called + sampled
- rng.shuffle(trimmed)
- rows.append({
- "query": ex["query"],
- "answers": ex["answers"],
- "tools": json.dumps(trimmed),
- "source": ex["source"],
- })
- trimmed = len(dataset) - len(rows) - dropped
- print(f"\nTool trimming (max {max_tools}): "
- f"trimmed {trimmed + dropped} examples, dropped {dropped}")
- return Dataset.from_list(rows)
-
-
-def _normalize_tool_names(dataset):
- """Normalize all tool names to snake_case in both tools and answers fields."""
- from src.data import to_snake_case
-
- rows = []
- for i in range(len(dataset)):
- ex = dataset[i]
- try:
- tools = json.loads(ex["tools"])
- except (json.JSONDecodeError, TypeError):
- rows.append(ex)
- continue
- try:
- answers = json.loads(ex["answers"])
- except (json.JSONDecodeError, TypeError):
- answers = []
-
- for t in tools:
- if isinstance(t, dict) and "name" in t:
- t["name"] = to_snake_case(t["name"])
- for a in answers:
- if isinstance(a, dict) and "name" in a:
- a["name"] = to_snake_case(a["name"])
-
- rows.append({
- "query": ex["query"],
- "answers": json.dumps(answers),
- "tools": json.dumps(tools),
- "source": ex["source"],
- })
- return Dataset.from_list(rows)
-
-
-def _augment_irrelevance(dataset, ratio=0.1, seed=42):
- """Create irrelevance samples by swapping tools with unrelated queries.
-
- For a fraction of examples with tool calls, replace the available tools
- with tools from a different example and set the answer to [] (no calls).
- Ensures no overlap between the query's needed tools and the donor tools.
- """
- rng = random.Random(seed)
-
- all_answers = dataset["answers"]
- all_tools = dataset["tools"]
- all_queries = dataset["query"]
-
- with_calls = []
- call_names = {}
- tool_names_by_idx = {}
-
- for i, ans_str in enumerate(all_answers):
- try:
- answers = json.loads(ans_str)
- except (json.JSONDecodeError, TypeError):
- continue
- if not answers:
- continue
- with_calls.append(i)
- call_names[i] = {c["name"] for c in answers if isinstance(c, dict)}
- try:
- tools = json.loads(all_tools[i])
- tool_names_by_idx[i] = {t["name"] for t in tools if isinstance(t, dict)}
- except (json.JSONDecodeError, TypeError):
- tool_names_by_idx[i] = set()
-
- n_aug = int(len(with_calls) * ratio)
- selected = rng.sample(with_calls, min(n_aug, len(with_calls)))
-
- rows = []
- for idx in selected:
- orig_call_names = call_names[idx]
- for _ in range(5):
- donor = rng.choice(with_calls)
- if donor == idx:
- continue
- # Ensure none of the query's needed tools appear in donor's tool set
- if not (orig_call_names & tool_names_by_idx.get(donor, set())):
- rows.append({
- "query": all_queries[idx],
- "answers": "[]",
- "tools": all_tools[donor],
- "source": "irrelevance-aug",
- })
- break
-
- if rows:
- print(f"\nIrrelevance augmentation: {len(rows)} samples "
- f"({len(rows)/len(dataset)*100:.1f}% of dataset)")
- return concatenate_datasets([dataset, Dataset.from_list(rows)])
- return dataset
-
-
-def load_and_combine():
- parts = []
- for name, hf_id in DATASETS.items():
- split_override = None
- if isinstance(hf_id, tuple):
- if len(hf_id) == 3:
- repo, subset, split_override = hf_id
- else:
- repo, subset = hf_id
- label = f"{repo}" + (f" [{subset}]" if subset else "")
- else:
- repo, subset = hf_id, None
- label = repo
- print(f"\nLoading {name} ({label})...")
- try:
- if split_override:
- raw = load_dataset(repo, subset, split=split_override) if subset else load_dataset(repo, split=split_override)
- else:
- ds = load_dataset(repo, subset) if subset else load_dataset(repo)
- split = "train" if "train" in ds else list(ds.keys())[0]
- raw = ds[split]
- except Exception as e:
- print(f" ERROR: {e}")
- continue
-
- print(f" Converting to xlam format ({len(raw):,} rows)...")
- converted = CONVERTERS[name](raw)
- converted = converted.add_column("source", [name] * len(converted))
-
- before = len(converted)
- converted = converted.filter(
- lambda ex: (
- not _has_placeholder_args(ex["answers"])
- and _answer_calls_valid_tools(ex)
- and _answer_uses_valid_params(ex)
- ),
- desc=f"Filtering {name}",
- num_proc=_NUM_WORKERS,
- )
- dropped = before - len(converted)
- if dropped:
- print(f" Filtered out {dropped} low-quality examples ({dropped*100/before:.1f}%)")
-
- print(f" -> {len(converted)} examples")
- parts.append(converted)
-
- combined = concatenate_datasets(parts)
-
- # Drop no-call examples with empty tools (trivial negatives)
- before_empty = len(combined)
- combined = combined.filter(
- lambda ex: json.loads(ex["answers"]) != [] or json.loads(ex["tools"]) != [],
- desc="Removing empty-tools no-call examples",
- num_proc=_NUM_WORKERS,
- )
- empty_dropped = before_empty - len(combined)
- if empty_dropped:
- print(f"\nRemoved {empty_dropped} no-call examples with empty tools")
-
- before_dedup = len(combined)
- combined = _deduplicate(combined)
- dedup_dropped = before_dedup - len(combined)
- if dedup_dropped:
- print(f"\nDeduplicated: removed {dedup_dropped} duplicate queries")
-
- before_trim = len(combined)
- combined = _trim_tools(combined, max_tools=5, seed=42)
- print(f" {before_trim} -> {len(combined)} after tool trimming")
-
- combined = _normalize_tool_names(combined)
- print(f" Normalized tool names to snake_case")
-
- combined = _augment_irrelevance(combined, ratio=0.1, seed=42)
-
- empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])
- print(f"\n{'='*60}")
- print(f"Combined dataset: {len(combined)} examples ({empty_count} with empty answers = no-call examples)")
- print(f"Columns: {combined.column_names}")
-
- counts = Counter(combined["source"])
- print(f"\nBreakdown:")
- for src, cnt in counts.items():
- print(f" {src}: {cnt}")
-
- seen = set()
- for i in range(len(combined)):
- src = combined[i]["source"]
- if src in seen:
- continue
- seen.add(src)
- print(f"\n--- Sample from {src} ---")
- for k in ["query", "answers", "tools"]:
- v = str(combined[i][k])[:200]
- print(f" {k}: {v}")
-
- return combined
-
-
-if __name__ == "__main__":
- combined = load_and_combine()
- out_path = "data/tool_calls"
- combined.save_to_disk(out_path)
- print(f"\nSaved to {out_path}/")
-
- from datasets import load_from_disk
- reloaded = load_from_disk(out_path)
- assert len(reloaded) == len(combined), f"Row count mismatch: {len(reloaded)} != {len(combined)}"
- assert reloaded.column_names == combined.column_names, "Column mismatch"
- assert reloaded[0] == combined[0], "First row mismatch"
- print(f"Verified: {len(reloaded)} rows, columns {reloaded.column_names}")
diff --git a/src/data.py b/src/data.py
index b9aa2fe..8ce413d 100644
--- a/src/data.py
+++ b/src/data.py
@@ -234,19 +234,21 @@ def _load_unified_dataset():
continue
try:
- from .gcs import download_synth_data
+ from datasets import load_dataset as _hf_load
+ print("Downloading dataset from HuggingFace (Cactus-Compute/tool-calls)...")
+ ds = _hf_load("Cactus-Compute/tool-calls", split="train", token=True)
target = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR
- if download_synth_data(target):
- ds = load_from_disk(target)
- print(f"Loaded synth dataset from GCS -> {target} ({len(ds)} rows)")
- _unified_dataset_cache = _set_audio_backend(ds)
- return _unified_dataset_cache
+ os.makedirs(target, exist_ok=True)
+ ds.save_to_disk(target)
+ print(f"Loaded from HuggingFace -> {target} ({len(ds)} rows)")
+ _unified_dataset_cache = _set_audio_backend(ds)
+ return _unified_dataset_cache
except Exception as e:
- print(f"Warning: GCS download failed: {e}")
+ print(f"Warning: HuggingFace download failed: {e}")
raise FileNotFoundError(
f"Dataset not found at {_SHM_UNIFIED_DIR} or {_DISK_UNIFIED_DIR}. "
- f"Run 'needle tokenize' or 'python scripts/synthesize_tools_data.py' first."
+ f"Run 'needle tokenize' or 'python scripts/push_to_hf.py' first."
)
@@ -371,12 +373,50 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
return model_path
+_HF_TOKENIZED_REPO = "Cactus-Compute/tool-calls-tokenized"
+
+
+def _download_tokenized_from_hf():
+ """Download tokenized .npy files from HuggingFace Hub into CACHE_DIR."""
+ from huggingface_hub import snapshot_download
+
+ local = snapshot_download(
+ _HF_TOKENIZED_REPO,
+ repo_type="dataset",
+ allow_patterns="tokenized_data/*",
+ token=True,
+ )
+ src = os.path.join(local, "tokenized_data")
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ for fname in os.listdir(src):
+ dst = os.path.join(CACHE_DIR, fname)
+ if not os.path.exists(dst):
+ os.symlink(os.path.join(src, fname), dst)
+
+
+def _download_tokenizer_from_hf():
+ """Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""
+ from huggingface_hub import snapshot_download
+
+ local = snapshot_download(
+ _HF_TOKENIZED_REPO,
+ repo_type="dataset",
+ allow_patterns="tokenizer/*",
+ token=True,
+ )
+ src = os.path.join(local, "tokenizer")
+ os.makedirs(TOKENIZER_DIR, exist_ok=True)
+ for fname in os.listdir(src):
+ dst = os.path.join(TOKENIZER_DIR, fname)
+ if not os.path.exists(dst):
+ os.symlink(os.path.join(src, fname), dst)
+
+
def get_tokenizer(max_samples=None):
model_path = TOKENIZER_PREFIX + ".model"
if not os.path.exists(model_path):
try:
- from .gcs import download_tokenizer
- download_tokenizer(TOKENIZER_DIR)
+ _download_tokenizer_from_hf()
except Exception:
pass
if not os.path.exists(model_path):
@@ -648,7 +688,6 @@ def _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,
query_arr[i, :l] = toks[:l]
np.save(cache_path + "_query_only.npy", query_arr)
- # Individual tool tokens + mapping arrays
tool_texts_individual = []
tool_ex_idx = []
tool_is_pos = []
@@ -932,8 +971,7 @@ def load_prepared_data(split, mmap=False):
meta = _load_cache_metadata(split)
if meta is None:
try:
- from .gcs import download_tokenized_data
- download_tokenized_data(CACHE_DIR)
+ _download_tokenized_from_hf()
meta = _load_cache_metadata(split)
except Exception:
pass
diff --git a/src/export.py b/src/export.py
index 8b5d00e..edd4cc5 100644
--- a/src/export.py
+++ b/src/export.py
@@ -38,7 +38,7 @@ def export_submodel(checkpoint_path, factor, output_path):
def slice_leaf(key_path, leaf):
arr = np.asarray(leaf)
- if arr.ndim != 2:
+ if arr.ndim not in (2, 3):
return arr
parent_name = None
@@ -51,13 +51,22 @@ def export_submodel(checkpoint_path, factor, output_path):
if parent_name is None:
return arr
- rows, cols = arr.shape
- if parent_name in ("gate_proj", "up_proj"):
- if cols == d_ff:
- return arr[:, :d_ff_new]
- elif parent_name == "down_proj":
- if rows == d_ff:
- return arr[:d_ff_new, :]
+ if arr.ndim == 2:
+ rows, cols = arr.shape
+ if parent_name in ("gate_proj", "up_proj"):
+ if cols == d_ff:
+ return arr[:, :d_ff_new]
+ elif parent_name == "down_proj":
+ if rows == d_ff:
+ return arr[:d_ff_new, :]
+ elif arr.ndim == 3:
+ _, rows, cols = arr.shape
+ if parent_name in ("gate_proj", "up_proj"):
+ if cols == d_ff:
+ return arr[:, :, :d_ff_new]
+ elif parent_name == "down_proj":
+ if rows == d_ff:
+ return arr[:, :d_ff_new, :]
return arr
diff --git a/src/model.py b/src/model.py
index 5c94aa7..81b3134 100644
--- a/src/model.py
+++ b/src/model.py
@@ -255,6 +255,30 @@ class EncoderBlock(nn.Module):
return x
+class _EncoderScanBody(nn.Module):
+ """Wraps EncoderBlock for nn.scan: carry = (x, mask, rope, ffn_mask)."""
+ num_heads: int
+ num_kv_heads: int
+ d_model: int
+ d_ff: int
+ num_layers: int
+ dtype: jnp.dtype = jnp.bfloat16
+ activation: str = "drelu"
+ dropout_rate: float = 0.0
+ conv_kernel_size: int = 0
+ deterministic: bool = True
+
+ @nn.compact
+ def __call__(self, carry, _):
+ x, mask, rope, ffn_mask = carry
+ x = EncoderBlock(
+ self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,
+ self.num_layers, self.dtype, self.activation, self.dropout_rate,
+ self.conv_kernel_size,
+ )(x, mask, rope, ffn_mask, self.deterministic)
+ return (x, mask, rope, ffn_mask), None
+
+
class Encoder(nn.Module):
"""Self-attention encoder. Returns (output, mask) tuple."""
config: TransformerConfig
@@ -265,12 +289,17 @@ class Encoder(nn.Module):
dt = cfg.jax_dtype
x = x.astype(dt)
- for i in range(cfg.num_encoder_layers):
- x = nn.remat(EncoderBlock, static_argnums=(5,))(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
- cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,
- cfg.conv_kernel_size, name=f"block_{i}"
- )(x, mask, rope, ffn_mask, deterministic)
+ ScanBlock = nn.scan(
+ nn.remat(_EncoderScanBody),
+ variable_axes={"params": 0},
+ split_rngs={"params": True, "dropout": True},
+ length=cfg.num_encoder_layers,
+ )
+ (x, _, _, _), _ = ScanBlock(
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
+ cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,
+ cfg.conv_kernel_size, deterministic, name="layers",
+ )((x, mask, rope, ffn_mask), None)
x = ZCRMSNorm(dtype=dt, name="final_norm")(x)
return x, mask
@@ -311,6 +340,28 @@ class DecoderBlock(nn.Module):
+class _DecoderScanBody(nn.Module):
+ """Wraps DecoderBlock for nn.scan: carry = (x, encoder_out, self_mask, cross_mask, rope, ffn_mask)."""
+ num_heads: int
+ num_kv_heads: int
+ d_model: int
+ d_ff: int
+ num_layers: int
+ dtype: jnp.dtype = jnp.bfloat16
+ activation: str = "drelu"
+ dropout_rate: float = 0.0
+ deterministic: bool = True
+
+ @nn.compact
+ def __call__(self, carry, _):
+ x, encoder_out, self_mask, cross_mask, rope, ffn_mask = carry
+ x = DecoderBlock(
+ self.num_heads, self.num_kv_heads, self.d_model, self.d_ff,
+ self.num_layers, self.dtype, self.activation, self.dropout_rate,
+ )(x, encoder_out, self_mask, cross_mask, rope, ffn_mask, self.deterministic)
+ return (x, encoder_out, self_mask, cross_mask, rope, ffn_mask), None
+
+
class Decoder(nn.Module):
config: TransformerConfig
@@ -320,10 +371,17 @@ class Decoder(nn.Module):
dt = cfg.jax_dtype
x = x.astype(dt)
- for i in range(cfg.num_decoder_layers):
- x = nn.remat(DecoderBlock, static_argnums=(7,))(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, cfg.dropout_rate, name=f"block_{i}"
- )(x, encoder_out, self_mask, cross_mask, rope, ffn_mask, deterministic)
+ ScanBlock = nn.scan(
+ nn.remat(_DecoderScanBody),
+ variable_axes={"params": 0},
+ split_rngs={"params": True, "dropout": True},
+ length=cfg.num_decoder_layers,
+ )
+ (x, _, _, _, _, _), _ = ScanBlock(
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
+ cfg.total_layers, dt, cfg.activation, cfg.dropout_rate,
+ deterministic, name="layers",
+ )((x, encoder_out, self_mask, cross_mask, rope, ffn_mask), None)
x = ZCRMSNorm(dtype=dt)(x)
return x
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index f6214ec..cc673ec 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -1,8 +1,8 @@
"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.
-Downloads the synthesized tool-calling dataset from GCS
-(gs://cactus-dataset/synth_tool_calls/), trains the SentencePiece tokenizer,
-and tokenizes train + val splits, caching everything locally.
+Downloads the tool-calling dataset from HuggingFace (Cactus-Compute/tool-calls),
+trains the SentencePiece tokenizer, and tokenizes train + val splits,
+caching everything locally.
Usage:
needle tokenize # full run
@@ -29,6 +29,36 @@ from .data import (
)
+_HF_TOKENIZED_REPO = "Cactus-Compute/tool-calls-tokenized"
+
+
+def _push_to_hf(cache_dir, tokenizer_dir):
+ """Upload tokenized data and tokenizer to HuggingFace Hub."""
+ from huggingface_hub import HfApi
+
+ api = HfApi()
+ api.create_repo(_HF_TOKENIZED_REPO, repo_type="dataset", private=True, exist_ok=True)
+
+ print(f"Uploading tokenized data to {_HF_TOKENIZED_REPO} ...")
+ api.upload_folder(
+ folder_path=cache_dir,
+ repo_id=_HF_TOKENIZED_REPO,
+ repo_type="dataset",
+ path_in_repo="tokenized_data",
+ allow_patterns=["*.npy", "*.json"],
+ )
+
+ print(f"Uploading tokenizer to {_HF_TOKENIZED_REPO} ...")
+ api.upload_folder(
+ folder_path=tokenizer_dir,
+ repo_id=_HF_TOKENIZED_REPO,
+ repo_type="dataset",
+ path_in_repo="tokenizer",
+ allow_patterns=["*.model", "*.vocab"],
+ )
+ print("HuggingFace upload complete.")
+
+
def _clear_local_caches():
"""Remove local .data_cache/, tokenizer/, and stale unified dataset directories."""
for d in [CACHE_DIR, TOKENIZER_DIR, _DISK_UNIFIED_DIR, _SHM_UNIFIED_DIR]:
@@ -38,23 +68,29 @@ def _clear_local_caches():
def _download_synth_dataset():
- """Download synthesized tool-calling dataset from GCS."""
- from .gcs import download_synth_data
+ """Download synthesized tool-calling dataset from HuggingFace."""
+ from datasets import load_dataset
target = _SHM_UNIFIED_DIR if _shm_available() else _DISK_UNIFIED_DIR
- print(f"Downloading synth dataset to {target} ...")
- if not download_synth_data(target):
+ print(f"Downloading dataset from HuggingFace (Cactus-Compute/tool-calls)...")
+ try:
+ ds = load_dataset("Cactus-Compute/tool-calls", split="train", token=True)
+ except Exception as e:
raise FileNotFoundError(
- "Synth dataset not found at gs://cactus-dataset/synth_tool_calls/. "
- "Run 'python scripts/synthesize_tools_data.py' first."
+ f"Dataset not found on HuggingFace (Cactus-Compute/tool-calls): {e}\n"
+ "Run 'python scripts/push_to_hf.py' first."
)
+ import os
+ os.makedirs(target, exist_ok=True)
+ ds.save_to_disk(target)
+ print(f"Saved {len(ds):,} rows to {target}")
def tokenize(args):
print("=== Clearing existing caches ===")
_clear_local_caches()
- print("\n=== Downloading synth dataset from GCS ===")
+ print("\n=== Downloading dataset from HuggingFace ===")
_download_synth_dataset()
print("\n=== Training tokenizer ===")
@@ -89,8 +125,6 @@ def tokenize(args):
_save_cache_metadata(split, text_cache_id, len(kept_indices),
max_enc_len, max_dec_len)
- from .gcs import upload_tokenized_data, upload_tokenizer
- upload_tokenizer(TOKENIZER_DIR)
- upload_tokenized_data(CACHE_DIR)
+ _push_to_hf(CACHE_DIR, TOKENIZER_DIR)
print("\n=== Tokenization pipeline complete ===")
diff --git a/src/train.py b/src/train.py
index fc88861..f49d9de 100644
--- a/src/train.py
+++ b/src/train.py
@@ -57,6 +57,8 @@ def scale_by_muon(momentum=0.95, ns_steps=5):
del params
def ortho(g):
+ if g.ndim == 3:
+ return jax.vmap(_newton_schulz, in_axes=(0,))(g)
if g.ndim == 2:
return _newton_schulz(g, steps=ns_steps)
return g
@@ -76,7 +78,7 @@ def _param_labels(params):
def _label(path, leaf):
name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel" and leaf.ndim == 2:
+ if name == "kernel" and leaf.ndim in (2, 3):
return "muon"
return "adam"
@@ -180,17 +182,25 @@ def _make_prune_mask(params, sparsity, group_size):
"""Compute block-prune mask entirely on-device (no numpy round-trips).
Returns binary mask tree matching param shapes and dtypes.
+ Handles both 2D (in, out) and 3D (num_layers, in, out) kernels from nn.scan.
"""
all_scores = []
for _, leaf in jax.tree_util.tree_leaves_with_path(params):
- if leaf.ndim != 2:
- continue
- in_feat, out_feat = leaf.shape
- gs = min(group_size, in_feat)
- pad = (gs - in_feat % gs) % gs
- w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf
- scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()
- all_scores.append(scores)
+ if leaf.ndim == 2:
+ in_feat, out_feat = leaf.shape
+ gs = min(group_size, in_feat)
+ pad = (gs - in_feat % gs) % gs
+ w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf
+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()
+ all_scores.append(scores)
+ elif leaf.ndim == 3:
+ num_layers, in_feat, out_feat = leaf.shape
+ gs = min(group_size, in_feat)
+ pad = (gs - in_feat % gs) % gs
+ for li in range(num_layers):
+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]
+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()
+ all_scores.append(scores)
if not all_scores:
return jax.tree.map(jnp.ones_like, params)
@@ -198,6 +208,18 @@ def _make_prune_mask(params, sparsity, group_size):
threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)
def _leaf_mask(path, leaf):
+ if leaf.ndim == 3:
+ num_layers, in_feat, out_feat = leaf.shape
+ gs = min(group_size, in_feat)
+ pad = (gs - in_feat % gs) % gs
+ masks = []
+ for li in range(num_layers):
+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]
+ w_grouped = w.reshape(-1, gs, out_feat)
+ block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)
+ m = jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat]
+ masks.append(m)
+ return jnp.stack(masks, axis=0).astype(leaf.dtype)
if leaf.ndim != 2:
return jnp.ones_like(leaf)
in_feat, out_feat = leaf.shape
@@ -215,6 +237,8 @@ def _quantize_params(params, group_size=32):
"""Fake-quantize all Dense kernels in the param tree."""
def _maybe_quantize(path, leaf):
name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
+ if name == "kernel" and leaf.ndim == 3:
+ return jax.vmap(_fake_quantize_int4, in_axes=(0,))(leaf)
if name == "kernel" and leaf.ndim == 2:
return _fake_quantize_int4(leaf, group_size=group_size)
return leaf
@@ -239,9 +263,14 @@ def _clip_contrastive_loss(q_emb, t_emb, log_temp):
return (jnp.mean(loss_q) + jnp.mean(loss_t)) / 2.0
-def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):
+def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, do_quantize):
pad_id = 0
- q_params = _quantize_params(params, group_size=_GROUP_SIZE)
+ q_params = jax.lax.cond(
+ do_quantize,
+ lambda p: _quantize_params(p, group_size=_GROUP_SIZE),
+ lambda p: p,
+ params,
+ )
src_mask = make_padding_mask(src, pad_id)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
logits, slot_div = state.apply_fn(
@@ -261,9 +290,14 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rn
return ce_loss + z_loss
-def _contrastive_loss_fn(state, params, query_tokens, tool_tokens, rng):
+def _contrastive_loss_fn(state, params, query_tokens, tool_tokens, rng, do_quantize):
"""Compute CLIP contrastive loss on query/tool pairs."""
- q_params = _quantize_params(params, group_size=_GROUP_SIZE)
+ q_params = jax.lax.cond(
+ do_quantize,
+ lambda p: _quantize_params(p, group_size=_GROUP_SIZE),
+ lambda p: p,
+ params,
+ )
q_emb, t_emb, log_temp = state.apply_fn(
{"params": q_params},
query_tokens, tool_tokens,
@@ -293,87 +327,36 @@ def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):
return jnp.concatenate(rows, axis=0)
-def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):
- ema_decay = 0.999
- loss, grads = jax.value_and_grad(
- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
- )(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- loss = jax.lax.pmean(loss, axis_name="batch")
- grad_norm = optax.global_norm(grads)
- state = state.apply_gradients(grads=grads)
- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)
- return state, ema_params, loss, grad_norm
-
-
-def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask):
- """Text training step with fused prune mask application."""
+def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng, do_quantize, do_contrastive, do_prune):
+ """Unified train step with boolean flags for contrastive loss and pruning."""
ema_decay = 0.999
- loss, grads = jax.value_and_grad(
- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
- )(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- loss = jax.lax.pmean(loss, axis_name="batch")
- grad_norm = optax.global_norm(grads)
- state = state.apply_gradients(grads=grads)
- masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)
- state = state.replace(params=masked_params)
- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)
- return state, ema_params, loss, grad_norm
-
-def _train_step_fused(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng):
- """Fused text + contrastive training step — single backward pass & optimizer step."""
- ema_decay = 0.999
def combined_loss(p):
- text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
- cl_loss = _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng)
+ text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, do_quantize)
+ cl_loss = jax.lax.cond(
+ do_contrastive,
+ lambda: _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng, do_quantize),
+ lambda: 0.0,
+ )
return text_loss + _CONTRASTIVE_WEIGHT * cl_loss, text_loss
- (loss, text_loss), grads = jax.value_and_grad(combined_loss, has_aux=True)(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- text_loss = jax.lax.pmean(text_loss, axis_name="batch")
- grad_norm = optax.global_norm(grads)
- state = state.apply_gradients(grads=grads)
- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)
- return state, ema_params, text_loss, grad_norm
-
-def _train_step_fused_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng):
- """Fused text + contrastive training step with prune mask application."""
- ema_decay = 0.999
-
- def combined_loss(p):
- text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
- cl_loss = _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng)
- return text_loss + _CONTRASTIVE_WEIGHT * cl_loss, text_loss
-
(loss, text_loss), grads = jax.value_and_grad(combined_loss, has_aux=True)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
text_loss = jax.lax.pmean(text_loss, axis_name="batch")
grad_norm = optax.global_norm(grads)
state = state.apply_gradients(grads=grads)
- masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)
- state = state.replace(params=masked_params)
- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)
+ new_params = jax.lax.cond(
+ do_prune,
+ lambda: jax.tree.map(lambda w, m: w * m, state.params, prune_mask),
+ lambda: state.params,
+ )
+ state = state.replace(params=new_params)
+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, new_params)
return state, ema_params, text_loss, grad_norm
-
-
def _make_p_train_step():
- return jax.pmap(_train_step_text, axis_name="batch", donate_argnums=(0, 1))
-
-
-def _make_p_train_step_masked():
- return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))
-
-
-def _make_p_train_step_fused():
- return jax.pmap(_train_step_fused, axis_name="batch", donate_argnums=(0, 1))
-
-
-def _make_p_train_step_fused_masked():
- return jax.pmap(_train_step_fused_masked, axis_name="batch", donate_argnums=(0, 1))
+ return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0, 1))
def _make_val_loss_fn(apply_fn):
@@ -521,9 +504,6 @@ def train(args):
_MAT_FF_WIDTHS = ()
n_widths = 1 + len(_MAT_FF_WIDTHS) if _MAT_FF_WIDTHS else 1
p_train_step = _make_p_train_step()
- p_train_step_masked = _make_p_train_step_masked()
- p_train_step_fused = _make_p_train_step_fused()
- p_train_step_fused_masked = _make_p_train_step_fused_masked()
np.random.seed(args.seed)
rng = jax.random.PRNGKey(args.seed)
@@ -598,6 +578,13 @@ def train(args):
sparsity_ratio = getattr(args, "sparsity_ratio", 0.0)
prune_mask = None
gradual_sparsify_done = False
+ # Dummy all-ones prune mask (replicated) — used when pruning is inactive
+ _unrep_params = jax_utils.unreplicate(state).params
+ dummy_prune_mask = jax_utils.replicate(jax.tree.map(jnp.ones_like, _unrep_params))
+ del _unrep_params
+ # Dummy contrastive arrays — used when contrastive is inactive
+ dummy_cl_tokens = jnp.zeros((num_devices, args.batch_size, 128), dtype=jnp.int32)
+ dummy_cl_rng = jax.random.split(jax.random.PRNGKey(0), num_devices)
prune_interval = getattr(args, "prune_interval", 100)
prune_start_frac = getattr(args, "prune_start_frac", 0.33)
@@ -647,36 +634,30 @@ def train(args):
rng, text_rng = jax.random.split(rng)
text_rngs = jax.random.split(text_rng, num_devices)
+ do_qat = jnp.broadcast_to(jnp.array(global_step % 100 == 0), (num_devices,))
+
do_contrastive = cl_batch_iter is not None and global_step % 100 == 0
+ cl_q_b, cl_t_b, cl_rngs = dummy_cl_tokens, dummy_cl_tokens, dummy_cl_rng
if do_contrastive:
try:
cl_q, cl_t = next(cl_batch_iter)
cl_q_b = shard_batch(cl_q, num_devices)
cl_t_b = shard_batch(cl_t, num_devices)
+ rng, cl_rng = jax.random.split(rng)
+ cl_rngs = jax.random.split(cl_rng, num_devices)
except StopIteration:
cl_batch_iter = None
do_contrastive = False
- if do_contrastive:
- rng, cl_rng = jax.random.split(rng)
- cl_rngs = jax.random.split(cl_rng, num_devices)
- if prune_mask is not None:
- state, ema_params, loss, grad_norm = p_train_step_fused_masked(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b, cl_q_b, cl_t_b, cl_rngs,
- )
- else:
- state, ema_params, loss, grad_norm = p_train_step_fused(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b, cl_q_b, cl_t_b, cl_rngs,
- )
- else:
- if prune_mask is not None:
- state, ema_params, loss, grad_norm = p_train_step_masked(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,
- )
- else:
- state, ema_params, loss, grad_norm = p_train_step(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,
- )
+ do_cl = jnp.broadcast_to(jnp.array(do_contrastive), (num_devices,))
+ do_prune = jnp.broadcast_to(jnp.array(prune_mask is not None), (num_devices,))
+ cur_prune_mask = prune_mask if prune_mask is not None else dummy_prune_mask
+
+ state, ema_params, loss, grad_norm = p_train_step(
+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask,
+ cur_prune_mask, text_ffn_mask, text_rngs, lm_b,
+ cl_q_b, cl_t_b, cl_rngs, do_qat, do_cl, do_prune,
+ )
text_loss_val = float(loss[0])
text_losses.append(text_loss_val)