Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Add toucan upload
justinl66c4106f3a2bb2
diff --git a/dataset-scripts/toucan/preview_toucan.py b/dataset-scripts/toucan/preview_toucan.pynew file mode 100644index 0000000..14fc984--- /dev/null+++ b/dataset-scripts/toucan/preview_toucan.py@@ -0,0 +1,109 @@+import ast+import json+import uuid+from datasets import load_dataset+from langdetect import detect, DetectorFactory++DetectorFactory.seed = 0++NUM_EXAMPLES = 1000+OUT_FILE = "toucan_preview.jsonl"+++def strip_titles(obj):+ if isinstance(obj, dict):+ return {k: strip_titles(v) for k, v in obj.items() if k != "title"}+ if isinstance(obj, list):+ return [strip_titles(v) for v in obj]+ return obj+++def build_tool_subset(available_tools, target_names):+ by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}+ subset = [by_name[n] for n in target_names if n in by_name]+ for tool in available_tools:+ if len(subset) >= 5:+ break+ if tool.get("function", {}).get("name") not in target_names:+ subset.append(tool)+ return [strip_titles(t) for t in subset[:5]]+++def parse_tool_call(m):+ try:+ fc = ast.literal_eval(m["content"])+ except Exception:+ return None+ try:+ args = json.loads(fc.get("arguments", "{}"))+ except Exception:+ args = fc.get("arguments", {})+ return {+ "id": f"call_1",+ "type": "function",+ "function": {+ "name": fc.get("name", ""),+ "arguments": args,+ },+ }+++def convert(row):+ target_names = [n.strip() for n in row["target_tools"].split(",")]+ if len(target_names) > 3:+ return []++ try:+ if detect(row["question"]) != "en":+ return []+ except Exception:+ return []++ available_tools = json.loads(row["tools"])+ tool_subset = build_tool_subset(available_tools, target_names)++ messages = json.loads(row["messages"])+ query = next((m["content"] for m in messages if m.get("role") == "user"), None)+ if query is None:+ return []++ records = []+ current_calls = []+ for m in messages:+ role = m.get("role")+ if role == "tool_call":+ tc = parse_tool_call(m)+ if tc:+ tc["id"] = f"call_{len(current_calls) + 1}"+ current_calls.append(tc)+ elif role in ("tool_response", "assistant") and current_calls:+ records.append({+ "id": str(uuid.uuid4()),+ "messages": [+ {"role": "user", "content": query},+ {"role": "assistant", "tool_calls": current_calls},+ ],+ "tools": tool_subset,+ })+ current_calls = []++ return records+++ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train", streaming=True)++count = 0+with open(OUT_FILE, "w", encoding="utf-8") as f:+ for row in ds:+ if count >= NUM_EXAMPLES:+ break+ for record in convert(row):+ if count >= NUM_EXAMPLES:+ break+ f.write(json.dumps(record, ensure_ascii=False) + "\n")+ print(f"--- Example {count} ---")+ print(json.dumps(record, indent=2))+ print()+ count += 1++print(f"Saved {count} examples to {OUT_FILE}")diff --git a/dataset-scripts/toucan/upload_toucan.py b/dataset-scripts/toucan/upload_toucan.pynew file mode 100644index 0000000..6cbc0c2--- /dev/null+++ b/dataset-scripts/toucan/upload_toucan.py@@ -0,0 +1,161 @@+import ast+import datetime+import json+import math+import os+import subprocess+import uuid+from datasets import load_dataset+from langdetect import detect, DetectorFactory++DetectorFactory.seed = 0++BUCKET_PATH = "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1"+OUT_DIR = "toucan_openai_jsonl/train"+SHARD_SIZE = 2000+++def strip_titles(obj):+ if isinstance(obj, dict):+ return {k: strip_titles(v) for k, v in obj.items() if k != "title"}+ if isinstance(obj, list):+ return [strip_titles(v) for v in obj]+ return obj+++def build_tool_subset(available_tools, target_names):+ by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}+ subset = [by_name[n] for n in target_names if n in by_name]+ for tool in available_tools:+ if len(subset) >= 5:+ break+ if tool.get("function", {}).get("name") not in target_names:+ subset.append(tool)+ return [strip_titles(t) for t in subset[:5]]+++def parse_tool_call(m):+ try:+ fc = ast.literal_eval(m["content"])+ except Exception:+ return None+ try:+ args = json.loads(fc.get("arguments", "{}"))+ except Exception:+ args = fc.get("arguments", {})+ return {+ "id": "call_1",+ "type": "function",+ "function": {+ "name": fc.get("name", ""),+ "arguments": args,+ },+ }+++def convert(row):+ target_names = [n.strip() for n in row["target_tools"].split(",")]+ if len(target_names) > 3:+ return []++ try:+ if detect(row["question"]) != "en":+ return []+ except Exception:+ return []++ available_tools = json.loads(row["tools"])+ tool_subset = build_tool_subset(available_tools, target_names)++ messages = json.loads(row["messages"])+ query = next((m["content"] for m in messages if m.get("role") == "user"), None)+ if query is None:+ return []++ records = []+ current_calls = []+ for m in messages:+ role = m.get("role")+ if role == "tool_call":+ tc = parse_tool_call(m)+ if tc:+ tc["id"] = f"call_{len(current_calls) + 1}"+ current_calls.append(tc)+ elif role in ("tool_response", "assistant") and current_calls:+ records.append({+ "id": str(uuid.uuid4()),+ "messages": [+ {"role": "user", "content": query},+ {"role": "assistant", "tool_calls": current_calls},+ ],+ "tools": tool_subset,+ })+ current_calls = []++ return records+++def convert_and_shard(ds):+ os.makedirs(OUT_DIR, exist_ok=True)+ records = []+ skipped = 0+ for row in ds:+ converted = convert(row)+ if not converted:+ skipped += 1+ continue+ records.extend(converted)++ print(f" {len(records)} records from {len(ds) - skipped} source rows, {skipped} skipped")++ num_shards = math.ceil(len(records) / SHARD_SIZE)+ for s in range(num_shards):+ start = s * SHARD_SIZE+ end = min((s + 1) * SHARD_SIZE, len(records))+ path = os.path.join(OUT_DIR, f"part-{s:05d}.jsonl")+ with open(path, "w", encoding="utf-8") as f:+ for record in records[start:end]:+ f.write(json.dumps(record, ensure_ascii=False) + "\n")+ print(f" wrote shard {s + 1}/{num_shards}: {path}")++ return len(records), num_shards+++def write_manifest(num_examples, num_shards):+ manifest = {+ "source": "Agent-Ark/Toucan-1.5M",+ "format": "openai_tool_calls",+ "created": datetime.datetime.utcnow().isoformat() + "Z",+ "shard_size": SHARD_SIZE,+ "num_shards": num_shards,+ "num_examples": num_examples,+ "bucket_path": BUCKET_PATH,+ "tool_registry": "inline_per_record",+ }+ path = "toucan_openai_jsonl/MANIFEST.json"+ with open(path, "w") as f:+ json.dump(manifest, f, indent=2)+ print(f" wrote {path}")+++def upload():+ print("Uploading to GCS...")+ subprocess.run(+ ["gsutil.cmd", "-m", "cp", "-r", "toucan_openai_jsonl", BUCKET_PATH],+ check=True,+ )+ print(f"Done. Files at {BUCKET_PATH}/toucan_openai_jsonl/")+++if __name__ == "__main__":+ print("Downloading dataset...")+ ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train")+ print(f" {len(ds)} examples")++ print("Converting and sharding...")+ num_examples, num_shards = convert_and_shard(ds)++ print("Writing manifest...")+ write_manifest(num_examples, num_shards)++ upload()diff --git a/src/data.py b/src/data.pyindex b916035..e14fd7d 100644--- a/src/data.py+++ b/src/data.py@@ -3,6 +3,7 @@ import json as _jsonimport multiprocessing as mpimport osimport queue+import subprocessimport threadingos.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"@@ -13,7 +14,7 @@ logging.getLogger("transformers").setLevel(logging.ERROR)logging.getLogger("huggingface_hub").setLevel(logging.ERROR)import numpy as np-from datasets import Audio, load_from_disk+from datasets import Audio, Dataset, concatenate_datasets, load_from_diskfrom tqdm import tqdmimport sentencepiece as spm@@ -23,6 +24,10 @@ TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")GCS_DATASET_PATH = "gs://cactus-dataset/tool_calls"+JSONL_DATASET_PATHS = [+ "gs://needle-datasets-bucket/datasets/xlam/openai_jsonl_v1/xlam_openai_jsonl/train",+ "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1/toucan_openai_jsonl/train",+]_MIN_SHM_BYTES = 200 * 1024**3@@ -56,6 +61,57 @@ TRANSCRIBE_ID = 5_unified_dataset_cache = None+def _load_jsonl_shards(gcs_paths):+ rows = []+ for gcs_path in gcs_paths:+ try:+ result = subprocess.run(+ ["gcloud", "storage", "ls", f"{gcs_path}/*.jsonl"],+ capture_output=True,+ text=True,+ check=True,+ )+ shard_uris = [line.strip() for line in result.stdout.splitlines() if line.strip()]+ except Exception as e:+ print(f"Warning: could not list shards at {gcs_path}: {e}")+ continue++ for uri in shard_uris:+ fname = os.path.basename(uri)+ local_path = os.path.join(CACHE_DIR, fname)+ try:+ subprocess.run(+ ["gcloud", "storage", "cp", uri, local_path],+ capture_output=True,+ check=True,+ )+ except Exception as e:+ print(f"Warning: could not download {uri}: {e}")+ continue++ with open(local_path, "r", encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ try:+ record = _json.loads(line)+ except Exception:+ continue+ messages = record.get("messages", [])+ query = messages[0]["content"] if messages else ""+ tool_calls = messages[1].get("tool_calls") if len(messages) > 1 else None+ rows.append({+ "query": query,+ "tools": _json.dumps(record.get("tools", [])),+ "answers": _json.dumps(tool_calls if tool_calls is not None else []),+ })++ if not rows:+ return None+ return Dataset.from_list(rows)++def _load_unified_dataset():"""Load the unified dataset from GCS (via gcsfs) or local fallback.@@ -80,16 +136,24 @@ def _load_unified_dataset():try:ds = load_from_disk(GCS_DATASET_PATH)print(f"Loaded unified dataset from GCS ({len(ds)} rows)")- _unified_dataset_cache = _set_audio_backend(ds)- return _unified_dataset_cache+ ds = _set_audio_backend(ds)except Exception as e:- gcs_err = e+ raise FileNotFoundError(+ f"Unified dataset not found. Run scripts/build_dataset.py first, "+ f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "+ f"GCS error: {e}"+ )- raise FileNotFoundError(- f"Unified dataset not found. Run scripts/build_dataset.py first, "- f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "- f"GCS error: {gcs_err}"- )+ try:+ jsonl_ds = _load_jsonl_shards(JSONL_DATASET_PATHS)+ if jsonl_ds is not None:+ ds = concatenate_datasets([ds, jsonl_ds])+ print(f"Concatenated JSONL shards: total {len(ds)} rows")+ except Exception as e:+ print(f"Warning: failed to load JSONL shards, continuing with Arrow only: {e}")++ _unified_dataset_cache = ds+ return _unified_dataset_cachedef _set_audio_backend(ds):@@ -392,6 +456,15 @@ def _load_cache_metadata(split):return None+def _tools_to_names(tools_json):+ try:+ tools = _json.loads(tools_json) if isinstance(tools_json, str) else tools_json+ names = [t["function"]["name"] for t in tools if t.get("function", {}).get("name")]+ return _json.dumps(names)+ except Exception:+ return tools_json++def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, batch_size=None):"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.@@ -447,7 +520,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, batool_call_id = tokenizer.tool_call_token_idenc_texts = [ex["query"] for ex in ds]- tools_texts = [ex["tools"] for ex in ds]+ tools_texts = [_tools_to_names(ex["tools"]) for ex in ds]ans_texts = [ex["answers"] for ex in ds]num_workers = min(os.cpu_count() or 1, 8)