Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Add toucan upload

justinl66c4106f3a2bb2
diff --git a/dataset-scripts/toucan/preview_toucan.py b/dataset-scripts/toucan/preview_toucan.py
new file mode 100644
index 0000000..14fc984
--- /dev/null
+++ b/dataset-scripts/toucan/preview_toucan.py
@@ -0,0 +1,109 @@
+import ast
+import json
+import uuid
+from datasets import load_dataset
+from langdetect import detect, DetectorFactory
+
+DetectorFactory.seed = 0
+
+NUM_EXAMPLES = 1000
+OUT_FILE = "toucan_preview.jsonl"
+
+
+def strip_titles(obj):
+ if isinstance(obj, dict):
+ return {k: strip_titles(v) for k, v in obj.items() if k != "title"}
+ if isinstance(obj, list):
+ return [strip_titles(v) for v in obj]
+ return obj
+
+
+def build_tool_subset(available_tools, target_names):
+ by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}
+ subset = [by_name[n] for n in target_names if n in by_name]
+ for tool in available_tools:
+ if len(subset) >= 5:
+ break
+ if tool.get("function", {}).get("name") not in target_names:
+ subset.append(tool)
+ return [strip_titles(t) for t in subset[:5]]
+
+
+def parse_tool_call(m):
+ try:
+ fc = ast.literal_eval(m["content"])
+ except Exception:
+ return None
+ try:
+ args = json.loads(fc.get("arguments", "{}"))
+ except Exception:
+ args = fc.get("arguments", {})
+ return {
+ "id": f"call_1",
+ "type": "function",
+ "function": {
+ "name": fc.get("name", ""),
+ "arguments": args,
+ },
+ }
+
+
+def convert(row):
+ target_names = [n.strip() for n in row["target_tools"].split(",")]
+ if len(target_names) > 3:
+ return []
+
+ try:
+ if detect(row["question"]) != "en":
+ return []
+ except Exception:
+ return []
+
+ available_tools = json.loads(row["tools"])
+ tool_subset = build_tool_subset(available_tools, target_names)
+
+ messages = json.loads(row["messages"])
+ query = next((m["content"] for m in messages if m.get("role") == "user"), None)
+ if query is None:
+ return []
+
+ records = []
+ current_calls = []
+ for m in messages:
+ role = m.get("role")
+ if role == "tool_call":
+ tc = parse_tool_call(m)
+ if tc:
+ tc["id"] = f"call_{len(current_calls) + 1}"
+ current_calls.append(tc)
+ elif role in ("tool_response", "assistant") and current_calls:
+ records.append({
+ "id": str(uuid.uuid4()),
+ "messages": [
+ {"role": "user", "content": query},
+ {"role": "assistant", "tool_calls": current_calls},
+ ],
+ "tools": tool_subset,
+ })
+ current_calls = []
+
+ return records
+
+
+ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train", streaming=True)
+
+count = 0
+with open(OUT_FILE, "w", encoding="utf-8") as f:
+ for row in ds:
+ if count >= NUM_EXAMPLES:
+ break
+ for record in convert(row):
+ if count >= NUM_EXAMPLES:
+ break
+ f.write(json.dumps(record, ensure_ascii=False) + "\n")
+ print(f"--- Example {count} ---")
+ print(json.dumps(record, indent=2))
+ print()
+ count += 1
+
+print(f"Saved {count} examples to {OUT_FILE}")
diff --git a/dataset-scripts/toucan/upload_toucan.py b/dataset-scripts/toucan/upload_toucan.py
new file mode 100644
index 0000000..6cbc0c2
--- /dev/null
+++ b/dataset-scripts/toucan/upload_toucan.py
@@ -0,0 +1,161 @@
+import ast
+import datetime
+import json
+import math
+import os
+import subprocess
+import uuid
+from datasets import load_dataset
+from langdetect import detect, DetectorFactory
+
+DetectorFactory.seed = 0
+
+BUCKET_PATH = "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1"
+OUT_DIR = "toucan_openai_jsonl/train"
+SHARD_SIZE = 2000
+
+
+def strip_titles(obj):
+ if isinstance(obj, dict):
+ return {k: strip_titles(v) for k, v in obj.items() if k != "title"}
+ if isinstance(obj, list):
+ return [strip_titles(v) for v in obj]
+ return obj
+
+
+def build_tool_subset(available_tools, target_names):
+ by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}
+ subset = [by_name[n] for n in target_names if n in by_name]
+ for tool in available_tools:
+ if len(subset) >= 5:
+ break
+ if tool.get("function", {}).get("name") not in target_names:
+ subset.append(tool)
+ return [strip_titles(t) for t in subset[:5]]
+
+
+def parse_tool_call(m):
+ try:
+ fc = ast.literal_eval(m["content"])
+ except Exception:
+ return None
+ try:
+ args = json.loads(fc.get("arguments", "{}"))
+ except Exception:
+ args = fc.get("arguments", {})
+ return {
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": fc.get("name", ""),
+ "arguments": args,
+ },
+ }
+
+
+def convert(row):
+ target_names = [n.strip() for n in row["target_tools"].split(",")]
+ if len(target_names) > 3:
+ return []
+
+ try:
+ if detect(row["question"]) != "en":
+ return []
+ except Exception:
+ return []
+
+ available_tools = json.loads(row["tools"])
+ tool_subset = build_tool_subset(available_tools, target_names)
+
+ messages = json.loads(row["messages"])
+ query = next((m["content"] for m in messages if m.get("role") == "user"), None)
+ if query is None:
+ return []
+
+ records = []
+ current_calls = []
+ for m in messages:
+ role = m.get("role")
+ if role == "tool_call":
+ tc = parse_tool_call(m)
+ if tc:
+ tc["id"] = f"call_{len(current_calls) + 1}"
+ current_calls.append(tc)
+ elif role in ("tool_response", "assistant") and current_calls:
+ records.append({
+ "id": str(uuid.uuid4()),
+ "messages": [
+ {"role": "user", "content": query},
+ {"role": "assistant", "tool_calls": current_calls},
+ ],
+ "tools": tool_subset,
+ })
+ current_calls = []
+
+ return records
+
+
+def convert_and_shard(ds):
+ os.makedirs(OUT_DIR, exist_ok=True)
+ records = []
+ skipped = 0
+ for row in ds:
+ converted = convert(row)
+ if not converted:
+ skipped += 1
+ continue
+ records.extend(converted)
+
+ print(f" {len(records)} records from {len(ds) - skipped} source rows, {skipped} skipped")
+
+ num_shards = math.ceil(len(records) / SHARD_SIZE)
+ for s in range(num_shards):
+ start = s * SHARD_SIZE
+ end = min((s + 1) * SHARD_SIZE, len(records))
+ path = os.path.join(OUT_DIR, f"part-{s:05d}.jsonl")
+ with open(path, "w", encoding="utf-8") as f:
+ for record in records[start:end]:
+ f.write(json.dumps(record, ensure_ascii=False) + "\n")
+ print(f" wrote shard {s + 1}/{num_shards}: {path}")
+
+ return len(records), num_shards
+
+
+def write_manifest(num_examples, num_shards):
+ manifest = {
+ "source": "Agent-Ark/Toucan-1.5M",
+ "format": "openai_tool_calls",
+ "created": datetime.datetime.utcnow().isoformat() + "Z",
+ "shard_size": SHARD_SIZE,
+ "num_shards": num_shards,
+ "num_examples": num_examples,
+ "bucket_path": BUCKET_PATH,
+ "tool_registry": "inline_per_record",
+ }
+ path = "toucan_openai_jsonl/MANIFEST.json"
+ with open(path, "w") as f:
+ json.dump(manifest, f, indent=2)
+ print(f" wrote {path}")
+
+
+def upload():
+ print("Uploading to GCS...")
+ subprocess.run(
+ ["gsutil.cmd", "-m", "cp", "-r", "toucan_openai_jsonl", BUCKET_PATH],
+ check=True,
+ )
+ print(f"Done. Files at {BUCKET_PATH}/toucan_openai_jsonl/")
+
+
+if __name__ == "__main__":
+ print("Downloading dataset...")
+ ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train")
+ print(f" {len(ds)} examples")
+
+ print("Converting and sharding...")
+ num_examples, num_shards = convert_and_shard(ds)
+
+ print("Writing manifest...")
+ write_manifest(num_examples, num_shards)
+
+ upload()
diff --git a/src/data.py b/src/data.py
index b916035..e14fd7d 100644
--- a/src/data.py
+++ b/src/data.py
@@ -3,6 +3,7 @@ import json as _json
import multiprocessing as mp
import os
import queue
+import subprocess
import threading
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
@@ -13,7 +14,7 @@ logging.getLogger("transformers").setLevel(logging.ERROR)
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
import numpy as np
-from datasets import Audio, load_from_disk
+from datasets import Audio, Dataset, concatenate_datasets, load_from_disk
from tqdm import tqdm
import sentencepiece as spm
@@ -23,6 +24,10 @@ TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")
TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
GCS_DATASET_PATH = "gs://cactus-dataset/tool_calls"
+JSONL_DATASET_PATHS = [
+ "gs://needle-datasets-bucket/datasets/xlam/openai_jsonl_v1/xlam_openai_jsonl/train",
+ "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1/toucan_openai_jsonl/train",
+]
_MIN_SHM_BYTES = 200 * 1024**3
@@ -56,6 +61,57 @@ TRANSCRIBE_ID = 5
_unified_dataset_cache = None
+def _load_jsonl_shards(gcs_paths):
+ rows = []
+ for gcs_path in gcs_paths:
+ try:
+ result = subprocess.run(
+ ["gcloud", "storage", "ls", f"{gcs_path}/*.jsonl"],
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ shard_uris = [line.strip() for line in result.stdout.splitlines() if line.strip()]
+ except Exception as e:
+ print(f"Warning: could not list shards at {gcs_path}: {e}")
+ continue
+
+ for uri in shard_uris:
+ fname = os.path.basename(uri)
+ local_path = os.path.join(CACHE_DIR, fname)
+ try:
+ subprocess.run(
+ ["gcloud", "storage", "cp", uri, local_path],
+ capture_output=True,
+ check=True,
+ )
+ except Exception as e:
+ print(f"Warning: could not download {uri}: {e}")
+ continue
+
+ with open(local_path, "r", encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ record = _json.loads(line)
+ except Exception:
+ continue
+ messages = record.get("messages", [])
+ query = messages[0]["content"] if messages else ""
+ tool_calls = messages[1].get("tool_calls") if len(messages) > 1 else None
+ rows.append({
+ "query": query,
+ "tools": _json.dumps(record.get("tools", [])),
+ "answers": _json.dumps(tool_calls if tool_calls is not None else []),
+ })
+
+ if not rows:
+ return None
+ return Dataset.from_list(rows)
+
+
def _load_unified_dataset():
"""Load the unified dataset from GCS (via gcsfs) or local fallback.
@@ -80,16 +136,24 @@ def _load_unified_dataset():
try:
ds = load_from_disk(GCS_DATASET_PATH)
print(f"Loaded unified dataset from GCS ({len(ds)} rows)")
- _unified_dataset_cache = _set_audio_backend(ds)
- return _unified_dataset_cache
+ ds = _set_audio_backend(ds)
except Exception as e:
- gcs_err = e
+ raise FileNotFoundError(
+ f"Unified dataset not found. Run scripts/build_dataset.py first, "
+ f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "
+ f"GCS error: {e}"
+ )
- raise FileNotFoundError(
- f"Unified dataset not found. Run scripts/build_dataset.py first, "
- f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "
- f"GCS error: {gcs_err}"
- )
+ try:
+ jsonl_ds = _load_jsonl_shards(JSONL_DATASET_PATHS)
+ if jsonl_ds is not None:
+ ds = concatenate_datasets([ds, jsonl_ds])
+ print(f"Concatenated JSONL shards: total {len(ds)} rows")
+ except Exception as e:
+ print(f"Warning: failed to load JSONL shards, continuing with Arrow only: {e}")
+
+ _unified_dataset_cache = ds
+ return _unified_dataset_cache
def _set_audio_backend(ds):
@@ -392,6 +456,15 @@ def _load_cache_metadata(split):
return None
+def _tools_to_names(tools_json):
+ try:
+ tools = _json.loads(tools_json) if isinstance(tools_json, str) else tools_json
+ names = [t["function"]["name"] for t in tools if t.get("function", {}).get("name")]
+ return _json.dumps(names)
+ except Exception:
+ return tools_json
+
+
def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, batch_size=None):
"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.
@@ -447,7 +520,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, ba
tool_call_id = tokenizer.tool_call_token_id
enc_texts = [ex["query"] for ex in ds]
- tools_texts = [ex["tools"] for ex in ds]
+ tools_texts = [_tools_to_names(ex["tools"]) for ex in ds]
ans_texts = [ex["answers"] for ex in ds]
num_workers = min(os.cpu_count() or 1, 8)