Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Remove upload scripts

justinl6604dfe7d34370
diff --git a/dataset-scripts/toucan/preview_toucan.py b/dataset-scripts/toucan/preview_toucan.py
deleted file mode 100644
index 14fc984..0000000
--- a/dataset-scripts/toucan/preview_toucan.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import ast
-import json
-import uuid
-from datasets import load_dataset
-from langdetect import detect, DetectorFactory
-
-DetectorFactory.seed = 0
-
-NUM_EXAMPLES = 1000
-OUT_FILE = "toucan_preview.jsonl"
-
-
-def strip_titles(obj):
- if isinstance(obj, dict):
- return {k: strip_titles(v) for k, v in obj.items() if k != "title"}
- if isinstance(obj, list):
- return [strip_titles(v) for v in obj]
- return obj
-
-
-def build_tool_subset(available_tools, target_names):
- by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}
- subset = [by_name[n] for n in target_names if n in by_name]
- for tool in available_tools:
- if len(subset) >= 5:
- break
- if tool.get("function", {}).get("name") not in target_names:
- subset.append(tool)
- return [strip_titles(t) for t in subset[:5]]
-
-
-def parse_tool_call(m):
- try:
- fc = ast.literal_eval(m["content"])
- except Exception:
- return None
- try:
- args = json.loads(fc.get("arguments", "{}"))
- except Exception:
- args = fc.get("arguments", {})
- return {
- "id": f"call_1",
- "type": "function",
- "function": {
- "name": fc.get("name", ""),
- "arguments": args,
- },
- }
-
-
-def convert(row):
- target_names = [n.strip() for n in row["target_tools"].split(",")]
- if len(target_names) > 3:
- return []
-
- try:
- if detect(row["question"]) != "en":
- return []
- except Exception:
- return []
-
- available_tools = json.loads(row["tools"])
- tool_subset = build_tool_subset(available_tools, target_names)
-
- messages = json.loads(row["messages"])
- query = next((m["content"] for m in messages if m.get("role") == "user"), None)
- if query is None:
- return []
-
- records = []
- current_calls = []
- for m in messages:
- role = m.get("role")
- if role == "tool_call":
- tc = parse_tool_call(m)
- if tc:
- tc["id"] = f"call_{len(current_calls) + 1}"
- current_calls.append(tc)
- elif role in ("tool_response", "assistant") and current_calls:
- records.append({
- "id": str(uuid.uuid4()),
- "messages": [
- {"role": "user", "content": query},
- {"role": "assistant", "tool_calls": current_calls},
- ],
- "tools": tool_subset,
- })
- current_calls = []
-
- return records
-
-
-ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train", streaming=True)
-
-count = 0
-with open(OUT_FILE, "w", encoding="utf-8") as f:
- for row in ds:
- if count >= NUM_EXAMPLES:
- break
- for record in convert(row):
- if count >= NUM_EXAMPLES:
- break
- f.write(json.dumps(record, ensure_ascii=False) + "\n")
- print(f"--- Example {count} ---")
- print(json.dumps(record, indent=2))
- print()
- count += 1
-
-print(f"Saved {count} examples to {OUT_FILE}")
diff --git a/dataset-scripts/toucan/upload_toucan.py b/dataset-scripts/toucan/upload_toucan.py
deleted file mode 100644
index 6cbc0c2..0000000
--- a/dataset-scripts/toucan/upload_toucan.py
+++ /dev/null
@@ -1,161 +0,0 @@
-import ast
-import datetime
-import json
-import math
-import os
-import subprocess
-import uuid
-from datasets import load_dataset
-from langdetect import detect, DetectorFactory
-
-DetectorFactory.seed = 0
-
-BUCKET_PATH = "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1"
-OUT_DIR = "toucan_openai_jsonl/train"
-SHARD_SIZE = 2000
-
-
-def strip_titles(obj):
- if isinstance(obj, dict):
- return {k: strip_titles(v) for k, v in obj.items() if k != "title"}
- if isinstance(obj, list):
- return [strip_titles(v) for v in obj]
- return obj
-
-
-def build_tool_subset(available_tools, target_names):
- by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}
- subset = [by_name[n] for n in target_names if n in by_name]
- for tool in available_tools:
- if len(subset) >= 5:
- break
- if tool.get("function", {}).get("name") not in target_names:
- subset.append(tool)
- return [strip_titles(t) for t in subset[:5]]
-
-
-def parse_tool_call(m):
- try:
- fc = ast.literal_eval(m["content"])
- except Exception:
- return None
- try:
- args = json.loads(fc.get("arguments", "{}"))
- except Exception:
- args = fc.get("arguments", {})
- return {
- "id": "call_1",
- "type": "function",
- "function": {
- "name": fc.get("name", ""),
- "arguments": args,
- },
- }
-
-
-def convert(row):
- target_names = [n.strip() for n in row["target_tools"].split(",")]
- if len(target_names) > 3:
- return []
-
- try:
- if detect(row["question"]) != "en":
- return []
- except Exception:
- return []
-
- available_tools = json.loads(row["tools"])
- tool_subset = build_tool_subset(available_tools, target_names)
-
- messages = json.loads(row["messages"])
- query = next((m["content"] for m in messages if m.get("role") == "user"), None)
- if query is None:
- return []
-
- records = []
- current_calls = []
- for m in messages:
- role = m.get("role")
- if role == "tool_call":
- tc = parse_tool_call(m)
- if tc:
- tc["id"] = f"call_{len(current_calls) + 1}"
- current_calls.append(tc)
- elif role in ("tool_response", "assistant") and current_calls:
- records.append({
- "id": str(uuid.uuid4()),
- "messages": [
- {"role": "user", "content": query},
- {"role": "assistant", "tool_calls": current_calls},
- ],
- "tools": tool_subset,
- })
- current_calls = []
-
- return records
-
-
-def convert_and_shard(ds):
- os.makedirs(OUT_DIR, exist_ok=True)
- records = []
- skipped = 0
- for row in ds:
- converted = convert(row)
- if not converted:
- skipped += 1
- continue
- records.extend(converted)
-
- print(f" {len(records)} records from {len(ds) - skipped} source rows, {skipped} skipped")
-
- num_shards = math.ceil(len(records) / SHARD_SIZE)
- for s in range(num_shards):
- start = s * SHARD_SIZE
- end = min((s + 1) * SHARD_SIZE, len(records))
- path = os.path.join(OUT_DIR, f"part-{s:05d}.jsonl")
- with open(path, "w", encoding="utf-8") as f:
- for record in records[start:end]:
- f.write(json.dumps(record, ensure_ascii=False) + "\n")
- print(f" wrote shard {s + 1}/{num_shards}: {path}")
-
- return len(records), num_shards
-
-
-def write_manifest(num_examples, num_shards):
- manifest = {
- "source": "Agent-Ark/Toucan-1.5M",
- "format": "openai_tool_calls",
- "created": datetime.datetime.utcnow().isoformat() + "Z",
- "shard_size": SHARD_SIZE,
- "num_shards": num_shards,
- "num_examples": num_examples,
- "bucket_path": BUCKET_PATH,
- "tool_registry": "inline_per_record",
- }
- path = "toucan_openai_jsonl/MANIFEST.json"
- with open(path, "w") as f:
- json.dump(manifest, f, indent=2)
- print(f" wrote {path}")
-
-
-def upload():
- print("Uploading to GCS...")
- subprocess.run(
- ["gsutil.cmd", "-m", "cp", "-r", "toucan_openai_jsonl", BUCKET_PATH],
- check=True,
- )
- print(f"Done. Files at {BUCKET_PATH}/toucan_openai_jsonl/")
-
-
-if __name__ == "__main__":
- print("Downloading dataset...")
- ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train")
- print(f" {len(ds)} examples")
-
- print("Converting and sharding...")
- num_examples, num_shards = convert_and_shard(ds)
-
- print("Writing manifest...")
- write_manifest(num_examples, num_shards)
-
- upload()
diff --git a/dataset-scripts/xlam/preview_xlam.py b/dataset-scripts/xlam/preview_xlam.py
deleted file mode 100644
index aabb9c5..0000000
--- a/dataset-scripts/xlam/preview_xlam.py
+++ /dev/null
@@ -1,102 +0,0 @@
-import json
-from datasets import load_dataset
-
-NUM_EXAMPLES = 1000
-OUT_FILE = "xlam_preview.jsonl"
-
-TYPE_MAP = {
- "str": "string",
- "int": "integer",
- "float": "number",
- "bool": "boolean",
- "list": "array",
- "dict": "object",
-}
-
-
-def normalize_type(raw):
- base = raw.split(",")[0].strip().lower()
- if base.startswith(("list[", "tuple[", "set[", "list", "tuple", "set")):
- return "array"
- if base.startswith("callable"):
- return "string"
- if base.startswith(("dict[", "union[")):
- return "object"
- return TYPE_MAP.get(base, base)
-
-
-def to_openai_tool(tool):
- props = {}
- required = []
- for pname, pinfo in tool.get("parameters", {}).items():
- props[pname] = {
- "type": normalize_type(pinfo.get("type", "string")),
- "description": pinfo.get("description", ""),
- }
- if pinfo.get("required", False):
- required.append(pname)
-
- schema = {"type": "object", "properties": props}
- if required:
- schema["required"] = required
-
- return {
- "type": "function",
- "function": {
- "name": tool.get("name", ""),
- "description": tool.get("description", ""),
- "parameters": schema,
- },
- }
-
-
-def to_openai_tool_call(ans, idx):
- return {
- "id": f"call_{idx}",
- "type": "function",
- "function": {
- "name": ans.get("name", ""),
- "arguments": ans.get("arguments", {}),
- },
- }
-
-
-def maybe_parse(val):
- if isinstance(val, str):
- try:
- return json.loads(val)
- except Exception:
- return val
- return val
-
-
-ds = load_dataset("Salesforce/xlam-function-calling-60k", split="train")
-
-with open(OUT_FILE, "w", encoding="utf-8") as f:
- for i in range(NUM_EXAMPLES):
- ex = ds[i]
- query = maybe_parse(ex.get("query", ""))
- tools = maybe_parse(ex.get("tools", []))
- answers = maybe_parse(ex.get("answers", []))
-
- record = {
- "id": ex.get("id", f"ex_{i}"),
- "messages": [
- {"role": "user", "content": query},
- {
- "role": "assistant",
- "tool_calls": [
- to_openai_tool_call(a, j + 1)
- for j, a in enumerate(answers)
- ],
- },
- ],
- "tools": [to_openai_tool(t) for t in tools],
- }
- f.write(json.dumps(record, ensure_ascii=False) + "\n")
-
- print(f"--- Example {i} ---")
- print(json.dumps(record, indent=2))
- print()
-
-print(f"Saved {NUM_EXAMPLES} examples to {OUT_FILE}")
diff --git a/dataset-scripts/xlam/upload_xlam.py b/dataset-scripts/xlam/upload_xlam.py
deleted file mode 100644
index e2e6a94..0000000
--- a/dataset-scripts/xlam/upload_xlam.py
+++ /dev/null
@@ -1,153 +0,0 @@
-import os
-import math
-import json
-import subprocess
-from datasets import load_dataset
-
-BUCKET_PATH = "gs://needle-datasets-bucket/datasets/xlam/openai_jsonl_v1"
-OUT_DIR = "xlam_openai_jsonl/train"
-SHARD_SIZE = 2000
-
-
-TYPE_MAP = {
- "str": "string",
- "int": "integer",
- "float": "number",
- "bool": "boolean",
- "list": "array",
- "dict": "object",
-}
-
-
-def normalize_type(raw):
- base = raw.split(",")[0].strip().lower()
- if base.startswith(("list[", "tuple[", "set[", "list", "tuple", "set")):
- return "array"
- if base.startswith("callable"):
- return "string"
- if base.startswith(("dict[", "union[")):
- return "object"
- return TYPE_MAP.get(base, base)
-
-
-def to_openai_tool(tool):
- props = {}
- required = []
- for pname, pinfo in tool.get("parameters", {}).items():
- props[pname] = {
- "type": normalize_type(pinfo.get("type", "string")),
- "description": pinfo.get("description", ""),
- }
- if pinfo.get("required", False):
- required.append(pname)
-
- schema = {"type": "object", "properties": props}
- if required:
- schema["required"] = required
-
- return {
- "type": "function",
- "function": {
- "name": tool.get("name", ""),
- "description": tool.get("description", ""),
- "parameters": schema,
- },
- }
-
-
-def to_openai_tool_call(ans, idx):
- return {
- "id": f"call_{idx}",
- "type": "function",
- "function": {
- "name": ans.get("name", ""),
- "arguments": ans.get("arguments", {}),
- },
- }
-
-
-def maybe_parse(val):
- if isinstance(val, str):
- try:
- return json.loads(val)
- except Exception:
- return val
- return val
-
-
-def convert_and_shard(ds):
- os.makedirs(OUT_DIR, exist_ok=True)
- num_shards = math.ceil(len(ds) / SHARD_SIZE)
-
- for s in range(num_shards):
- start = s * SHARD_SIZE
- end = min((s + 1) * SHARD_SIZE, len(ds))
- path = os.path.join(OUT_DIR, f"part-{s:05d}.jsonl")
-
- with open(path, "w", encoding="utf-8") as f:
- for i in range(start, end):
- ex = ds[i]
- query = maybe_parse(ex.get("query", ""))
- tools = maybe_parse(ex.get("tools", []))
- answers = maybe_parse(ex.get("answers", []))
-
- record = {
- "id": ex.get("id", f"ex_{i}"),
- "messages": [
- {"role": "user", "content": query},
- {
- "role": "assistant",
- "tool_calls": [
- to_openai_tool_call(a, j + 1)
- for j, a in enumerate(answers)
- ],
- },
- ],
- "tools": [to_openai_tool(t) for t in tools],
- }
- f.write(json.dumps(record, ensure_ascii=False) + "\n")
-
- print(f" wrote shard {s + 1}/{num_shards}: {path}")
-
- return num_shards
-
-
-def write_manifest(num_examples, num_shards):
- import datetime
- manifest = {
- "source": "Salesforce/xlam-function-calling-60k",
- "format": "openai_tool_calls",
- "created": datetime.datetime.utcnow().isoformat() + "Z",
- "shard_size": SHARD_SIZE,
- "num_shards": num_shards,
- "num_examples": num_examples,
- "bucket_path": BUCKET_PATH,
- "tool_registry": "inline_per_record",
- }
- path = "xlam_openai_jsonl/MANIFEST.json"
- with open(path, "w") as f:
- json.dump(manifest, f, indent=2)
- print(f" wrote {path}")
-
-
-def upload():
- print("Uploading to GCS...")
- subprocess.run(
- ["gsutil.cmd", "-m", "cp", "-r", "xlam_openai_jsonl", BUCKET_PATH],
- check=True,
- )
- print(f"Done. Files at {BUCKET_PATH}/xlam_openai_jsonl/")
-
-
-if __name__ == "__main__":
- print("Downloading dataset...")
- ds = load_dataset("Salesforce/xlam-function-calling-60k", split="train")
- print(f" {len(ds)} examples")
-
- print("Converting and sharding...")
- num_shards = convert_and_shard(ds)
-
- print("Writing manifest...")
- write_manifest(len(ds), num_shards)
-
- upload()