Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Remove upload scripts
justinl6604dfe7d34370
diff --git a/dataset-scripts/toucan/preview_toucan.py b/dataset-scripts/toucan/preview_toucan.pydeleted file mode 100644index 14fc984..0000000--- a/dataset-scripts/toucan/preview_toucan.py+++ /dev/null@@ -1,109 +0,0 @@-import ast-import json-import uuid-from datasets import load_dataset-from langdetect import detect, DetectorFactory--DetectorFactory.seed = 0--NUM_EXAMPLES = 1000-OUT_FILE = "toucan_preview.jsonl"---def strip_titles(obj):- if isinstance(obj, dict):- return {k: strip_titles(v) for k, v in obj.items() if k != "title"}- if isinstance(obj, list):- return [strip_titles(v) for v in obj]- return obj---def build_tool_subset(available_tools, target_names):- by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}- subset = [by_name[n] for n in target_names if n in by_name]- for tool in available_tools:- if len(subset) >= 5:- break- if tool.get("function", {}).get("name") not in target_names:- subset.append(tool)- return [strip_titles(t) for t in subset[:5]]---def parse_tool_call(m):- try:- fc = ast.literal_eval(m["content"])- except Exception:- return None- try:- args = json.loads(fc.get("arguments", "{}"))- except Exception:- args = fc.get("arguments", {})- return {- "id": f"call_1",- "type": "function",- "function": {- "name": fc.get("name", ""),- "arguments": args,- },- }---def convert(row):- target_names = [n.strip() for n in row["target_tools"].split(",")]- if len(target_names) > 3:- return []-- try:- if detect(row["question"]) != "en":- return []- except Exception:- return []-- available_tools = json.loads(row["tools"])- tool_subset = build_tool_subset(available_tools, target_names)-- messages = json.loads(row["messages"])- query = next((m["content"] for m in messages if m.get("role") == "user"), None)- if query is None:- return []-- records = []- current_calls = []- for m in messages:- role = m.get("role")- if role == "tool_call":- tc = parse_tool_call(m)- if tc:- tc["id"] = f"call_{len(current_calls) + 1}"- current_calls.append(tc)- elif role in ("tool_response", "assistant") and current_calls:- records.append({- "id": str(uuid.uuid4()),- "messages": [- {"role": "user", "content": query},- {"role": "assistant", "tool_calls": current_calls},- ],- "tools": tool_subset,- })- current_calls = []-- return records---ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train", streaming=True)--count = 0-with open(OUT_FILE, "w", encoding="utf-8") as f:- for row in ds:- if count >= NUM_EXAMPLES:- break- for record in convert(row):- if count >= NUM_EXAMPLES:- break- f.write(json.dumps(record, ensure_ascii=False) + "\n")- print(f"--- Example {count} ---")- print(json.dumps(record, indent=2))- print()- count += 1--print(f"Saved {count} examples to {OUT_FILE}")diff --git a/dataset-scripts/toucan/upload_toucan.py b/dataset-scripts/toucan/upload_toucan.pydeleted file mode 100644index 6cbc0c2..0000000--- a/dataset-scripts/toucan/upload_toucan.py+++ /dev/null@@ -1,161 +0,0 @@-import ast-import datetime-import json-import math-import os-import subprocess-import uuid-from datasets import load_dataset-from langdetect import detect, DetectorFactory--DetectorFactory.seed = 0--BUCKET_PATH = "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1"-OUT_DIR = "toucan_openai_jsonl/train"-SHARD_SIZE = 2000---def strip_titles(obj):- if isinstance(obj, dict):- return {k: strip_titles(v) for k, v in obj.items() if k != "title"}- if isinstance(obj, list):- return [strip_titles(v) for v in obj]- return obj---def build_tool_subset(available_tools, target_names):- by_name = {t["function"]["name"]: t for t in available_tools if t.get("function", {}).get("name")}- subset = [by_name[n] for n in target_names if n in by_name]- for tool in available_tools:- if len(subset) >= 5:- break- if tool.get("function", {}).get("name") not in target_names:- subset.append(tool)- return [strip_titles(t) for t in subset[:5]]---def parse_tool_call(m):- try:- fc = ast.literal_eval(m["content"])- except Exception:- return None- try:- args = json.loads(fc.get("arguments", "{}"))- except Exception:- args = fc.get("arguments", {})- return {- "id": "call_1",- "type": "function",- "function": {- "name": fc.get("name", ""),- "arguments": args,- },- }---def convert(row):- target_names = [n.strip() for n in row["target_tools"].split(",")]- if len(target_names) > 3:- return []-- try:- if detect(row["question"]) != "en":- return []- except Exception:- return []-- available_tools = json.loads(row["tools"])- tool_subset = build_tool_subset(available_tools, target_names)-- messages = json.loads(row["messages"])- query = next((m["content"] for m in messages if m.get("role") == "user"), None)- if query is None:- return []-- records = []- current_calls = []- for m in messages:- role = m.get("role")- if role == "tool_call":- tc = parse_tool_call(m)- if tc:- tc["id"] = f"call_{len(current_calls) + 1}"- current_calls.append(tc)- elif role in ("tool_response", "assistant") and current_calls:- records.append({- "id": str(uuid.uuid4()),- "messages": [- {"role": "user", "content": query},- {"role": "assistant", "tool_calls": current_calls},- ],- "tools": tool_subset,- })- current_calls = []-- return records---def convert_and_shard(ds):- os.makedirs(OUT_DIR, exist_ok=True)- records = []- skipped = 0- for row in ds:- converted = convert(row)- if not converted:- skipped += 1- continue- records.extend(converted)-- print(f" {len(records)} records from {len(ds) - skipped} source rows, {skipped} skipped")-- num_shards = math.ceil(len(records) / SHARD_SIZE)- for s in range(num_shards):- start = s * SHARD_SIZE- end = min((s + 1) * SHARD_SIZE, len(records))- path = os.path.join(OUT_DIR, f"part-{s:05d}.jsonl")- with open(path, "w", encoding="utf-8") as f:- for record in records[start:end]:- f.write(json.dumps(record, ensure_ascii=False) + "\n")- print(f" wrote shard {s + 1}/{num_shards}: {path}")-- return len(records), num_shards---def write_manifest(num_examples, num_shards):- manifest = {- "source": "Agent-Ark/Toucan-1.5M",- "format": "openai_tool_calls",- "created": datetime.datetime.utcnow().isoformat() + "Z",- "shard_size": SHARD_SIZE,- "num_shards": num_shards,- "num_examples": num_examples,- "bucket_path": BUCKET_PATH,- "tool_registry": "inline_per_record",- }- path = "toucan_openai_jsonl/MANIFEST.json"- with open(path, "w") as f:- json.dump(manifest, f, indent=2)- print(f" wrote {path}")---def upload():- print("Uploading to GCS...")- subprocess.run(- ["gsutil.cmd", "-m", "cp", "-r", "toucan_openai_jsonl", BUCKET_PATH],- check=True,- )- print(f"Done. Files at {BUCKET_PATH}/toucan_openai_jsonl/")---if __name__ == "__main__":- print("Downloading dataset...")- ds = load_dataset("Agent-Ark/Toucan-1.5M", "SFT", split="train")- print(f" {len(ds)} examples")-- print("Converting and sharding...")- num_examples, num_shards = convert_and_shard(ds)-- print("Writing manifest...")- write_manifest(num_examples, num_shards)-- upload()diff --git a/dataset-scripts/xlam/preview_xlam.py b/dataset-scripts/xlam/preview_xlam.pydeleted file mode 100644index aabb9c5..0000000--- a/dataset-scripts/xlam/preview_xlam.py+++ /dev/null@@ -1,102 +0,0 @@-import json-from datasets import load_dataset--NUM_EXAMPLES = 1000-OUT_FILE = "xlam_preview.jsonl"--TYPE_MAP = {- "str": "string",- "int": "integer",- "float": "number",- "bool": "boolean",- "list": "array",- "dict": "object",-}---def normalize_type(raw):- base = raw.split(",")[0].strip().lower()- if base.startswith(("list[", "tuple[", "set[", "list", "tuple", "set")):- return "array"- if base.startswith("callable"):- return "string"- if base.startswith(("dict[", "union[")):- return "object"- return TYPE_MAP.get(base, base)---def to_openai_tool(tool):- props = {}- required = []- for pname, pinfo in tool.get("parameters", {}).items():- props[pname] = {- "type": normalize_type(pinfo.get("type", "string")),- "description": pinfo.get("description", ""),- }- if pinfo.get("required", False):- required.append(pname)-- schema = {"type": "object", "properties": props}- if required:- schema["required"] = required-- return {- "type": "function",- "function": {- "name": tool.get("name", ""),- "description": tool.get("description", ""),- "parameters": schema,- },- }---def to_openai_tool_call(ans, idx):- return {- "id": f"call_{idx}",- "type": "function",- "function": {- "name": ans.get("name", ""),- "arguments": ans.get("arguments", {}),- },- }---def maybe_parse(val):- if isinstance(val, str):- try:- return json.loads(val)- except Exception:- return val- return val---ds = load_dataset("Salesforce/xlam-function-calling-60k", split="train")--with open(OUT_FILE, "w", encoding="utf-8") as f:- for i in range(NUM_EXAMPLES):- ex = ds[i]- query = maybe_parse(ex.get("query", ""))- tools = maybe_parse(ex.get("tools", []))- answers = maybe_parse(ex.get("answers", []))-- record = {- "id": ex.get("id", f"ex_{i}"),- "messages": [- {"role": "user", "content": query},- {- "role": "assistant",- "tool_calls": [- to_openai_tool_call(a, j + 1)- for j, a in enumerate(answers)- ],- },- ],- "tools": [to_openai_tool(t) for t in tools],- }- f.write(json.dumps(record, ensure_ascii=False) + "\n")-- print(f"--- Example {i} ---")- print(json.dumps(record, indent=2))- print()--print(f"Saved {NUM_EXAMPLES} examples to {OUT_FILE}")diff --git a/dataset-scripts/xlam/upload_xlam.py b/dataset-scripts/xlam/upload_xlam.pydeleted file mode 100644index e2e6a94..0000000--- a/dataset-scripts/xlam/upload_xlam.py+++ /dev/null@@ -1,153 +0,0 @@-import os-import math-import json-import subprocess-from datasets import load_dataset--BUCKET_PATH = "gs://needle-datasets-bucket/datasets/xlam/openai_jsonl_v1"-OUT_DIR = "xlam_openai_jsonl/train"-SHARD_SIZE = 2000---TYPE_MAP = {- "str": "string",- "int": "integer",- "float": "number",- "bool": "boolean",- "list": "array",- "dict": "object",-}---def normalize_type(raw):- base = raw.split(",")[0].strip().lower()- if base.startswith(("list[", "tuple[", "set[", "list", "tuple", "set")):- return "array"- if base.startswith("callable"):- return "string"- if base.startswith(("dict[", "union[")):- return "object"- return TYPE_MAP.get(base, base)---def to_openai_tool(tool):- props = {}- required = []- for pname, pinfo in tool.get("parameters", {}).items():- props[pname] = {- "type": normalize_type(pinfo.get("type", "string")),- "description": pinfo.get("description", ""),- }- if pinfo.get("required", False):- required.append(pname)-- schema = {"type": "object", "properties": props}- if required:- schema["required"] = required-- return {- "type": "function",- "function": {- "name": tool.get("name", ""),- "description": tool.get("description", ""),- "parameters": schema,- },- }---def to_openai_tool_call(ans, idx):- return {- "id": f"call_{idx}",- "type": "function",- "function": {- "name": ans.get("name", ""),- "arguments": ans.get("arguments", {}),- },- }---def maybe_parse(val):- if isinstance(val, str):- try:- return json.loads(val)- except Exception:- return val- return val---def convert_and_shard(ds):- os.makedirs(OUT_DIR, exist_ok=True)- num_shards = math.ceil(len(ds) / SHARD_SIZE)-- for s in range(num_shards):- start = s * SHARD_SIZE- end = min((s + 1) * SHARD_SIZE, len(ds))- path = os.path.join(OUT_DIR, f"part-{s:05d}.jsonl")-- with open(path, "w", encoding="utf-8") as f:- for i in range(start, end):- ex = ds[i]- query = maybe_parse(ex.get("query", ""))- tools = maybe_parse(ex.get("tools", []))- answers = maybe_parse(ex.get("answers", []))-- record = {- "id": ex.get("id", f"ex_{i}"),- "messages": [- {"role": "user", "content": query},- {- "role": "assistant",- "tool_calls": [- to_openai_tool_call(a, j + 1)- for j, a in enumerate(answers)- ],- },- ],- "tools": [to_openai_tool(t) for t in tools],- }- f.write(json.dumps(record, ensure_ascii=False) + "\n")-- print(f" wrote shard {s + 1}/{num_shards}: {path}")-- return num_shards---def write_manifest(num_examples, num_shards):- import datetime- manifest = {- "source": "Salesforce/xlam-function-calling-60k",- "format": "openai_tool_calls",- "created": datetime.datetime.utcnow().isoformat() + "Z",- "shard_size": SHARD_SIZE,- "num_shards": num_shards,- "num_examples": num_examples,- "bucket_path": BUCKET_PATH,- "tool_registry": "inline_per_record",- }- path = "xlam_openai_jsonl/MANIFEST.json"- with open(path, "w") as f:- json.dump(manifest, f, indent=2)- print(f" wrote {path}")---def upload():- print("Uploading to GCS...")- subprocess.run(- ["gsutil.cmd", "-m", "cp", "-r", "xlam_openai_jsonl", BUCKET_PATH],- check=True,- )- print(f"Done. Files at {BUCKET_PATH}/xlam_openai_jsonl/")---if __name__ == "__main__":- print("Downloading dataset...")- ds = load_dataset("Salesforce/xlam-function-calling-60k", split="train")- print(f" {len(ds)} examples")-- print("Converting and sharding...")- num_shards = convert_and_shard(ds)-- print("Writing manifest...")- write_manifest(len(ds), num_shards)-- upload()