Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

shuffle tools

HenryNdubuakua5f1f5047e13
diff --git a/src/cli.py b/src/cli.py
index 4a61322..2a3b5d8 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -15,7 +15,7 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--checkpoint", type=str, default=None)
- p.add_argument("--epochs", type=int, default=10)
+ p.add_argument("--epochs", type=int, default=5)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
@@ -64,6 +64,8 @@ def main():
help="Loss weight for argument value tokens (default: 2.0)")
p.add_argument("--w-key", type=float, default=1.5,
help="Loss weight for argument key tokens (default: 1.5)")
+ p.add_argument("--shuffle-tools", action=argparse.BooleanOptionalAction, default=True,
+ help="Shuffle tool order in encoder input (default: True)")
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
diff --git a/src/data.py b/src/data.py
index d4db8d3..e736eb0 100644
--- a/src/data.py
+++ b/src/data.py
@@ -126,6 +126,19 @@ def _mark_json_key_in_args(s, char_w, key, weight):
char_w[m.start() + 1:m.start() + 1 + len(key)], weight)
+def _shuffle_tools_json(tools_str, seed):
+ """Parse tools JSON array, shuffle order deterministically, re-serialize."""
+ try:
+ tools = _json.loads(tools_str)
+ except (ValueError, TypeError):
+ return tools_str
+ if not isinstance(tools, list) or len(tools) <= 1:
+ return tools_str
+ rng = np.random.RandomState(seed)
+ rng.shuffle(tools)
+ return _json.dumps(tools, separators=(",", ":"))
+
+
def _token_weights_for_answer(answer_str, token_ids, sp_model,
w_name=3.0, w_value=2.0, w_key=1.5):
"""Compute per-token loss weights based on JSON structure.
@@ -380,9 +393,9 @@ def _tokenizer_hash():
def _cache_key(prefix, n_samples, max_enc_len, max_dec_len,
- w_name=3.0, w_value=2.0, w_key=1.5):
+ w_name=3.0, w_value=2.0, w_key=1.5, shuffle_tools=True):
tok_hash = _tokenizer_hash()
- key = f"{prefix}_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}_{w_name}_{w_value}_{w_key}"
+ key = f"{prefix}_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}_{w_name}_{w_value}_{w_key}_{shuffle_tools}"
return hashlib.md5(key.encode()).hexdigest()[:12]
@@ -411,7 +424,7 @@ def _load_cache_metadata(split):
def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_dec_len=DEFAULT_MAX_DEC_LEN,
- w_name=3.0, w_value=2.0, w_key=1.5):
+ w_name=3.0, w_value=2.0, w_key=1.5, shuffle_tools=True):
"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.
Encoder input: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
@@ -423,7 +436,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
"""
cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
- w_name, w_value, w_key)
+ w_name, w_value, w_key, shuffle_tools)
cache_path = os.path.join(CACHE_DIR, cache_id)
tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
@@ -450,6 +463,9 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
tools_texts = [ex["tools"] for ex in ds]
ans_texts = [ex["answers"] for ex in ds]
+ if shuffle_tools:
+ tools_texts = [_shuffle_tools_json(t, seed=i) for i, t in enumerate(tools_texts)]
+
num_workers = min(os.cpu_count() or 1, 8)
model_path = TOKENIZER_PREFIX + ".model"
chunk_size = max(1, len(enc_texts) // (num_workers * 4))
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 14cee39..51f607e 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -56,12 +56,13 @@ def tokenize(args):
w_name = getattr(args, "w_name", 3.0)
w_value = getattr(args, "w_value", 2.0)
w_key = getattr(args, "w_key", 1.5)
+ shuffle_tools = getattr(args, "shuffle_tools", True)
_, _, _, _, kept_indices = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,
- w_name=w_name, w_value=w_value, w_key=w_key,
+ w_name=w_name, w_value=w_value, w_key=w_key, shuffle_tools=shuffle_tools,
)
text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
- w_name, w_value, w_key)
+ w_name, w_value, w_key, shuffle_tools)
_save_cache_metadata(split, text_cache_id, len(kept_indices),
max_enc_len, max_dec_len)