Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
shuffle tools
HenryNdubuakua5f1f5047e13
diff --git a/src/cli.py b/src/cli.pyindex 4a61322..2a3b5d8 100644--- a/src/cli.py+++ b/src/cli.py@@ -15,7 +15,7 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--checkpoint", type=str, default=None)- p.add_argument("--epochs", type=int, default=10)+ p.add_argument("--epochs", type=int, default=5)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)@@ -64,6 +64,8 @@ def main():help="Loss weight for argument value tokens (default: 2.0)")p.add_argument("--w-key", type=float, default=1.5,help="Loss weight for argument key tokens (default: 1.5)")+ p.add_argument("--shuffle-tools", action=argparse.BooleanOptionalAction, default=True,+ help="Shuffle tool order in encoder input (default: True)")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)diff --git a/src/data.py b/src/data.pyindex d4db8d3..e736eb0 100644--- a/src/data.py+++ b/src/data.py@@ -126,6 +126,19 @@ def _mark_json_key_in_args(s, char_w, key, weight):char_w[m.start() + 1:m.start() + 1 + len(key)], weight)+def _shuffle_tools_json(tools_str, seed):+ """Parse tools JSON array, shuffle order deterministically, re-serialize."""+ try:+ tools = _json.loads(tools_str)+ except (ValueError, TypeError):+ return tools_str+ if not isinstance(tools, list) or len(tools) <= 1:+ return tools_str+ rng = np.random.RandomState(seed)+ rng.shuffle(tools)+ return _json.dumps(tools, separators=(",", ":"))++def _token_weights_for_answer(answer_str, token_ids, sp_model,w_name=3.0, w_value=2.0, w_key=1.5):"""Compute per-token loss weights based on JSON structure.@@ -380,9 +393,9 @@ def _tokenizer_hash():def _cache_key(prefix, n_samples, max_enc_len, max_dec_len,- w_name=3.0, w_value=2.0, w_key=1.5):+ w_name=3.0, w_value=2.0, w_key=1.5, shuffle_tools=True):tok_hash = _tokenizer_hash()- key = f"{prefix}_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}_{w_name}_{w_value}_{w_key}"+ key = f"{prefix}_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}_{w_name}_{w_value}_{w_key}_{shuffle_tools}"return hashlib.md5(key.encode()).hexdigest()[:12]@@ -411,7 +424,7 @@ def _load_cache_metadata(split):def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_dec_len=DEFAULT_MAX_DEC_LEN,- w_name=3.0, w_value=2.0, w_key=1.5):+ w_name=3.0, w_value=2.0, w_key=1.5, shuffle_tools=True):"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.Encoder input: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.@@ -423,7 +436,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_"""cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,- w_name, w_value, w_key)+ w_name, w_value, w_key, shuffle_tools)cache_path = os.path.join(CACHE_DIR, cache_id)tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]@@ -450,6 +463,9 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_tools_texts = [ex["tools"] for ex in ds]ans_texts = [ex["answers"] for ex in ds]+ if shuffle_tools:+ tools_texts = [_shuffle_tools_json(t, seed=i) for i, t in enumerate(tools_texts)]+num_workers = min(os.cpu_count() or 1, 8)model_path = TOKENIZER_PREFIX + ".model"chunk_size = max(1, len(enc_texts) // (num_workers * 4))diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 14cee39..51f607e 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -56,12 +56,13 @@ def tokenize(args):w_name = getattr(args, "w_name", 3.0)w_value = getattr(args, "w_value", 2.0)w_key = getattr(args, "w_key", 1.5)+ shuffle_tools = getattr(args, "shuffle_tools", True)_, _, _, _, kept_indices = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,- w_name=w_name, w_value=w_value, w_key=w_key,+ w_name=w_name, w_value=w_value, w_key=w_key, shuffle_tools=shuffle_tools,)text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,- w_name, w_value, w_key)+ w_name, w_value, w_key, shuffle_tools)_save_cache_metadata(split, text_cache_id, len(kept_indices),max_enc_len, max_dec_len)