Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
tools redesign
HenryNdubuaku8f1d6b328d7a
diff --git a/src/data.py b/src/data.pyindex 3a33e72..0e03c45 100644--- a/src/data.py+++ b/src/data.py@@ -97,8 +97,8 @@ EOS_ID = 1BOS_ID = 2UNK_ID = 3TOOL_CALL_ID = 4-TRANSCRIBE_ID = 5-TOOLS_ID = 6+TOOLS_ID = 5+DEFER_ID = 6DEFAULT_MAX_ENC_LEN = 1024DEFAULT_MAX_DEC_LEN = 512@@ -109,17 +109,15 @@ _unified_dataset_cache = Nonedef _mark_json_value(s, char_w, key, value_str, weight):"""Find '"key": "value_str"' or '"key": value_str' in s, mark value chars."""- # For string values: "key": "value"pattern_str = f'"{_re.escape(key)}"\\s*:\\s*"{_re.escape(value_str)}"'for m in _re.finditer(pattern_str, s):- # Mark only the value part (inside quotes)tail = s[m.start() + len(f'"{key}"'):m.end()]val_offset = tail.index(f'"{value_str}"') + 1val_start = m.start() + len(f'"{key}"') + val_offsetval_end = val_start + len(value_str)char_w[val_start:val_end] = np.maximum(char_w[val_start:val_end], weight)return- # For non-string values: "key": 123 or "key": true etc.+pattern_ns = f'"{_re.escape(key)}"\\s*:\\s*{_re.escape(value_str)}'for m in _re.finditer(pattern_ns, s):colon_offset = s[m.start():m.end()].index(':')@@ -286,14 +284,14 @@ class NeedleTokenizer:def tool_call_token_id(self):return TOOL_CALL_ID- @property- def transcribe_token_id(self):- return TRANSCRIBE_ID-@propertydef tools_token_id(self):return TOOLS_ID+ @property+ def defer_token_id(self):+ return DEFER_ID+@propertydef vocab_size(self):return self.sp.GetPieceSize()@@ -365,7 +363,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):eos_id=EOS_ID,bos_id=BOS_ID,unk_id=UNK_ID,- user_defined_symbols=["<tool_call>", "<transcribe>", "<tools>"],+ user_defined_symbols=["<tool_call>", "<tools>", "<defer>"],byte_fallback=True,normalization_rule_name="identity",num_threads=min(20, max(1, (os.cpu_count() or 1) // 4)),@@ -509,10 +507,21 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_tool_call_id = tokenizer.tool_call_token_idtools_sep_id = tokenizer.tools_token_id+ defer_id = tokenizer.defer_token_id+enc_texts = [ex["query"] for ex in ds]tools_texts = [ex["tools"] for ex in ds]ans_texts = [ex["answers"] for ex in ds]+ # Track which samples have empty tools (will use <defer> token instead)+ empty_tools_mask = []+ for t in tools_texts:+ try:+ parsed = _json.loads(t)+ empty_tools_mask.append(isinstance(parsed, list) and len(parsed) == 0)+ except (ValueError, TypeError):+ empty_tools_mask.append(False)+tool_counts = np.array([_count_tool_calls(a) for a in ans_texts], dtype=np.int32)if shuffle_tools:@@ -545,7 +554,8 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_n = len(ds)- def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):+ def _fill_sample(j, e_tok, t_tok, a_tok, ans_text, is_empty_tools,+ enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):"""Fill arrays at position j for one sample. Returns False if skipped."""al = len(a_tok)# Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1@@ -553,6 +563,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_return False# Encoder: [query..., <tools>, tools...] truncated to max_enc_len+ # For empty tools, use <defer> token instead of tokenized "[]"+ if is_empty_tools:+ t_tok = [defer_id]+# Reserve 1 slot for <tools> separator + at least 1 token for toolsmax_query = max_enc_len - 1 - 1 # room for <tools> + at least 1 tools tokenif len(e_tok) > max_query:@@ -593,8 +607,8 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_skipped = 0for i in tqdm(range(n), desc="Building pairs"):if not _fill_sample(i, all_enc_tokens[i], all_tools_tokens[i],- all_ans_tokens[i], ans_texts[i], enc_inputs,- dec_inputs, dec_targets, loss_mask):+ all_ans_tokens[i], ans_texts[i], empty_tools_mask[i],+ enc_inputs, dec_inputs, dec_targets, loss_mask):skipped += 1if skipped > 0:diff --git a/src/run.py b/src/run.pyindex f6d46c6..20dc1ae 100644--- a/src/run.py+++ b/src/run.py@@ -91,14 +91,30 @@ def load_checkpoint(path):def _build_encoder_input(tokenizer, query, tools, max_enc_len=DEFAULT_MAX_ENC_LEN):- """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""+ """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len.++ Empty tools ([]) are replaced with the <defer> token.+ """tools_sep_id = tokenizer.tools_token_id+ defer_id = tokenizer.defer_token_idq_toks = tokenizer.encode(query)- t_toks = tokenizer.encode(tools)- max_query = max_enc_len - 2++ # Use <defer> token for empty tools+ try:+ parsed = _json.loads(tools)+ is_empty = isinstance(parsed, list) and len(parsed) == 0+ except (ValueError, TypeError):+ is_empty = False++ if is_empty:+ t_toks = [defer_id]+ else:+ t_toks = tokenizer.encode(tools)++ max_query = max_enc_len - 2if len(q_toks) > max_query:q_toks = q_toks[:max_query]- remaining = max_enc_len - len(q_toks) - 1+ remaining = max_enc_len - len(q_toks) - 1t_toks = t_toks[:remaining]return q_toks + [tools_sep_id] + t_toks