Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
tool call
HenryNdubuaku9bc4abf82379
diff --git a/scripts/tools_data.py b/scripts/tools_data.pyindex c1498b5..53ccd4c 100755--- a/scripts/tools_data.py+++ b/scripts/tools_data.py@@ -21,9 +21,8 @@ Datasets and their native formats:3. glaiveai/glaive-function-calling-v2Tool definitions embedded in the system prompt; chat is a flat string with- USER:/A: role markers and <functioncall> tags. ~50% of examples are- irrelevance cases (answers=[]) where the tools don't match the query — these- are kept intentionally so the model learns to decline rather than hallucinate.+ USER:/A: role markers and <functioncall> tags. Examples with empty answers+ (answers=[]) are filtered out in the final combine step.4. Team-ACE/ToolACETool definitions as a JSON array in the system prompt; function calls use a@@ -253,8 +252,15 @@ def load_and_combine():parts.append(converted)combined = concatenate_datasets(parts)+ pre_filter = len(combined)++ # Filter out examples with empty answers+ combined = combined.filter(+ lambda ex: json.loads(ex["answers"]) != [],+ desc="Filtering empty answers",+ )print(f"\n{'='*60}")- print(f"Combined dataset: {len(combined)} examples")+ print(f"Combined dataset: {len(combined)} examples (filtered {pre_filter - len(combined)} empty answers)")print(f"Columns: {combined.column_names}")counts = Counter(combined["source"])diff --git a/src/cli.py b/src/cli.pyindex dd4aa88..9fd59e9 100644--- a/src/cli.py+++ b/src/cli.py@@ -18,10 +18,10 @@ HELP = """│ --d-model INT Model dim (default: 512) ││ --num-heads INT Attention heads (default: 8) ││ --num-kv-heads INT KV heads for GQA (default: num-heads)│- │ --num-layers INT Encoder layers (default: 8) │+ │ --num-layers INT Encoder layers (default: 4) ││ --num-dec-layers INT Decoder layers (default: 4) │- │ --max-enc-len INT Max encoder seq len (default: 1024) │- │ --max-dec-len INT Max decoder seq length (default: 512)│+ │ --max-enc-len INT Max encoder seq len (default: 256) │+ │ --max-dec-len INT Max decoder seq len (default: 1024) ││ --max-samples INT Training samples (default: all) ││ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) ││ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │@@ -56,8 +56,8 @@ HELP = """│ --checkpoint PATH Path to model checkpoint (required) ││ --batch-size INT Batch size (default: 32) ││ --max-eval-samples INT Evaluation samples (default: 1000) │- │ --max-enc-len INT Max encoder length (default: 1024) │- │ --max-dec-len INT Max decoder length (default: 512) │+ │ --max-enc-len INT Max encoder length (default: 256) │+ │ --max-dec-len INT Max decoder length (default: 1024) ││ --max-gen-len INT Max generation length (default: 512) ││ --temperature FLOAT Sampling temperature (default: 0.8) ││ --throughput-runs INT Throughput runs (default: 10) │@@ -101,10 +101,10 @@ def main():p.add_argument("--d-model", type=int, default=512)p.add_argument("--num-heads", type=int, default=8)p.add_argument("--num-kv-heads", type=int, default=None)- p.add_argument("--num-layers", type=int, default=8)+ p.add_argument("--num-layers", type=int, default=4)p.add_argument("--num-dec-layers", type=int, default=4)- p.add_argument("--max-enc-len", type=int, default=1024)- p.add_argument("--max-dec-len", type=int, default=512)+ p.add_argument("--max-enc-len", type=int, default=256)+ p.add_argument("--max-dec-len", type=int, default=1024)p.add_argument("--max-samples", type=int, default=None)p.add_argument("--warmup-ratio", type=float, default=0.05)p.add_argument("--wandb", action="store_true")@@ -141,22 +141,19 @@ def main():p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")p.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")- p.add_argument("--prompts", type=str, nargs="*")p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")p.add_argument("--max-len", type=int, default=512)- p.add_argument("--temperature", type=float, default=0.8)p.add_argument("--seed", type=int, default=0)p = sub.add_parser("test", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--max-eval-samples", type=int, default=1000)- p.add_argument("--max-enc-len", type=int, default=1024)- p.add_argument("--max-dec-len", type=int, default=512)+ p.add_argument("--max-enc-len", type=int, default=256)+ p.add_argument("--max-dec-len", type=int, default=1024)p.add_argument("--max-gen-len", type=int, default=512)p.add_argument("--tool-call-samples", type=int, default=200,help="Samples for tool-call accuracy eval (default: 200)")- p.add_argument("--temperature", type=float, default=0.8)p.add_argument("--throughput-runs", type=int, default=10)p = sub.add_parser("evaluate", add_help=False)diff --git a/src/data.py b/src/data.pyindex fc9cf5a..319dbcc 100644--- a/src/data.py+++ b/src/data.py@@ -173,12 +173,36 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):return hashlib.md5(key.encode()).hexdigest()[:12]-def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):+def _compact_tools(tools_json):+ """Strip verbose schema descriptions, keep only function name + param name/type."""+ import json+ try:+ tools = json.loads(tools_json)+ except (json.JSONDecodeError, TypeError):+ return tools_json+ compact = []+ for t in tools:+ params = {}+ raw_params = t.get("parameters", {})+ props = raw_params.get("properties", raw_params) if isinstance(raw_params, dict) else {}+ for pname, pdef in props.items():+ if isinstance(pdef, dict):+ params[pname] = pdef.get("type", "string")+ else:+ params[pname] = str(pdef)+ compact.append({"name": t["name"], "parameters": params})+ return json.dumps(compact)+++def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.- Encoder input: tokenize(query + " " + tools), truncated to max_enc_len.- Decoder input: [BOS, <tool_call>, answer_tokens...]- Decoder target: [<tool_call>, answer_tokens..., EOS]+ Encoder input: tokenize(query), truncated to max_enc_len.+ Decoder input: [BOS, <tool_call>, tools_tokens..., answer_tokens...]+ Decoder target: [<tool_call>, tools_tokens..., answer_tokens..., EOS]+ Loss mask: 1 only on answer + EOS positions (not tools prefix or padding).++ Returns (enc_inputs, dec_inputs, dec_targets, loss_mask)."""cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)@@ -189,6 +213,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):np.load(cache_path + "_enc.npy"),np.load(cache_path + "_dec_in.npy"),np.load(cache_path + "_dec_tgt.npy"),+ np.load(cache_path + "_loss_mask.npy"),)pad_id = tokenizer.pad_token_id@@ -196,79 +221,126 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):eos_id = tokenizer.eos_token_idtool_call_id = tokenizer.tool_call_token_id- # Build encoder texts (query + tools) and decoder texts (answers)- enc_texts = [ex["query"] + " " + ex["tools"] for ex in ds]- dec_texts = [ex["answers"] for ex in ds]+ # Encoder: query only. Decoder prefix: compact tools. Decoder answer: answers.+ enc_texts = [ex["query"] for ex in ds]+ tools_texts = [_compact_tools(ex["tools"]) for ex in ds]+ ans_texts = [ex["answers"] for ex in ds]num_workers = min(os.cpu_count() or 1, 8)model_path = TOKENIZER_PREFIX + ".model"-- # Parallel tokenization for encoder inputschunk_size = max(1, len(enc_texts) // (num_workers * 4))- enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]+ # Tokenize queries (encoder)+ enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]print(f"Tokenizing encoder inputs ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,initargs=(model_path, max_enc_len)) as pool:enc_results = pool.map(_tokenize_chunk, enc_chunks)all_enc_tokens = [tok for chunk in enc_results for tok in chunk]- # Parallel tokenization for decoder inputs (max_dec_len - 2 for BOS + task token)- dec_chunks = [dec_texts[i:i + chunk_size] for i in range(0, len(dec_texts), chunk_size)]-- print(f"Tokenizing decoder inputs ({num_workers} workers)...")+ # Tokenize tools (decoder prefix) — reserve 2 for BOS + <tool_call>, rest for tools + answer+ tools_chunks = [tools_texts[i:i + chunk_size] for i in range(0, len(tools_texts), chunk_size)]+ print(f"Tokenizing tools ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,initargs=(model_path, max_dec_len - 2)) as pool:- dec_results = pool.map(_tokenize_chunk, dec_chunks)- all_dec_tokens = [tok for chunk in dec_results for tok in chunk]+ tools_results = pool.map(_tokenize_chunk, tools_chunks)+ all_tools_tokens = [tok for chunk in tools_results for tok in chunk]++ # Tokenize answers (decoder generation target)+ ans_chunks = [ans_texts[i:i + chunk_size] for i in range(0, len(ans_texts), chunk_size)]+ print(f"Tokenizing answers ({num_workers} workers)...")+ with mp.Pool(num_workers, initializer=_init_worker,+ initargs=(model_path, max_dec_len)) as pool:+ ans_results = pool.map(_tokenize_chunk, ans_chunks)+ all_ans_tokens = [tok for chunk in ans_results for tok in chunk]n = len(ds)enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)+ loss_mask = np.zeros((n, max_dec_len), dtype=np.float32)+ skipped = 0for i in range(n):- # Encoder: truncated tokense_tok = all_enc_tokens[i]+ t_tok = all_tools_tokens[i]+ a_tok = all_ans_tokens[i]++ # Prefix length: BOS + <tool_call> + tools_tokens+ prefix_len = 2 + len(t_tok)+ # Total decoder length: prefix + answer + EOS+ total_dec = prefix_len + len(a_tok) + 1++ if total_dec > max_dec_len:+ # Truncate tools to fit answer + EOS+ available_for_tools = max_dec_len - 2 - len(a_tok) - 1+ if available_for_tools < 1:+ skipped += 1+ continue+ t_tok = t_tok[:available_for_tools]+ prefix_len = 2 + len(t_tok)+ total_dec = prefix_len + len(a_tok) + 1++ # Encoderel = len(e_tok)enc_inputs[i, :el] = e_tok- # Decoder answer tokens- d_tok = all_dec_tokens[i]- dl = len(d_tok)-- # Decoder input: [BOS, <tool_call>, answer_tokens...]+ # Decoder input: [BOS, <tool_call>, tools..., answer...]dec_inputs[i, 0] = bos_iddec_inputs[i, 1] = tool_call_id- if dl > 0:- dec_inputs[i, 2:2 + dl] = d_tok-- # Decoder target: [<tool_call>, answer_tokens..., EOS]+ tl = len(t_tok)+ if tl > 0:+ dec_inputs[i, 2:2 + tl] = t_tok+ al = len(a_tok)+ if al > 0:+ dec_inputs[i, prefix_len:prefix_len + al] = a_tok++ # Decoder target: [<tool_call>, tools..., answer..., EOS]dec_targets[i, 0] = tool_call_id- if dl > 0:- dec_targets[i, 1:1 + dl] = d_tok- dec_targets[i, 1 + dl] = eos_id+ if tl > 0:+ dec_targets[i, 1:1 + tl] = t_tok+ if al > 0:+ dec_targets[i, prefix_len - 1:prefix_len - 1 + al] = a_tok+ dec_targets[i, prefix_len - 1 + al] = eos_id++ # Loss mask: 1 on answer positions + EOS only+ loss_mask[i, prefix_len - 1:prefix_len - 1 + al + 1] = 1.0++ if skipped > 0:+ # Remove skipped rows (all-pad)+ keep = enc_inputs[:, 0] != pad_id+ enc_inputs = enc_inputs[keep]+ dec_inputs = dec_inputs[keep]+ dec_targets = dec_targets[keep]+ loss_mask = loss_mask[keep]+ print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")# Save cacheos.makedirs(CACHE_DIR, exist_ok=True)np.save(cache_path + "_enc.npy", enc_inputs)np.save(cache_path + "_dec_in.npy", dec_inputs)np.save(cache_path + "_dec_tgt.npy", dec_targets)- print(f"Cached {n:,} tool-call pairs to {CACHE_DIR}/{cache_id}")+ np.save(cache_path + "_loss_mask.npy", loss_mask)+ print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")- return enc_inputs, dec_inputs, dec_targets+ return enc_inputs, dec_inputs, dec_targets, loss_mask-def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):+def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None):n = len(enc_inputs)if shuffle:perm = np.random.permutation(n)enc_inputs = enc_inputs[perm]dec_inputs = dec_inputs[perm]dec_targets = dec_targets[perm]+ if loss_mask is not None:+ loss_mask = loss_mask[perm]for i in range(0, n - batch_size + 1, batch_size):- yield enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]+ batch = (enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size])+ if loss_mask is not None:+ batch = batch + (loss_mask[i : i + batch_size],)+ yield batchget_text_batches = get_batchesdiff --git a/src/run.py b/src/run.pyindex 45842ed..87b2b1d 100644--- a/src/run.py+++ b/src/run.py@@ -24,28 +24,37 @@ def load_checkpoint(path):return params, config-def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8, seed=0, stream=True, task_token_id=None):- enc_tokens = tokenizer.encode(prompt)+def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=0, stream=True, task_token_id=None):+ """Generate tool-call output.++ Encoder: query only.+ Decoder: prefilled with [BOS, <tool_call>, tools_tokens...], then greedy decode.+ """+ from .data import _compact_tools++ enc_tokens = tokenizer.encode(query)enc_input = jnp.array([enc_tokens])pad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_id+ tool_call_id = task_token_id if task_token_id is not None else tokenizer.tool_call_token_idsrc_mask = make_padding_mask(enc_input, pad_id)encoder_out = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")+ # Build decoder prefix: [BOS, <tool_call>, compact_tools_tokens...]+ tools_tokens = tokenizer.encode(_compact_tools(tools))+ prefix = [eos_id, tool_call_id] + tools_tokens+ prefix_len = len(prefix)+# Fixed-size buffer so JAX compiles oncetgt_mask = make_causal_mask(max_gen_len)dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)- dec_buffer = dec_buffer.at[0, 0].set(eos_id)-- # If task_token_id provided, set it at position 1 (matching transcribe() pattern)- start_pos = 0- if task_token_id is not None:- dec_buffer = dec_buffer.at[0, 1].set(task_token_id)- start_pos = 1+ # Fill prefix+ for j, tok in enumerate(prefix[:max_gen_len]):+ dec_buffer = dec_buffer.at[0, j].set(tok)@jax.jitdef decode_step(dec_buffer, encoder_out, src_mask):@@ -55,11 +64,10 @@ def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8,encoder_out,self_mask=tgt_mask,cross_mask=src_mask,- method="decode",+ method="decode",)return logits- rng = jax.random.PRNGKey(seed)generated_tokens = []if stream:@@ -69,10 +77,10 @@ def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8,# Compile once on first calllogits = decode_step(dec_buffer, encoder_out, src_mask)- for i in range(start_pos, max_gen_len - 1):- next_logits = logits[0, i] / temperature- rng, sample_rng = jax.random.split(rng)- next_token = jax.random.categorical(sample_rng, next_logits).item()+ # Greedy decoding from end of prefix+ for i in range(prefix_len - 1, max_gen_len - 1):+ next_logits = logits[0, i]+ next_token = int(jnp.argmax(next_logits))if next_token == eos_id:break@@ -204,42 +212,39 @@ def main(args):# --- Tool-call generation mode ---query = getattr(args, "query", None)- tools = getattr(args, "tools", None)- prompts = args.prompts+ tools = getattr(args, "tools", None) or "[]"if query:- prompt = query + " " + (tools or "[]")- prompts = [prompt]- elif not prompts:- prompts = [- 'What is the weather in San Francisco? [{"name": "get_weather", "parameters": {"location": "string"}}]',- 'Send an email to [email protected] saying hello [{"name": "send_email", "parameters": {"to": "string", "body": "string"}}]',- 'Get the current stock price of AAPL [{"name": "get_stock_price", "parameters": {"symbol": "string"}}]',+ queries = [(query, tools)]+ else:+ queries = [+ ('What is the weather in San Francisco?', '[{"name": "get_weather", "parameters": {"location": "string"}}]'),+ ('Send an email to [email protected] saying hello', '[{"name": "send_email", "parameters": {"to": "string", "body": "string"}}]'),+ ('Get the current stock price of AAPL', '[{"name": "get_stock_price", "parameters": {"symbol": "string"}}]'),]- for i, prompt in enumerate(prompts):+ for i, (q, t) in enumerate(queries):+ print(f"\nQuery: {q}")+ print(f"Tools: {t[:80]}{'...' if len(t) > 80 else ''}")generate(model,params,tokenizer,- prompt,+ q,+ tools=t,max_gen_len=args.max_len,- temperature=args.temperature,seed=args.seed + i,stream=True,- task_token_id=tokenizer.tool_call_token_id,)def parse_args():parser = argparse.ArgumentParser(description="Generate tool calls with trained transformer")parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")- parser.add_argument("--prompts", type=str, nargs="*", help="Raw prompts (query + tools combined)")parser.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")parser.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")parser.add_argument("--max-len", type=int, default=512)- parser.add_argument("--temperature", type=float, default=0.8)parser.add_argument("--seed", type=int, default=0)return parser.parse_args()diff --git a/src/test.py b/src/test.pyindex 01884ef..515b516 100644--- a/src/test.py+++ b/src/test.py@@ -25,12 +25,19 @@ def load_checkpoint(path):return params, config-def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id):+def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id, loss_mask=None):total_loss = 0.0total_tokens = 0- for src, tgt_in, tgt_out in get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=False):- src, tgt_in, tgt_out = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out)+ for batch in get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=False, loss_mask=loss_mask):+ if loss_mask is not None:+ src, tgt_in, tgt_out, lm = batch+ src, tgt_in, tgt_out, lm = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out), jnp.array(lm)+ mask = lm+ else:+ src, tgt_in, tgt_out = batch+ src, tgt_in, tgt_out = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out)+ mask = (tgt_out != pad_id).astype(jnp.float32)src_mask = make_padding_mask(src, pad_id)tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, pad_id)@@ -39,10 +46,9 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batchlogits = model.apply({"params": params}, src, tgt_in,src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,- )+ )loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)- mask = (tgt_out != pad_id).astype(jnp.float32)total_loss += float(jnp.sum(loss * mask))total_tokens += int(jnp.sum(mask))@@ -50,7 +56,7 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batchreturn math.exp(avg_nll)-def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is the weather? [{"name": "get_weather"}]', max_gen_len=64):+def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is the weather?', max_gen_len=64):enc_tokens = tokenizer.encode(prompt)enc_input = jnp.array([enc_tokens])pad_id = tokenizer.pad_token_id@@ -205,13 +211,11 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=samples = []for i, ex in enumerate(ds):- prompt = ex["query"] + " " + ex["tools"]ref_text = ex["answers"]pred_text = generate(- model, params, tokenizer, prompt,- max_gen_len=max_gen_len, temperature=0.0, seed=i, stream=False,- task_token_id=tokenizer.tool_call_token_id,+ model, params, tokenizer, ex["query"],+ tools=ex["tools"], max_gen_len=max_gen_len, seed=i, stream=False,).strip()# Parse reference@@ -331,10 +335,10 @@ def main(args):print(f"\nevaluating tool-call perplexity ({args.max_eval_samples} samples)...")ds = load_tool_calls("validation", max_samples=args.max_eval_samples)- enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(+ enc_inputs, dec_inputs, dec_targets, loss_mask_arr = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)- ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)+ ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id, loss_mask=loss_mask_arr)tc_samples = getattr(args, "tool_call_samples", 200)tc = None@@ -385,8 +389,8 @@ def parse_args():parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")parser.add_argument("--batch-size", type=int, default=32)parser.add_argument("--max-eval-samples", type=int, default=1000)- parser.add_argument("--max-enc-len", type=int, default=1024)- parser.add_argument("--max-dec-len", type=int, default=512)+ parser.add_argument("--max-enc-len", type=int, default=256)+ parser.add_argument("--max-dec-len", type=int, default=1024)parser.add_argument("--max-gen-len", type=int, default=512)parser.add_argument("--temperature", type=float, default=0.8)parser.add_argument("--throughput-runs", type=int, default=10)diff --git a/src/train.py b/src/train.pyindex 91ce210..0484b4f 100644--- a/src/train.py+++ b/src/train.py@@ -229,7 +229,7 @@ _MAT_FF_WIDTHS = () # precomputed d_ff widths per factor_D_FF = 2048 # set in train()-def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask):+def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask):pad_id = 0src_mask = make_padding_mask(src, pad_id)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)@@ -240,7 +240,7 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask):method="forward_masked",)logits_f32 = logits.astype(jnp.float32)- mask = (tgt_out != pad_id).astype(jnp.float32)+ mask = loss_maskce_loss = jnp.sum(optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask) / jnp.maximum(jnp.sum(mask), 1.0)@@ -289,10 +289,10 @@ def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):return jnp.concatenate(rows, axis=0)-def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask):+def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask):ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask)+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@ -302,11 +302,11 @@ def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mreturn state, ema_params, loss, grad_norm-def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask):+def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, loss_mask):"""Text training step with fused prune mask application."""ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask)+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@ -365,7 +365,7 @@ def _make_p_train_step_speech_masked():def _make_val_loss_fn(apply_fn):@jax.jit- def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):+ def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):pad_id = 0src_mask = make_padding_mask(src, pad_id)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)@@ -374,15 +374,14 @@ def _make_val_loss_fn(apply_fn):src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,)loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)- mask = (tgt_out != pad_id).astype(jnp.float32)- return jnp.sum(loss * mask), jnp.sum(mask)+ return jnp.sum(loss * loss_mask), jnp.sum(loss_mask)return val_loss_batchdef _make_mat_val_loss_fn(apply_fn, ff_width):"""Val loss for matryoshka sub-model at given FFN width."""@jax.jit- def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):+ def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):pad_id = 0src_mask = make_padding_mask(src, pad_id)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)@@ -394,8 +393,7 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):)trunc_logits = mat_logits[0].astype(jnp.float32)loss = optax.softmax_cross_entropy_with_integer_labels(trunc_logits, tgt_out)- mask = (tgt_out != pad_id).astype(jnp.float32)- return jnp.sum(loss * mask), jnp.sum(mask)+ return jnp.sum(loss * loss_mask), jnp.sum(loss_mask)return val_loss_batch@@ -480,15 +478,16 @@ def train(args):step_idx += 1print(f"\n[{step_idx}/{total_data_steps}] Preparing tool-call pairs...")- enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(+ enc_inputs, dec_inputs, dec_targets, train_loss_mask = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)- val_enc, val_dec_in, val_dec_tgt = prepare_tool_call_pairs(+ val_enc, val_dec_in, val_dec_tgt, val_loss_mask = prepare_tool_call_pairs(val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)val_enc = np.array(val_enc)val_dec_in = np.array(val_dec_in)val_dec_tgt = np.array(val_dec_tgt)+ val_loss_mask = np.array(val_loss_mask)print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")# --- Speech data ---@@ -671,7 +670,7 @@ def train(args):text_losses = []speech_losses = []- text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size)+ text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask)# Speech batches cycle via itertools.cyclespeech_iter = None@@ -683,7 +682,8 @@ def train(args):text_step_in_epoch = 0pbar = tqdm(text_batches, total=text_batches_per_epoch, desc=f"Epoch {epoch + 1}/{args.epochs}")- for src, tgt_in, tgt_out in pbar:+ for batch in pbar:+ src, tgt_in, tgt_out, lm = batch# --- Text step ---t0 = time.perf_counter()@@ -700,18 +700,20 @@ def train(args):src_b = _tile_for_mat(src)tgt_in_b = _tile_for_mat(tgt_in)tgt_out_b = _tile_for_mat(tgt_out)+ lm_b = _tile_for_mat(lm)else:src_b = shard_batch(src, num_devices)tgt_in_b = shard_batch(tgt_in, num_devices)tgt_out_b = shard_batch(tgt_out, num_devices)+ lm_b = shard_batch(lm, num_devices)if prune_mask is not None:state, ema_params, loss, grad_norm = p_train_step_masked(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask,+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, lm_b,)else:state, ema_params, loss, grad_norm = p_train_step(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask,+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, lm_b,)text_loss_val = float(loss[0])@@ -770,8 +772,8 @@ def train(args):_eval_params = jax_utils.unreplicate(ema_params)val_causal = make_causal_mask(args.max_dec_len)total_loss, total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):- vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal)+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):+ vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])total_loss += float(vl)total_toks += float(vt)last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))@@ -836,8 +838,8 @@ def train(args):eval_params = jax_utils.unreplicate(ema_params)val_causal = make_causal_mask(args.max_dec_len)total_loss, total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):- vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal)+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):+ vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])total_loss += float(vl)total_toks += float(vt)last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))@@ -854,8 +856,8 @@ def train(args):q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)q_total_loss, q_total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):- vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal)+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):+ vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal, vb[3])q_total_loss += float(vl)q_total_toks += float(vt)quant_val_ppl = float(math.exp(q_total_loss / max(q_total_toks, 1)))@@ -868,8 +870,8 @@ def train(args):for factor, ff_w in zip(_MAT_FACTORS, _MAT_FF_WIDTHS):mat_vl_fn = _make_mat_val_loss_fn(apply_fn, ff_w)mat_total_loss, mat_total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):- vl, vt = mat_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal)+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):+ vl, vt = mat_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])mat_total_loss += float(vl)mat_total_toks += float(vt)avg_loss = mat_total_loss / max(mat_total_toks, 1)