Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

tool call

HenryNdubuaku9bc4abf82379
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
index c1498b5..53ccd4c 100755
--- a/scripts/tools_data.py
+++ b/scripts/tools_data.py
@@ -21,9 +21,8 @@ Datasets and their native formats:
3. glaiveai/glaive-function-calling-v2
Tool definitions embedded in the system prompt; chat is a flat string with
- USER:/A: role markers and <functioncall> tags. ~50% of examples are
- irrelevance cases (answers=[]) where the tools don't match the query — these
- are kept intentionally so the model learns to decline rather than hallucinate.
+ USER:/A: role markers and <functioncall> tags. Examples with empty answers
+ (answers=[]) are filtered out in the final combine step.
4. Team-ACE/ToolACE
Tool definitions as a JSON array in the system prompt; function calls use a
@@ -253,8 +252,15 @@ def load_and_combine():
parts.append(converted)
combined = concatenate_datasets(parts)
+ pre_filter = len(combined)
+
+ # Filter out examples with empty answers
+ combined = combined.filter(
+ lambda ex: json.loads(ex["answers"]) != [],
+ desc="Filtering empty answers",
+ )
print(f"\n{'='*60}")
- print(f"Combined dataset: {len(combined)} examples")
+ print(f"Combined dataset: {len(combined)} examples (filtered {pre_filter - len(combined)} empty answers)")
print(f"Columns: {combined.column_names}")
counts = Counter(combined["source"])
diff --git a/src/cli.py b/src/cli.py
index dd4aa88..9fd59e9 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -18,10 +18,10 @@ HELP = """
│ --d-model INT Model dim (default: 512) │
│ --num-heads INT Attention heads (default: 8) │
│ --num-kv-heads INT KV heads for GQA (default: num-heads)│
- │ --num-layers INT Encoder layers (default: 8) │
+ │ --num-layers INT Encoder layers (default: 4) │
│ --num-dec-layers INT Decoder layers (default: 4) │
- │ --max-enc-len INT Max encoder seq len (default: 1024) │
- │ --max-dec-len INT Max decoder seq length (default: 512)│
+ │ --max-enc-len INT Max encoder seq len (default: 256) │
+ │ --max-dec-len INT Max decoder seq len (default: 1024) │
│ --max-samples INT Training samples (default: all) │
│ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │
│ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │
@@ -56,8 +56,8 @@ HELP = """
│ --checkpoint PATH Path to model checkpoint (required) │
│ --batch-size INT Batch size (default: 32) │
│ --max-eval-samples INT Evaluation samples (default: 1000) │
- │ --max-enc-len INT Max encoder length (default: 1024) │
- │ --max-dec-len INT Max decoder length (default: 512) │
+ │ --max-enc-len INT Max encoder length (default: 256) │
+ │ --max-dec-len INT Max decoder length (default: 1024) │
│ --max-gen-len INT Max generation length (default: 512) │
│ --temperature FLOAT Sampling temperature (default: 0.8) │
│ --throughput-runs INT Throughput runs (default: 10) │
@@ -101,10 +101,10 @@ def main():
p.add_argument("--d-model", type=int, default=512)
p.add_argument("--num-heads", type=int, default=8)
p.add_argument("--num-kv-heads", type=int, default=None)
- p.add_argument("--num-layers", type=int, default=8)
+ p.add_argument("--num-layers", type=int, default=4)
p.add_argument("--num-dec-layers", type=int, default=4)
- p.add_argument("--max-enc-len", type=int, default=1024)
- p.add_argument("--max-dec-len", type=int, default=512)
+ p.add_argument("--max-enc-len", type=int, default=256)
+ p.add_argument("--max-dec-len", type=int, default=1024)
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--warmup-ratio", type=float, default=0.05)
p.add_argument("--wandb", action="store_true")
@@ -141,22 +141,19 @@ def main():
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
p.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
- p.add_argument("--prompts", type=str, nargs="*")
p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")
p.add_argument("--max-len", type=int, default=512)
- p.add_argument("--temperature", type=float, default=0.8)
p.add_argument("--seed", type=int, default=0)
p = sub.add_parser("test", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--max-eval-samples", type=int, default=1000)
- p.add_argument("--max-enc-len", type=int, default=1024)
- p.add_argument("--max-dec-len", type=int, default=512)
+ p.add_argument("--max-enc-len", type=int, default=256)
+ p.add_argument("--max-dec-len", type=int, default=1024)
p.add_argument("--max-gen-len", type=int, default=512)
p.add_argument("--tool-call-samples", type=int, default=200,
help="Samples for tool-call accuracy eval (default: 200)")
- p.add_argument("--temperature", type=float, default=0.8)
p.add_argument("--throughput-runs", type=int, default=10)
p = sub.add_parser("evaluate", add_help=False)
diff --git a/src/data.py b/src/data.py
index fc9cf5a..319dbcc 100644
--- a/src/data.py
+++ b/src/data.py
@@ -173,12 +173,36 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):
return hashlib.md5(key.encode()).hexdigest()[:12]
-def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):
+def _compact_tools(tools_json):
+ """Strip verbose schema descriptions, keep only function name + param name/type."""
+ import json
+ try:
+ tools = json.loads(tools_json)
+ except (json.JSONDecodeError, TypeError):
+ return tools_json
+ compact = []
+ for t in tools:
+ params = {}
+ raw_params = t.get("parameters", {})
+ props = raw_params.get("properties", raw_params) if isinstance(raw_params, dict) else {}
+ for pname, pdef in props.items():
+ if isinstance(pdef, dict):
+ params[pname] = pdef.get("type", "string")
+ else:
+ params[pname] = str(pdef)
+ compact.append({"name": t["name"], "parameters": params})
+ return json.dumps(compact)
+
+
+def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.
- Encoder input: tokenize(query + " " + tools), truncated to max_enc_len.
- Decoder input: [BOS, <tool_call>, answer_tokens...]
- Decoder target: [<tool_call>, answer_tokens..., EOS]
+ Encoder input: tokenize(query), truncated to max_enc_len.
+ Decoder input: [BOS, <tool_call>, tools_tokens..., answer_tokens...]
+ Decoder target: [<tool_call>, tools_tokens..., answer_tokens..., EOS]
+ Loss mask: 1 only on answer + EOS positions (not tools prefix or padding).
+
+ Returns (enc_inputs, dec_inputs, dec_targets, loss_mask).
"""
cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)
@@ -189,6 +213,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):
np.load(cache_path + "_enc.npy"),
np.load(cache_path + "_dec_in.npy"),
np.load(cache_path + "_dec_tgt.npy"),
+ np.load(cache_path + "_loss_mask.npy"),
)
pad_id = tokenizer.pad_token_id
@@ -196,79 +221,126 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):
eos_id = tokenizer.eos_token_id
tool_call_id = tokenizer.tool_call_token_id
- # Build encoder texts (query + tools) and decoder texts (answers)
- enc_texts = [ex["query"] + " " + ex["tools"] for ex in ds]
- dec_texts = [ex["answers"] for ex in ds]
+ # Encoder: query only. Decoder prefix: compact tools. Decoder answer: answers.
+ enc_texts = [ex["query"] for ex in ds]
+ tools_texts = [_compact_tools(ex["tools"]) for ex in ds]
+ ans_texts = [ex["answers"] for ex in ds]
num_workers = min(os.cpu_count() or 1, 8)
model_path = TOKENIZER_PREFIX + ".model"
-
- # Parallel tokenization for encoder inputs
chunk_size = max(1, len(enc_texts) // (num_workers * 4))
- enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]
+ # Tokenize queries (encoder)
+ enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]
print(f"Tokenizing encoder inputs ({num_workers} workers)...")
with mp.Pool(num_workers, initializer=_init_worker,
initargs=(model_path, max_enc_len)) as pool:
enc_results = pool.map(_tokenize_chunk, enc_chunks)
all_enc_tokens = [tok for chunk in enc_results for tok in chunk]
- # Parallel tokenization for decoder inputs (max_dec_len - 2 for BOS + task token)
- dec_chunks = [dec_texts[i:i + chunk_size] for i in range(0, len(dec_texts), chunk_size)]
-
- print(f"Tokenizing decoder inputs ({num_workers} workers)...")
+ # Tokenize tools (decoder prefix) — reserve 2 for BOS + <tool_call>, rest for tools + answer
+ tools_chunks = [tools_texts[i:i + chunk_size] for i in range(0, len(tools_texts), chunk_size)]
+ print(f"Tokenizing tools ({num_workers} workers)...")
with mp.Pool(num_workers, initializer=_init_worker,
initargs=(model_path, max_dec_len - 2)) as pool:
- dec_results = pool.map(_tokenize_chunk, dec_chunks)
- all_dec_tokens = [tok for chunk in dec_results for tok in chunk]
+ tools_results = pool.map(_tokenize_chunk, tools_chunks)
+ all_tools_tokens = [tok for chunk in tools_results for tok in chunk]
+
+ # Tokenize answers (decoder generation target)
+ ans_chunks = [ans_texts[i:i + chunk_size] for i in range(0, len(ans_texts), chunk_size)]
+ print(f"Tokenizing answers ({num_workers} workers)...")
+ with mp.Pool(num_workers, initializer=_init_worker,
+ initargs=(model_path, max_dec_len)) as pool:
+ ans_results = pool.map(_tokenize_chunk, ans_chunks)
+ all_ans_tokens = [tok for chunk in ans_results for tok in chunk]
n = len(ds)
enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)
dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)
dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)
+ loss_mask = np.zeros((n, max_dec_len), dtype=np.float32)
+ skipped = 0
for i in range(n):
- # Encoder: truncated tokens
e_tok = all_enc_tokens[i]
+ t_tok = all_tools_tokens[i]
+ a_tok = all_ans_tokens[i]
+
+ # Prefix length: BOS + <tool_call> + tools_tokens
+ prefix_len = 2 + len(t_tok)
+ # Total decoder length: prefix + answer + EOS
+ total_dec = prefix_len + len(a_tok) + 1
+
+ if total_dec > max_dec_len:
+ # Truncate tools to fit answer + EOS
+ available_for_tools = max_dec_len - 2 - len(a_tok) - 1
+ if available_for_tools < 1:
+ skipped += 1
+ continue
+ t_tok = t_tok[:available_for_tools]
+ prefix_len = 2 + len(t_tok)
+ total_dec = prefix_len + len(a_tok) + 1
+
+ # Encoder
el = len(e_tok)
enc_inputs[i, :el] = e_tok
- # Decoder answer tokens
- d_tok = all_dec_tokens[i]
- dl = len(d_tok)
-
- # Decoder input: [BOS, <tool_call>, answer_tokens...]
+ # Decoder input: [BOS, <tool_call>, tools..., answer...]
dec_inputs[i, 0] = bos_id
dec_inputs[i, 1] = tool_call_id
- if dl > 0:
- dec_inputs[i, 2:2 + dl] = d_tok
-
- # Decoder target: [<tool_call>, answer_tokens..., EOS]
+ tl = len(t_tok)
+ if tl > 0:
+ dec_inputs[i, 2:2 + tl] = t_tok
+ al = len(a_tok)
+ if al > 0:
+ dec_inputs[i, prefix_len:prefix_len + al] = a_tok
+
+ # Decoder target: [<tool_call>, tools..., answer..., EOS]
dec_targets[i, 0] = tool_call_id
- if dl > 0:
- dec_targets[i, 1:1 + dl] = d_tok
- dec_targets[i, 1 + dl] = eos_id
+ if tl > 0:
+ dec_targets[i, 1:1 + tl] = t_tok
+ if al > 0:
+ dec_targets[i, prefix_len - 1:prefix_len - 1 + al] = a_tok
+ dec_targets[i, prefix_len - 1 + al] = eos_id
+
+ # Loss mask: 1 on answer positions + EOS only
+ loss_mask[i, prefix_len - 1:prefix_len - 1 + al + 1] = 1.0
+
+ if skipped > 0:
+ # Remove skipped rows (all-pad)
+ keep = enc_inputs[:, 0] != pad_id
+ enc_inputs = enc_inputs[keep]
+ dec_inputs = dec_inputs[keep]
+ dec_targets = dec_targets[keep]
+ loss_mask = loss_mask[keep]
+ print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")
# Save cache
os.makedirs(CACHE_DIR, exist_ok=True)
np.save(cache_path + "_enc.npy", enc_inputs)
np.save(cache_path + "_dec_in.npy", dec_inputs)
np.save(cache_path + "_dec_tgt.npy", dec_targets)
- print(f"Cached {n:,} tool-call pairs to {CACHE_DIR}/{cache_id}")
+ np.save(cache_path + "_loss_mask.npy", loss_mask)
+ print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")
- return enc_inputs, dec_inputs, dec_targets
+ return enc_inputs, dec_inputs, dec_targets, loss_mask
-def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):
+def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None):
n = len(enc_inputs)
if shuffle:
perm = np.random.permutation(n)
enc_inputs = enc_inputs[perm]
dec_inputs = dec_inputs[perm]
dec_targets = dec_targets[perm]
+ if loss_mask is not None:
+ loss_mask = loss_mask[perm]
for i in range(0, n - batch_size + 1, batch_size):
- yield enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]
+ batch = (enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size])
+ if loss_mask is not None:
+ batch = batch + (loss_mask[i : i + batch_size],)
+ yield batch
get_text_batches = get_batches
diff --git a/src/run.py b/src/run.py
index 45842ed..87b2b1d 100644
--- a/src/run.py
+++ b/src/run.py
@@ -24,28 +24,37 @@ def load_checkpoint(path):
return params, config
-def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8, seed=0, stream=True, task_token_id=None):
- enc_tokens = tokenizer.encode(prompt)
+def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=0, stream=True, task_token_id=None):
+ """Generate tool-call output.
+
+ Encoder: query only.
+ Decoder: prefilled with [BOS, <tool_call>, tools_tokens...], then greedy decode.
+ """
+ from .data import _compact_tools
+
+ enc_tokens = tokenizer.encode(query)
enc_input = jnp.array([enc_tokens])
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
+ tool_call_id = task_token_id if task_token_id is not None else tokenizer.tool_call_token_id
src_mask = make_padding_mask(enc_input, pad_id)
encoder_out = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
+ # Build decoder prefix: [BOS, <tool_call>, compact_tools_tokens...]
+ tools_tokens = tokenizer.encode(_compact_tools(tools))
+ prefix = [eos_id, tool_call_id] + tools_tokens
+ prefix_len = len(prefix)
+
# Fixed-size buffer so JAX compiles once
tgt_mask = make_causal_mask(max_gen_len)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
- dec_buffer = dec_buffer.at[0, 0].set(eos_id)
-
- # If task_token_id provided, set it at position 1 (matching transcribe() pattern)
- start_pos = 0
- if task_token_id is not None:
- dec_buffer = dec_buffer.at[0, 1].set(task_token_id)
- start_pos = 1
+ # Fill prefix
+ for j, tok in enumerate(prefix[:max_gen_len]):
+ dec_buffer = dec_buffer.at[0, j].set(tok)
@jax.jit
def decode_step(dec_buffer, encoder_out, src_mask):
@@ -55,11 +64,10 @@ def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8,
encoder_out,
self_mask=tgt_mask,
cross_mask=src_mask,
- method="decode",
+ method="decode",
)
return logits
- rng = jax.random.PRNGKey(seed)
generated_tokens = []
if stream:
@@ -69,10 +77,10 @@ def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8,
# Compile once on first call
logits = decode_step(dec_buffer, encoder_out, src_mask)
- for i in range(start_pos, max_gen_len - 1):
- next_logits = logits[0, i] / temperature
- rng, sample_rng = jax.random.split(rng)
- next_token = jax.random.categorical(sample_rng, next_logits).item()
+ # Greedy decoding from end of prefix
+ for i in range(prefix_len - 1, max_gen_len - 1):
+ next_logits = logits[0, i]
+ next_token = int(jnp.argmax(next_logits))
if next_token == eos_id:
break
@@ -204,42 +212,39 @@ def main(args):
# --- Tool-call generation mode ---
query = getattr(args, "query", None)
- tools = getattr(args, "tools", None)
- prompts = args.prompts
+ tools = getattr(args, "tools", None) or "[]"
if query:
- prompt = query + " " + (tools or "[]")
- prompts = [prompt]
- elif not prompts:
- prompts = [
- 'What is the weather in San Francisco? [{"name": "get_weather", "parameters": {"location": "string"}}]',
- 'Send an email to [email protected] saying hello [{"name": "send_email", "parameters": {"to": "string", "body": "string"}}]',
- 'Get the current stock price of AAPL [{"name": "get_stock_price", "parameters": {"symbol": "string"}}]',
+ queries = [(query, tools)]
+ else:
+ queries = [
+ ('What is the weather in San Francisco?', '[{"name": "get_weather", "parameters": {"location": "string"}}]'),
+ ('Send an email to [email protected] saying hello', '[{"name": "send_email", "parameters": {"to": "string", "body": "string"}}]'),
+ ('Get the current stock price of AAPL', '[{"name": "get_stock_price", "parameters": {"symbol": "string"}}]'),
]
- for i, prompt in enumerate(prompts):
+ for i, (q, t) in enumerate(queries):
+ print(f"\nQuery: {q}")
+ print(f"Tools: {t[:80]}{'...' if len(t) > 80 else ''}")
generate(
model,
params,
tokenizer,
- prompt,
+ q,
+ tools=t,
max_gen_len=args.max_len,
- temperature=args.temperature,
seed=args.seed + i,
stream=True,
- task_token_id=tokenizer.tool_call_token_id,
)
def parse_args():
parser = argparse.ArgumentParser(description="Generate tool calls with trained transformer")
parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
- parser.add_argument("--prompts", type=str, nargs="*", help="Raw prompts (query + tools combined)")
parser.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
parser.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")
parser.add_argument("--max-len", type=int, default=512)
- parser.add_argument("--temperature", type=float, default=0.8)
parser.add_argument("--seed", type=int, default=0)
return parser.parse_args()
diff --git a/src/test.py b/src/test.py
index 01884ef..515b516 100644
--- a/src/test.py
+++ b/src/test.py
@@ -25,12 +25,19 @@ def load_checkpoint(path):
return params, config
-def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id):
+def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id, loss_mask=None):
total_loss = 0.0
total_tokens = 0
- for src, tgt_in, tgt_out in get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=False):
- src, tgt_in, tgt_out = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out)
+ for batch in get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=False, loss_mask=loss_mask):
+ if loss_mask is not None:
+ src, tgt_in, tgt_out, lm = batch
+ src, tgt_in, tgt_out, lm = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out), jnp.array(lm)
+ mask = lm
+ else:
+ src, tgt_in, tgt_out = batch
+ src, tgt_in, tgt_out = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out)
+ mask = (tgt_out != pad_id).astype(jnp.float32)
src_mask = make_padding_mask(src, pad_id)
tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, pad_id)
@@ -39,10 +46,9 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch
logits = model.apply(
{"params": params}, src, tgt_in,
src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- )
+ )
loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)
- mask = (tgt_out != pad_id).astype(jnp.float32)
total_loss += float(jnp.sum(loss * mask))
total_tokens += int(jnp.sum(mask))
@@ -50,7 +56,7 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch
return math.exp(avg_nll)
-def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is the weather? [{"name": "get_weather"}]', max_gen_len=64):
+def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is the weather?', max_gen_len=64):
enc_tokens = tokenizer.encode(prompt)
enc_input = jnp.array([enc_tokens])
pad_id = tokenizer.pad_token_id
@@ -205,13 +211,11 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
samples = []
for i, ex in enumerate(ds):
- prompt = ex["query"] + " " + ex["tools"]
ref_text = ex["answers"]
pred_text = generate(
- model, params, tokenizer, prompt,
- max_gen_len=max_gen_len, temperature=0.0, seed=i, stream=False,
- task_token_id=tokenizer.tool_call_token_id,
+ model, params, tokenizer, ex["query"],
+ tools=ex["tools"], max_gen_len=max_gen_len, seed=i, stream=False,
).strip()
# Parse reference
@@ -331,10 +335,10 @@ def main(args):
print(f"\nevaluating tool-call perplexity ({args.max_eval_samples} samples)...")
ds = load_tool_calls("validation", max_samples=args.max_eval_samples)
- enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(
+ enc_inputs, dec_inputs, dec_targets, loss_mask_arr = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
- ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)
+ ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id, loss_mask=loss_mask_arr)
tc_samples = getattr(args, "tool_call_samples", 200)
tc = None
@@ -385,8 +389,8 @@ def parse_args():
parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--max-eval-samples", type=int, default=1000)
- parser.add_argument("--max-enc-len", type=int, default=1024)
- parser.add_argument("--max-dec-len", type=int, default=512)
+ parser.add_argument("--max-enc-len", type=int, default=256)
+ parser.add_argument("--max-dec-len", type=int, default=1024)
parser.add_argument("--max-gen-len", type=int, default=512)
parser.add_argument("--temperature", type=float, default=0.8)
parser.add_argument("--throughput-runs", type=int, default=10)
diff --git a/src/train.py b/src/train.py
index 91ce210..0484b4f 100644
--- a/src/train.py
+++ b/src/train.py
@@ -229,7 +229,7 @@ _MAT_FF_WIDTHS = () # precomputed d_ff widths per factor
_D_FF = 2048 # set in train()
-def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask):
+def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask):
pad_id = 0
src_mask = make_padding_mask(src, pad_id)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
@@ -240,7 +240,7 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask):
method="forward_masked",
)
logits_f32 = logits.astype(jnp.float32)
- mask = (tgt_out != pad_id).astype(jnp.float32)
+ mask = loss_mask
ce_loss = jnp.sum(
optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
) / jnp.maximum(jnp.sum(mask), 1.0)
@@ -289,10 +289,10 @@ def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):
return jnp.concatenate(rows, axis=0)
-def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask):
+def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask):
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask)
+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@ -302,11 +302,11 @@ def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_m
return state, ema_params, loss, grad_norm
-def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask):
+def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, loss_mask):
"""Text training step with fused prune mask application."""
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask)
+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@ -365,7 +365,7 @@ def _make_p_train_step_speech_masked():
def _make_val_loss_fn(apply_fn):
@jax.jit
- def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):
+ def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):
pad_id = 0
src_mask = make_padding_mask(src, pad_id)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
@@ -374,15 +374,14 @@ def _make_val_loss_fn(apply_fn):
src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,
)
loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
- mask = (tgt_out != pad_id).astype(jnp.float32)
- return jnp.sum(loss * mask), jnp.sum(mask)
+ return jnp.sum(loss * loss_mask), jnp.sum(loss_mask)
return val_loss_batch
def _make_mat_val_loss_fn(apply_fn, ff_width):
"""Val loss for matryoshka sub-model at given FFN width."""
@jax.jit
- def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):
+ def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):
pad_id = 0
src_mask = make_padding_mask(src, pad_id)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
@@ -394,8 +393,7 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):
)
trunc_logits = mat_logits[0].astype(jnp.float32)
loss = optax.softmax_cross_entropy_with_integer_labels(trunc_logits, tgt_out)
- mask = (tgt_out != pad_id).astype(jnp.float32)
- return jnp.sum(loss * mask), jnp.sum(mask)
+ return jnp.sum(loss * loss_mask), jnp.sum(loss_mask)
return val_loss_batch
@@ -480,15 +478,16 @@ def train(args):
step_idx += 1
print(f"\n[{step_idx}/{total_data_steps}] Preparing tool-call pairs...")
- enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(
+ enc_inputs, dec_inputs, dec_targets, train_loss_mask = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
- val_enc, val_dec_in, val_dec_tgt = prepare_tool_call_pairs(
+ val_enc, val_dec_in, val_dec_tgt, val_loss_mask = prepare_tool_call_pairs(
val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
val_enc = np.array(val_enc)
val_dec_in = np.array(val_dec_in)
val_dec_tgt = np.array(val_dec_tgt)
+ val_loss_mask = np.array(val_loss_mask)
print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")
# --- Speech data ---
@@ -671,7 +670,7 @@ def train(args):
text_losses = []
speech_losses = []
- text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size)
+ text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask)
# Speech batches cycle via itertools.cycle
speech_iter = None
@@ -683,7 +682,8 @@ def train(args):
text_step_in_epoch = 0
pbar = tqdm(text_batches, total=text_batches_per_epoch, desc=f"Epoch {epoch + 1}/{args.epochs}")
- for src, tgt_in, tgt_out in pbar:
+ for batch in pbar:
+ src, tgt_in, tgt_out, lm = batch
# --- Text step ---
t0 = time.perf_counter()
@@ -700,18 +700,20 @@ def train(args):
src_b = _tile_for_mat(src)
tgt_in_b = _tile_for_mat(tgt_in)
tgt_out_b = _tile_for_mat(tgt_out)
+ lm_b = _tile_for_mat(lm)
else:
src_b = shard_batch(src, num_devices)
tgt_in_b = shard_batch(tgt_in, num_devices)
tgt_out_b = shard_batch(tgt_out, num_devices)
+ lm_b = shard_batch(lm, num_devices)
if prune_mask is not None:
state, ema_params, loss, grad_norm = p_train_step_masked(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask,
+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, lm_b,
)
else:
state, ema_params, loss, grad_norm = p_train_step(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask,
+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, lm_b,
)
text_loss_val = float(loss[0])
@@ -770,8 +772,8 @@ def train(args):
_eval_params = jax_utils.unreplicate(ema_params)
val_causal = make_causal_mask(args.max_dec_len)
total_loss, total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
- vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal)
+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):
+ vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
total_loss += float(vl)
total_toks += float(vt)
last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
@@ -836,8 +838,8 @@ def train(args):
eval_params = jax_utils.unreplicate(ema_params)
val_causal = make_causal_mask(args.max_dec_len)
total_loss, total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
- vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal)
+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):
+ vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
total_loss += float(vl)
total_toks += float(vt)
last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
@@ -854,8 +856,8 @@ def train(args):
q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)
q_total_loss, q_total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
- vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal)
+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):
+ vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal, vb[3])
q_total_loss += float(vl)
q_total_toks += float(vt)
quant_val_ppl = float(math.exp(q_total_loss / max(q_total_toks, 1)))
@@ -868,8 +870,8 @@ def train(args):
for factor, ff_w in zip(_MAT_FACTORS, _MAT_FF_WIDTHS):
mat_vl_fn = _make_mat_val_loss_fn(apply_fn, ff_w)
mat_total_loss, mat_total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
- vl, vt = mat_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal)
+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):
+ vl, vt = mat_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
mat_total_loss += float(vl)
mat_total_toks += float(vt)
avg_loss = mat_total_loss / max(mat_total_toks, 1)