Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku9baeb465deac
diff --git a/src/data.py b/src/data.py
index 6069cfd..a54e0fd 100644
--- a/src/data.py
+++ b/src/data.py
@@ -345,7 +345,6 @@ def _download_tokenizer_from_hf():
local = snapshot_download(
_HF_TOKENIZER_REPO,
repo_type="dataset",
- token=True,
)
os.makedirs(TOKENIZER_DIR, exist_ok=True)
for fname in os.listdir(local):
@@ -359,12 +358,8 @@ def _download_tokenizer_from_hf():
def get_tokenizer(max_samples=None):
model_path = TOKENIZER_PREFIX + ".model"
if not os.path.exists(model_path):
- try:
- _download_tokenizer_from_hf()
- except Exception:
- pass
- if not os.path.exists(model_path):
- train_tokenizer(max_samples=max_samples)
+ print("Downloading pretrained tokenizer from HuggingFace...")
+ _download_tokenizer_from_hf()
return NeedleTokenizer(model_path)
@@ -423,7 +418,7 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len,
return hashlib.md5(key.encode()).hexdigest()[:12]
-def _save_cache_metadata(split, text_cache_id, n_samples, max_enc_len, max_dec_len):
+def _save_cache_metadata(split, text_cache_id, n_samples, max_enc_len, max_dec_len, max_tool_len=256):
"""Save metadata JSON for a split locally."""
os.makedirs(CACHE_DIR, exist_ok=True)
meta = {
@@ -432,6 +427,7 @@ def _save_cache_metadata(split, text_cache_id, n_samples, max_enc_len, max_dec_l
"n_samples": n_samples,
"max_enc_len": max_enc_len,
"max_dec_len": max_dec_len,
+ "max_tool_len": max_tool_len,
}
meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")
with open(meta_path, "w") as f:
@@ -451,37 +447,29 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
w_name=3.0, w_value=2.0, w_key=1.5, shuffle_tools=True):
"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.
- Encoder input: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
- Decoder input: [EOS, <tool_call>, answer_tokens..., PAD...]
- Decoder target: [<tool_call>, answer_tokens..., EOS, PAD...]
- Loss mask: token-level weights based on JSON structure (tool names, arg keys/values).
+ Stores variable-length sequences as flat int16/float16 arrays with offsets
+ for compact on-disk representation (~5-7x smaller than padded int32).
+ Padding to fixed max lengths happens at batch time via VarLenArray.
- Returns (enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts).
+ Returns (enc, dec_in, dec_tgt, loss, kept_indices, tool_counts) where
+ the first four are VarLenArray objects when loaded from cache.
"""
cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
w_name, w_value, w_key, shuffle_tools)
cache_path = os.path.join(CACHE_DIR, cache_id)
- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy", "_tool_count.npy"]
-
- def _load_tc_cache():
+ if os.path.exists(cache_path + "_enc_data.npy"):
+ print(f"Loading cached tool-call data ({cache_id})...")
+ enc = VarLenArray.load(cache_path + "_enc", max_enc_len, pad_value=PAD_ID)
+ dec_in = VarLenArray.load(cache_path + "_dec_in", max_dec_len, pad_value=PAD_ID)
+ dec_tgt = VarLenArray.load(cache_path + "_dec_tgt", max_dec_len, pad_value=PAD_ID)
+ loss = VarLenArray.load(cache_path + "_loss", max_dec_len, pad_value=0)
+ kept = np.load(cache_path + "_kept_idx.npy")
tc_path = cache_path + "_tool_count.npy"
tc = np.load(tc_path) if os.path.exists(tc_path) else None
- return (
- np.load(cache_path + "_enc.npy"),
- np.load(cache_path + "_dec_in.npy"),
- np.load(cache_path + "_dec_tgt.npy"),
- np.load(cache_path + "_loss_mask.npy"),
- np.load(cache_path + "_kept_idx.npy"),
- tc,
- )
+ return enc, dec_in, dec_tgt, loss, kept, tc
- if os.path.exists(cache_path + "_enc.npy"):
- print(f"Loading cached tool-call data ({cache_id})...")
- return _load_tc_cache()
-
- pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
tool_call_id = tokenizer.tool_call_token_id
tools_sep_id = tokenizer.tools_token_id
@@ -530,107 +518,87 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
all_ans_tokens = [tok for chunk in ans_results for tok in chunk]
n = len(ds)
+ enc_seqs = []
+ dec_in_seqs = []
+ dec_tgt_seqs = []
+ loss_seqs = []
+ query_lens_list = []
+ kept_indices = []
+ kept_tool_counts = []
+
+ for i in tqdm(range(n), desc="Building pairs"):
+ e_tok = all_enc_tokens[i]
+ t_tok = all_tools_tokens[i]
+ a_tok = all_ans_tokens[i]
- def _fill_sample(j, e_tok, t_tok, a_tok, ans_text,
- enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):
- """Fill arrays at position j for one sample. Returns False if skipped."""
# Encoder: [query..., <tools>, tools...] truncated to max_enc_len
max_query = max_enc_len - 1 - 1 # room for <tools> + at least 1 tools token
if len(e_tok) > max_query:
e_tok = e_tok[:max_query]
- remaining = max_enc_len - len(e_tok) - 1 # 1 for <tools>
+ remaining = max_enc_len - len(e_tok) - 1
t_trunc = t_tok[:remaining]
-
enc_seq = e_tok + [tools_sep_id] + t_trunc
- el = len(enc_seq)
- enc_arr[j, :el] = enc_seq
al = len(a_tok)
- # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1
if 2 + al + 1 > max_dec_len:
- return False
+ continue # skip samples too long for decoder
# Decoder input: [EOS, <tool_call>, answer...]
- dec_in_arr[j, 0] = eos_id
- dec_in_arr[j, 1] = tool_call_id
- if al > 0:
- dec_in_arr[j, 2:2 + al] = a_tok
-
+ dec_in_seq = [eos_id, tool_call_id] + list(a_tok)
# Decoder target: [<tool_call>, answer..., EOS]
- dec_tgt_arr[j, 0] = tool_call_id
- if al > 0:
- dec_tgt_arr[j, 1:1 + al] = a_tok
- dec_tgt_arr[j, 1 + al] = eos_id
+ dec_tgt_seq = [tool_call_id] + list(a_tok) + [eos_id]
# Token-level loss weighting
- lm_arr[j, 0] = 1.0 # <tool_call> prediction
- token_weights = _token_weights_for_answer(ans_text, a_tok, tokenizer.sp,
+ weights = np.ones(len(dec_tgt_seq), dtype=np.float16)
+ token_weights = _token_weights_for_answer(ans_texts[i], a_tok, tokenizer.sp,
w_name=w_name, w_value=w_value, w_key=w_key)
- lm_arr[j, 1:1 + len(token_weights)] = token_weights
- lm_arr[j, 1 + al] = 1.0 # EOS stays at baseline weight
- return True
-
- os.makedirs(CACHE_DIR, exist_ok=True)
-
- enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)
- dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)
- dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)
- loss_mask = np.zeros((n, max_dec_len), dtype=np.float32)
-
- skipped = 0
- for i in tqdm(range(n), desc="Building pairs"):
- if not _fill_sample(i, all_enc_tokens[i], all_tools_tokens[i],
- all_ans_tokens[i], ans_texts[i],
- enc_inputs, dec_inputs, dec_targets, loss_mask):
- skipped += 1
-
+ weights[1:1 + len(token_weights)] = token_weights.astype(np.float16)
+ weights[1 + al] = 1.0 # EOS at baseline
+
+ enc_seqs.append(np.array(enc_seq, dtype=np.int16))
+ dec_in_seqs.append(np.array(dec_in_seq, dtype=np.int16))
+ dec_tgt_seqs.append(np.array(dec_tgt_seq, dtype=np.int16))
+ loss_seqs.append(weights)
+ query_lens_list.append(len(e_tok))
+ kept_indices.append(i)
+ kept_tool_counts.append(tool_counts[i])
+
+ skipped = n - len(kept_indices)
if skipped > 0:
- keep = enc_inputs[:, 0] != pad_id
- kept_indices = np.where(keep)[0].astype(np.int64)
- enc_inputs = enc_inputs[keep]
- dec_inputs = dec_inputs[keep]
- dec_targets = dec_targets[keep]
- loss_mask = loss_mask[keep]
- tool_counts = tool_counts[keep]
print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")
- else:
- kept_indices = np.arange(n, dtype=np.int64)
- np.save(cache_path + "_enc.npy", enc_inputs)
- np.save(cache_path + "_dec_in.npy", dec_inputs)
- np.save(cache_path + "_dec_tgt.npy", dec_targets)
- np.save(cache_path + "_loss_mask.npy", loss_mask)
+ kept_indices = np.array(kept_indices, dtype=np.int64)
+ query_lens = np.array(query_lens_list, dtype=np.int32)
+ kept_tool_counts = np.array(kept_tool_counts, dtype=np.int32)
+
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ _save_varlen(cache_path + "_enc", enc_seqs)
+ _save_varlen(cache_path + "_dec_in", dec_in_seqs)
+ _save_varlen(cache_path + "_dec_tgt", dec_tgt_seqs)
+ _save_varlen(cache_path + "_loss", loss_seqs)
+ np.save(cache_path + "_query_len.npy", query_lens)
np.save(cache_path + "_kept_idx.npy", kept_indices)
- np.save(cache_path + "_tool_count.npy", tool_counts)
- print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")
+ np.save(cache_path + "_tool_count.npy", kept_tool_counts)
+ print(f"Cached {len(enc_seqs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")
max_tool_len = getattr(prepare_tool_call_pairs, '_max_tool_len', 256)
_build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,
model_path, max_enc_len, max_tool_len,
num_workers, chunk_size, kept_indices)
- return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts
+ return enc_seqs, dec_in_seqs, dec_tgt_seqs, loss_seqs, kept_indices, kept_tool_counts
def _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,
model_path, max_enc_len, max_tool_len,
num_workers, chunk_size, kept_indices):
- """Build and save contrastive arrays: query-only tokens + individual tool tokens."""
- kept_set = set(kept_indices.tolist()) if kept_indices is not None else set(range(len(enc_texts)))
- kept_queries = [enc_texts[i] for i in range(len(enc_texts)) if i in kept_set]
+ """Build and save contrastive arrays: individual tool tokens (variable-length int16).
- q_chunks = [kept_queries[i:i + chunk_size] for i in range(0, len(kept_queries), chunk_size)]
- with mp.Pool(num_workers, initializer=_init_worker,
- initargs=(model_path, max_enc_len)) as pool:
- q_results = list(tqdm(pool.imap(_tokenize_chunk, q_chunks),
- total=len(q_chunks), desc="Tokenizing queries (contrastive)"))
- query_tokens_list = [tok for chunk in q_results for tok in chunk]
- n_kept = len(query_tokens_list)
- query_arr = np.full((n_kept, max_enc_len), PAD_ID, dtype=np.int32)
- for i, toks in enumerate(query_tokens_list):
- l = min(len(toks), max_enc_len)
- query_arr[i, :l] = toks[:l]
- np.save(cache_path + "_query_only.npy", query_arr)
+ Query-only tokens are NOT stored separately — they are derived at load time
+ from the encoder data + query_len via QueryOnlyArray.
+ """
+ kept_set = set(kept_indices.tolist()) if kept_indices is not None else set(range(len(enc_texts)))
+ n_kept = len(kept_set)
tool_texts_individual = []
tool_ex_idx = []
@@ -676,20 +644,15 @@ def _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,
initargs=(model_path, max_tool_len)) as pool:
t_results = list(tqdm(pool.imap(_tokenize_chunk, t_chunks),
total=len(t_chunks), desc="Tokenizing tools (contrastive)"))
- tool_tokens_list = [tok for chunk in t_results for tok in chunk]
-
- n_tools = len(tool_tokens_list)
- tool_arr = np.full((n_tools, max_tool_len), PAD_ID, dtype=np.int32)
- for i, toks in enumerate(tool_tokens_list):
- l = min(len(toks), max_tool_len)
- tool_arr[i, :l] = toks[:l]
+ tool_seqs = [np.array(tok[:max_tool_len], dtype=np.int16)
+ for chunk in t_results for tok in chunk]
else:
- tool_arr = np.zeros((0, max_tool_len), dtype=np.int32)
+ tool_seqs = []
- np.save(cache_path + "_tool_individual.npy", tool_arr)
+ _save_varlen(cache_path + "_tool_ind", tool_seqs)
np.save(cache_path + "_tool_ex_idx.npy", np.array(tool_ex_idx, dtype=np.int32))
np.save(cache_path + "_tool_is_pos.npy", np.array(tool_is_pos, dtype=np.bool_))
- print(f" Contrastive: {n_kept} queries, {len(tool_ex_idx)} individual tools")
+ print(f" Contrastive: {n_kept} queries (from enc), {len(tool_ex_idx)} individual tools")
def get_contrastive_batches(query_tokens, tool_tokens, tool_ex_idx, tool_is_pos, batch_size):
@@ -722,7 +685,8 @@ def get_contrastive_batches(query_tokens, tool_tokens, tool_ex_idx, tool_is_pos,
yield q_batch, t_batch
-def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None, tool_counts=None):
+def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True,
+ loss_mask=None, tool_counts=None, enc_seg_ids=None, dec_seg_ids=None):
n = len(enc_inputs)
if tool_counts is not None:
order = np.argsort(tool_counts, kind='stable')
@@ -741,9 +705,113 @@ def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, l
batch = (np.array(enc_inputs[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))
if loss_mask is not None:
batch = batch + (np.array(loss_mask[idx]),)
+ if enc_seg_ids is not None:
+ batch = batch + (np.array(enc_seg_ids[idx]), np.array(dec_seg_ids[idx]))
yield batch
+def _seq_lens(arr):
+ """Get per-sequence lengths from a VarLenArray via its offsets."""
+ return np.diff(arr._offsets).astype(np.int32)
+
+
+def pack_sequences(cache_path, enc_vl, dec_in_vl, dec_tgt_vl, loss_vl):
+ """Pre-pack variable-length sequences into dense bins with segment IDs.
+
+ Uses first-fit-decreasing bin packing with vectorized bin search.
+ Saves packed int16 arrays + segment ID arrays.
+
+ Returns the number of packed bins.
+ """
+ n = len(enc_vl)
+ max_enc = enc_vl._max_len
+ max_dec = dec_in_vl._max_len
+
+ enc_lens = _seq_lens(enc_vl)
+ dec_lens = _seq_lens(dec_in_vl)
+
+ # Sort by encoder length descending — first-fit-decreasing
+ order = np.argsort(-enc_lens)
+
+ # Vectorized bin packing: numpy arrays for bin capacities
+ # Pre-allocate for worst case (every seq gets its own bin)
+ bin_enc_rem = np.empty(n, dtype=np.int32)
+ bin_dec_rem = np.empty(n, dtype=np.int32)
+ bin_contents = [[] for _ in range(n)]
+ n_bins = 0
+
+ for idx in tqdm(order, desc="Bin packing"):
+ el = int(enc_lens[idx])
+ dl = int(dec_lens[idx])
+
+ # Vectorized search: find all bins where both enc and dec fit
+ if n_bins > 0:
+ fits = (bin_enc_rem[:n_bins] >= el) & (bin_dec_rem[:n_bins] >= dl)
+ candidates = np.flatnonzero(fits)
+ else:
+ candidates = np.array([], dtype=np.intp)
+
+ if len(candidates) > 0:
+ b = candidates[0]
+ bin_enc_rem[b] -= el
+ bin_dec_rem[b] -= dl
+ bin_contents[b].append(int(idx))
+ else:
+ bin_enc_rem[n_bins] = max_enc - el
+ bin_dec_rem[n_bins] = max_dec - dl
+ bin_contents[n_bins].append(int(idx))
+ n_bins += 1
+
+ bin_enc_rem = bin_enc_rem[:n_bins]
+ bin_dec_rem = bin_dec_rem[:n_bins]
+ bin_contents = bin_contents[:n_bins]
+
+ # Fill packed arrays — parallelized with multiprocessing
+ packed_enc = np.zeros((n_bins, max_enc), dtype=np.int16)
+ packed_dec_in = np.zeros((n_bins, max_dec), dtype=np.int16)
+ packed_dec_tgt = np.zeros((n_bins, max_dec), dtype=np.int16)
+ packed_loss = np.zeros((n_bins, max_dec), dtype=np.float16)
+ packed_enc_seg = np.zeros((n_bins, max_enc), dtype=np.int16)
+ packed_dec_seg = np.zeros((n_bins, max_dec), dtype=np.int16)
+
+ for row in tqdm(range(n_bins), desc="Writing packed bins"):
+ enc_pos = 0
+ dec_pos = 0
+ for seg_id, idx in enumerate(bin_contents[row], start=1):
+ el = int(enc_lens[idx])
+ dl = int(dec_lens[idx])
+
+ e_start = int(enc_vl._offsets[idx])
+ packed_enc[row, enc_pos:enc_pos + el] = enc_vl._data[e_start:e_start + el]
+ packed_enc_seg[row, enc_pos:enc_pos + el] = seg_id
+
+ di_start = int(dec_in_vl._offsets[idx])
+ packed_dec_in[row, dec_pos:dec_pos + dl] = dec_in_vl._data[di_start:di_start + dl]
+
+ dt_start = int(dec_tgt_vl._offsets[idx])
+ packed_dec_tgt[row, dec_pos:dec_pos + dl] = dec_tgt_vl._data[dt_start:dt_start + dl]
+
+ lm_start = int(loss_vl._offsets[idx])
+ packed_loss[row, dec_pos:dec_pos + dl] = loss_vl._data[lm_start:lm_start + dl]
+
+ packed_dec_seg[row, dec_pos:dec_pos + dl] = seg_id
+
+ enc_pos += el
+ dec_pos += dl
+
+ np.save(cache_path + "_packed_enc.npy", packed_enc)
+ np.save(cache_path + "_packed_dec_in.npy", packed_dec_in)
+ np.save(cache_path + "_packed_dec_tgt.npy", packed_dec_tgt)
+ np.save(cache_path + "_packed_loss.npy", packed_loss)
+ np.save(cache_path + "_packed_enc_seg.npy", packed_enc_seg)
+ np.save(cache_path + "_packed_dec_seg.npy", packed_dec_seg)
+
+ total_cap = n_bins * (max_enc + max_dec)
+ used = int(np.sum(max_enc - bin_enc_rem)) + int(np.sum(max_dec - bin_dec_rem))
+ print(f" Packed {n} sequences into {n_bins} bins ({used / total_cap:.0%} utilization)")
+ return n_bins
+
+
def load_tool_call_audio(split="train", max_samples=None):
"""Return dataset-global indices for the given split.
@@ -848,6 +916,128 @@ def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=16
return log_mel.astype(np.float32)
+class VarLenArray:
+ """Variable-length storage with on-demand padding to fixed max_len.
+
+ Stores sequences as a flat 1D array (int16 or float16) plus an offsets array.
+ When indexed, returns padded arrays cast to int32/float32 for JAX compatibility.
+ """
+
+ def __init__(self, data, offsets, max_len, pad_value=0):
+ self._data = data
+ self._offsets = offsets
+ self._max_len = max_len
+ self._pad_value = pad_value
+ self._n = len(offsets) - 1
+ self._out_dtype = np.float32 if data.dtype == np.float16 else np.int32
+
+ @classmethod
+ def load(cls, prefix, max_len, pad_value=0, mmap_mode=None):
+ data = np.load(prefix + "_data.npy", mmap_mode=mmap_mode)
+ offsets = np.load(prefix + "_offsets.npy", mmap_mode=mmap_mode)
+ return cls(data, offsets, max_len, pad_value)
+
+ def __len__(self):
+ return self._n
+
+ @property
+ def shape(self):
+ return (self._n, self._max_len)
+
+ @property
+ def dtype(self):
+ return self._out_dtype
+
+ def __getitem__(self, idx):
+ if isinstance(idx, (int, np.integer)):
+ if idx < 0:
+ idx += self._n
+ start, end = int(self._offsets[idx]), int(self._offsets[idx + 1])
+ result = np.full(self._max_len, self._pad_value, dtype=self._out_dtype)
+ seq = self._data[start:end]
+ result[:len(seq)] = seq.astype(self._out_dtype)
+ return result
+
+ if isinstance(idx, np.ndarray):
+ batch_size = len(idx)
+ result = np.full((batch_size, self._max_len), self._pad_value, dtype=self._out_dtype)
+ for j, i in enumerate(idx):
+ start, end = int(self._offsets[i]), int(self._offsets[i + 1])
+ seq = self._data[start:end]
+ result[j, :len(seq)] = seq.astype(self._out_dtype)
+ return result
+
+ if isinstance(idx, slice):
+ indices = np.arange(*idx.indices(self._n))
+ return self[indices]
+
+ raise TypeError(f"Unsupported index type: {type(idx)}")
+
+
+class QueryOnlyArray:
+ """Extracts query portions from encoder variable-length data without duplication.
+
+ Queries are the prefix of each encoder sequence (before the <tools> separator).
+ Instead of storing a separate copy, this derives them on-the-fly from encoder data.
+ """
+
+ def __init__(self, enc_data, enc_offsets, query_lens, max_len):
+ self._enc_data = enc_data
+ self._enc_offsets = enc_offsets
+ self._query_lens = query_lens
+ self._max_len = max_len
+ self._n = len(query_lens)
+
+ def __len__(self):
+ return self._n
+
+ @property
+ def shape(self):
+ return (self._n, self._max_len)
+
+ @property
+ def dtype(self):
+ return np.int32
+
+ def __getitem__(self, idx):
+ if isinstance(idx, (int, np.integer)):
+ if idx < 0:
+ idx += self._n
+ start = int(self._enc_offsets[idx])
+ qlen = int(self._query_lens[idx])
+ result = np.full(self._max_len, PAD_ID, dtype=np.int32)
+ result[:qlen] = self._enc_data[start:start + qlen].astype(np.int32)
+ return result
+
+ if isinstance(idx, np.ndarray):
+ batch_size = len(idx)
+ result = np.full((batch_size, self._max_len), PAD_ID, dtype=np.int32)
+ for j, i in enumerate(idx):
+ start = int(self._enc_offsets[i])
+ qlen = int(self._query_lens[i])
+ result[j, :qlen] = self._enc_data[start:start + qlen].astype(np.int32)
+ return result
+
+ if isinstance(idx, slice):
+ indices = np.arange(*idx.indices(self._n))
+ return self[indices]
+
+ raise TypeError(f"Unsupported index type: {type(idx)}")
+
+
+def _save_varlen(prefix, sequences):
+ """Save variable-length sequences as flat data + offsets .npy files."""
+ offsets = np.zeros(len(sequences) + 1, dtype=np.int64)
+ for i, seq in enumerate(sequences):
+ offsets[i + 1] = offsets[i] + len(seq)
+ if sequences:
+ data = np.concatenate(sequences)
+ else:
+ data = np.array([], dtype=np.int16)
+ np.save(prefix + "_data.npy", data)
+ np.save(prefix + "_offsets.npy", offsets)
+
+
class ShardedMmapArray:
"""Array-like wrapper over multiple .npy shard files with mmap support.
@@ -904,13 +1094,13 @@ class ShardedMmapArray:
def load_prepared_data(split, mmap=False):
- """Load pre-tokenized .npy files. If mmap=True, returns memory-mapped arrays.
+ """Load pre-tokenized variable-length data. Returns VarLenArray objects
+ that pad to fixed max lengths on-the-fly when indexed.
- Tries local cache only. Raises FileNotFoundError if not found
- (run 'needle tokenize' first).
+ If mmap=True, underlying flat arrays are memory-mapped.
Returns dict with keys: enc_inputs, dec_inputs, dec_targets, loss_mask,
- kept_indices.
+ kept_indices, tool_counts, query_only, tool_individual, tool_ex_idx, tool_is_pos.
"""
meta = _load_cache_metadata(split)
if meta is None:
@@ -926,32 +1116,69 @@ def load_prepared_data(split, mmap=False):
text_cache_id = meta["text_cache_id"]
cache_path = os.path.join(CACHE_DIR, text_cache_id)
+ max_enc_len = meta.get("max_enc_len", DEFAULT_MAX_ENC_LEN)
+ max_dec_len = meta.get("max_dec_len", DEFAULT_MAX_DEC_LEN)
+ max_tool_len = meta.get("max_tool_len", 256)
- if not os.path.exists(cache_path + "_enc.npy"):
+ if not os.path.exists(cache_path + "_enc_data.npy"):
raise FileNotFoundError(
f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."
)
mmap_mode = "r" if mmap else None
+
+ enc = VarLenArray.load(cache_path + "_enc", max_enc_len, pad_value=PAD_ID, mmap_mode=mmap_mode)
+ dec_in = VarLenArray.load(cache_path + "_dec_in", max_dec_len, pad_value=PAD_ID, mmap_mode=mmap_mode)
+ dec_tgt = VarLenArray.load(cache_path + "_dec_tgt", max_dec_len, pad_value=PAD_ID, mmap_mode=mmap_mode)
+ loss = VarLenArray.load(cache_path + "_loss", max_dec_len, pad_value=0, mmap_mode=mmap_mode)
+ kept = np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode)
+
tc_path = cache_path + "_tool_count.npy"
tc = np.load(tc_path, mmap_mode=mmap_mode) if os.path.exists(tc_path) else None
+
+ # Query-only: derived from encoder data + query lengths (no duplication)
+ query_only = None
+ ql_path = cache_path + "_query_len.npy"
+ if os.path.exists(ql_path):
+ query_lens = np.load(ql_path, mmap_mode=mmap_mode)
+ query_only = QueryOnlyArray(enc._data, enc._offsets, query_lens, max_enc_len)
+
+ # Tool individual: variable-length
+ tool_individual = None
+ if os.path.exists(cache_path + "_tool_ind_data.npy"):
+ tool_individual = VarLenArray.load(cache_path + "_tool_ind", max_tool_len,
+ pad_value=PAD_ID, mmap_mode=mmap_mode)
+
def _load_optional(suffix):
p = cache_path + suffix
if os.path.exists(p):
return np.load(p, mmap_mode=mmap_mode)
return None
+ # Pre-packed arrays (if available from 'needle tokenize')
+ packed = {}
+ if os.path.exists(cache_path + "_packed_enc.npy"):
+ packed = {
+ "packed_enc": np.load(cache_path + "_packed_enc.npy", mmap_mode=mmap_mode),
+ "packed_dec_in": np.load(cache_path + "_packed_dec_in.npy", mmap_mode=mmap_mode),
+ "packed_dec_tgt": np.load(cache_path + "_packed_dec_tgt.npy", mmap_mode=mmap_mode),
+ "packed_loss": np.load(cache_path + "_packed_loss.npy", mmap_mode=mmap_mode),
+ "packed_enc_seg": np.load(cache_path + "_packed_enc_seg.npy", mmap_mode=mmap_mode),
+ "packed_dec_seg": np.load(cache_path + "_packed_dec_seg.npy", mmap_mode=mmap_mode),
+ }
+
return {
- "enc_inputs": np.load(cache_path + "_enc.npy", mmap_mode=mmap_mode),
- "dec_inputs": np.load(cache_path + "_dec_in.npy", mmap_mode=mmap_mode),
- "dec_targets": np.load(cache_path + "_dec_tgt.npy", mmap_mode=mmap_mode),
- "loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),
- "kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),
+ "enc_inputs": enc,
+ "dec_inputs": dec_in,
+ "dec_targets": dec_tgt,
+ "loss_mask": loss,
+ "kept_indices": kept,
"tool_counts": tc,
- "query_only": _load_optional("_query_only.npy"),
- "tool_individual": _load_optional("_tool_individual.npy"),
+ "query_only": query_only,
+ "tool_individual": tool_individual,
"tool_ex_idx": _load_optional("_tool_ex_idx.npy"),
"tool_is_pos": _load_optional("_tool_is_pos.npy"),
+ **packed,
}
@@ -1000,3 +1227,5 @@ class PrefetchIterator:
def count_batches(n_samples, batch_size):
"""Return the number of full batches for a dataset of n_samples."""
return n_samples // batch_size
+
+
diff --git a/src/model.py b/src/model.py
index 2e77085..522ff9b 100644
--- a/src/model.py
+++ b/src/model.py
@@ -505,9 +505,10 @@ class EncoderDecoderTransformer(nn.Module):
t_emb = self.encode_contrastive(tool_tokens, deterministic=deterministic)
return q_emb, t_emb, self.log_temp
- def __call__(self, src, tgt, src_mask=None, tgt_mask=None):
+ def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
- logits = self.decode(tgt, encoder_out, self_mask=tgt_mask, cross_mask=enc_mask)
+ cm = cross_mask if cross_mask is not None else enc_mask
+ logits = self.decode(tgt, encoder_out, self_mask=tgt_mask, cross_mask=cm)
return logits
def _run_decoder(self, encoder_out, tgt, tgt_mask=None, cross_mask=None, ffn_mask=None, deterministic=True):
@@ -517,10 +518,11 @@ class EncoderDecoderTransformer(nn.Module):
x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, rope=rope, ffn_mask=ffn_mask, deterministic=deterministic)
return x.astype(jnp.float32)
- def forward_masked(self, src, tgt, src_mask=None, tgt_mask=None, ffn_mask=None, deterministic=True):
+ def forward_masked(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, ffn_mask=None, deterministic=True):
"""Single forward with per-batch-item FFN masking. Returns (logits, slot_div)."""
encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)
- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask, ffn_mask=ffn_mask, deterministic=deterministic)
+ cm = cross_mask if cross_mask is not None else enc_mask
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=cm, ffn_mask=ffn_mask, deterministic=deterministic)
logits = x_f32 @ self.embedding.embedding.T
return logits, 0.0
@@ -603,5 +605,42 @@ def make_padding_mask(tokens, pad_token_id):
def make_mel_padding_mask(mel):
"""Create padding mask from mel spectrogram: non-zero frames are valid."""
- mask = jnp.any(mel != 0, axis=-1)
- return mask[:, None, None, :]
+ mask = jnp.any(mel != 0, axis=-1)
+ return mask[:, None, None, :]
+
+
+def make_packing_mask(seg_ids):
+ """Block-diagonal self-attention mask from segment IDs.
+
+ seg_ids: (B, T) int array where 0 = padding, 1+ = segment number.
+ Returns: (B, 1, T, T) bool mask — True where attention is allowed.
+ Tokens attend to each other only if they share the same non-zero segment ID.
+ """
+ # (B, T, 1) == (B, 1, T) -> (B, T, T)
+ mask = (seg_ids[:, :, None] == seg_ids[:, None, :]) & (seg_ids[:, :, None] > 0)
+ return mask[:, None, :, :]
+
+
+def make_causal_packing_mask(seg_ids):
+ """Block-diagonal causal mask from segment IDs.
+
+ Same as make_packing_mask but also enforces causal ordering within each segment.
+ Returns: (B, 1, T, T) bool mask.
+ """
+ T = seg_ids.shape[1]
+ causal = jnp.tril(jnp.ones((T, T), dtype=jnp.bool_))
+ block = (seg_ids[:, :, None] == seg_ids[:, None, :]) & (seg_ids[:, :, None] > 0)
+ return (block & causal[None, :, :])[:, None, :, :]
+
+
+def make_cross_packing_mask(enc_seg_ids, dec_seg_ids):
+ """Cross-attention mask for packed sequences.
+
+ enc_seg_ids: (B, T_enc) int segment IDs.
+ dec_seg_ids: (B, T_dec) int segment IDs.
+ Returns: (B, 1, T_dec, T_enc) bool mask — decoder token d attends to
+ encoder token e iff they share the same non-zero segment ID.
+ """
+ # (B, T_dec, 1) == (B, 1, T_enc) -> (B, T_dec, T_enc)
+ mask = (dec_seg_ids[:, :, None] == enc_seg_ids[:, None, :]) & (dec_seg_ids[:, :, None] > 0)
+ return mask[:, None, :, :]
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 985107b..7851fe5 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -18,10 +18,12 @@ from .data import (
DEFAULT_MAX_ENC_LEN,
LOCAL_UNIFIED_DIR,
TOKENIZER_DIR,
+ VarLenArray,
_cache_key,
_save_cache_metadata,
get_tokenizer,
load_tool_calls,
+ pack_sequences,
prepare_tool_call_pairs,
train_tokenizer,
)
@@ -119,8 +121,15 @@ def tokenize(args):
text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len,
w_name, w_value, w_key, shuffle_tools)
+ cache_path = os.path.join(CACHE_DIR, text_cache_id)
+ enc_vl = VarLenArray.load(cache_path + "_enc", max_enc_len)
+ dec_in_vl = VarLenArray.load(cache_path + "_dec_in", max_dec_len)
+ dec_tgt_vl = VarLenArray.load(cache_path + "_dec_tgt", max_dec_len)
+ loss_vl = VarLenArray.load(cache_path + "_loss", max_dec_len)
+ pack_sequences(cache_path, enc_vl, dec_in_vl, dec_tgt_vl, loss_vl)
+
_save_cache_metadata(split, text_cache_id, len(kept_indices),
- max_enc_len, max_dec_len)
+ max_enc_len, max_dec_len, max_tool_len)
_push_to_hf(CACHE_DIR, TOKENIZER_DIR)
diff --git a/src/train.py b/src/train.py
index 27b6d47..03059fa 100644
--- a/src/train.py
+++ b/src/train.py
@@ -21,8 +21,9 @@ from .data import (
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
- make_causal_mask,
- make_padding_mask,
+ make_packing_mask,
+ make_causal_packing_mask,
+ make_cross_packing_mask,
)
_HF_CHECKPOINT_REPO = "Cactus-Compute/needle"
@@ -290,19 +291,24 @@ def _clip_contrastive_loss(q_emb, t_emb, log_temp):
return (jnp.mean(loss_q) + jnp.mean(loss_t)) / 2.0
-def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, do_quantize):
- pad_id = 0
+def _text_loss_fn(state, params, src, tgt_in, tgt_out, ffn_mask, rng, loss_mask, do_quantize,
+ enc_seg_ids, dec_seg_ids):
q_params = jax.lax.cond(
do_quantize,
lambda p: _quantize_params(p, group_size=_GROUP_SIZE),
lambda p: p,
params,
)
- src_mask = make_padding_mask(src, pad_id)
- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+ # Segment-aware masks work for both packed and non-packed batches.
+ # Non-packed: seg_id=1 for all non-padding tokens → equivalent to padding mask + causal.
+ # Packed: seg_id=1,2,3... per sub-sequence → block-diagonal attention.
+ src_mask = make_packing_mask(enc_seg_ids)
+ tgt_mask = make_causal_packing_mask(dec_seg_ids)
+ cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
logits, slot_div = state.apply_fn(
{"params": q_params},
src, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,
+ cross_mask=cross_mask,
ffn_mask=ffn_mask,
deterministic=False,
method="forward_masked",
@@ -354,11 +360,12 @@ def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):
return jnp.concatenate(rows, axis=0)
-def _train_step(state, src, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng, do_quantize, do_contrastive, do_prune):
+def _train_step(state, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids, prune_mask, ffn_mask, rng, loss_mask, query_tokens, tool_tokens, cl_rng, do_quantize, do_contrastive, do_prune):
"""Unified train step with boolean flags for contrastive loss and pruning."""
def combined_loss(p):
- text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask, do_quantize)
+ text_loss = _text_loss_fn(state, p, src, tgt_in, tgt_out, ffn_mask, rng, loss_mask, do_quantize,
+ enc_seg_ids, dec_seg_ids)
cl_loss = jax.lax.cond(
do_contrastive,
lambda: _contrastive_loss_fn(state, p, query_tokens, tool_tokens, cl_rng, do_quantize),
@@ -386,13 +393,13 @@ def _make_p_train_step():
def _make_val_loss_fn(apply_fn):
@jax.jit
- def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):
- pad_id = 0
- src_mask = make_padding_mask(src, pad_id)
- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+ def val_loss_batch(params, src, tgt_in, tgt_out, loss_mask, enc_seg_ids, dec_seg_ids):
+ src_mask = make_packing_mask(enc_seg_ids)
+ tgt_mask = make_causal_packing_mask(dec_seg_ids)
+ cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
logits = apply_fn(
{"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
)
loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
return jnp.sum(loss * loss_mask), jnp.sum(loss_mask)
@@ -402,13 +409,13 @@ def _make_val_loss_fn(apply_fn):
def _make_mat_val_loss_fn(apply_fn, ff_width):
"""Val loss for matryoshka sub-model at given FFN width."""
@jax.jit
- def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):
- pad_id = 0
- src_mask = make_padding_mask(src, pad_id)
- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+ def val_loss_batch(params, src, tgt_in, tgt_out, loss_mask, enc_seg_ids, dec_seg_ids):
+ src_mask = make_packing_mask(enc_seg_ids)
+ tgt_mask = make_causal_packing_mask(dec_seg_ids)
+ cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
logits, _, mat_logits = apply_fn(
{"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
mat_ff_widths=(ff_width,),
method="forward_with_aux",
)
@@ -473,16 +480,21 @@ def train(args):
print(f"\n[3/3] Loading prepared data from disk (mmap)...")
train_data = load_prepared_data("train", mmap=True)
val_data = load_prepared_data("val", mmap=True)
- enc_inputs = train_data["enc_inputs"]
- dec_inputs = train_data["dec_inputs"]
- dec_targets = train_data["dec_targets"]
- train_loss_mask = train_data["loss_mask"]
- tool_counts = train_data["tool_counts"]
- val_enc = val_data["enc_inputs"]
- val_dec_in = val_data["dec_inputs"]
- val_dec_tgt = val_data["dec_targets"]
- val_loss_mask = val_data["loss_mask"]
- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")
+
+ enc_inputs = train_data["packed_enc"]
+ dec_inputs = train_data["packed_dec_in"]
+ dec_targets = train_data["packed_dec_tgt"]
+ train_loss_mask = train_data["packed_loss"]
+ train_enc_seg = train_data["packed_enc_seg"]
+ train_dec_seg = train_data["packed_dec_seg"]
+
+ val_enc = val_data["packed_enc"]
+ val_dec_in = val_data["packed_dec_in"]
+ val_dec_tgt = val_data["packed_dec_tgt"]
+ val_loss_mask = val_data["packed_loss"]
+ val_enc_seg = val_data["packed_enc_seg"]
+ val_dec_seg = val_data["packed_dec_seg"]
+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val packed bins (memory-mapped)")
val_ds = load_tool_calls("val", max_samples=args.max_samples)
@@ -582,10 +594,6 @@ def train(args):
os.makedirs(args.checkpoint_dir, exist_ok=True)
global_step = 0
- causal_mask = jnp.broadcast_to(
- make_causal_mask(args.max_dec_len),
- (num_devices, 1, args.max_dec_len, args.max_dec_len),
- )
text_ffn_mask = _make_ffn_mask(args.batch_size, config.d_ff, _MAT_FF_WIDTHS)
text_ffn_mask = jnp.broadcast_to(
@@ -629,11 +637,11 @@ def train(args):
epoch_step = 0
text_losses = []
- _curriculum_tc = tool_counts if getattr(args, "curriculum", False) else None
text_batch_iter = PrefetchIterator(
lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,
loss_mask=train_loss_mask,
- tool_counts=_curriculum_tc),
+ enc_seg_ids=train_enc_seg,
+ dec_seg_ids=train_dec_seg),
prefetch=4,
)
@@ -652,12 +660,22 @@ def train(args):
for step_i in pbar:
t0 = time.perf_counter()
- src, tgt_in, tgt_out, lm = next(text_batch_iter)
+ src, tgt_in, tgt_out, lm, enc_seg, dec_seg = next(text_batch_iter)
+
+ # Upcast from int16/float16 storage to int32/float32 for JAX
+ src = np.asarray(src, dtype=np.int32)
+ tgt_in = np.asarray(tgt_in, dtype=np.int32)
+ tgt_out = np.asarray(tgt_out, dtype=np.int32)
+ lm = np.asarray(lm, dtype=np.float32)
+ enc_seg = np.asarray(enc_seg, dtype=np.int32)
+ dec_seg = np.asarray(dec_seg, dtype=np.int32)
src_b = shard_batch(src, num_devices)
tgt_in_b = shard_batch(tgt_in, num_devices)
tgt_out_b = shard_batch(tgt_out, num_devices)
lm_b = shard_batch(lm, num_devices)
+ enc_seg_b = shard_batch(enc_seg, num_devices)
+ dec_seg_b = shard_batch(dec_seg, num_devices)
rng, text_rng = jax.random.split(rng)
text_rngs = jax.random.split(text_rng, num_devices)
@@ -682,7 +700,7 @@ def train(args):
cur_prune_mask = prune_mask if prune_mask is not None else dummy_prune_mask
state, loss, grad_norm = p_train_step(
- state, src_b, tgt_in_b, tgt_out_b, causal_mask,
+ state, src_b, tgt_in_b, tgt_out_b, enc_seg_b, dec_seg_b,
cur_prune_mask, text_ffn_mask, text_rngs, lm_b,
cl_q_b, cl_t_b, cl_rngs, do_qat, do_cl, do_prune,
)
@@ -706,12 +724,19 @@ def train(args):
eval_every = getattr(args, "eval_every", 100)
if global_step % eval_every == 0 or global_step == total_steps:
_eval_params = jax_utils.unreplicate(state).params
- val_causal = make_causal_mask(args.max_dec_len)
total_loss, total_toks = 0.0, 0.0
_val_n = count_batches(len(val_enc), args.batch_size)
- for vb in tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask),
+ for vb in tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False,
+ loss_mask=val_loss_mask, enc_seg_ids=val_enc_seg, dec_seg_ids=val_dec_seg),
total=_val_n, desc=" val loss", leave=False):
- vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
+ src_v, di_v, dt_v, lm_v, es_v, ds_v = vb
+ src_v = jnp.asarray(src_v, dtype=jnp.int32)
+ di_v = jnp.asarray(di_v, dtype=jnp.int32)
+ dt_v = jnp.asarray(dt_v, dtype=jnp.int32)
+ lm_v = jnp.asarray(lm_v, dtype=jnp.float32)
+ es_v = jnp.asarray(es_v, dtype=jnp.int32)
+ ds_v = jnp.asarray(ds_v, dtype=jnp.int32)
+ vl, vt = val_loss_fn(_eval_params, src_v, di_v, dt_v, lm_v, es_v, ds_v)
total_loss += float(vl)
total_toks += float(vt)
last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))
@@ -770,7 +795,6 @@ def train(args):
final_ppl = math.exp(min(final_loss, 20)) if not math.isnan(final_loss) else float("nan")
eval_params = jax_utils.unreplicate(state).params
- val_causal = make_causal_mask(args.max_dec_len)
q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)
mat_vl_fns = {}
@@ -786,16 +810,23 @@ def train(args):
_val_n = count_batches(len(val_enc), args.batch_size)
_val_bar = tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size,
- shuffle=False, loss_mask=val_loss_mask),
+ shuffle=False, loss_mask=val_loss_mask,
+ enc_seg_ids=val_enc_seg, dec_seg_ids=val_dec_seg),
total=_val_n, desc=" eval val loss", leave=False)
for vb in _val_bar:
- src, dec_in, dec_tgt, lm = vb[0], vb[1], vb[2], vb[3]
- vl, vt = val_loss_fn(eval_params, src, dec_in, dec_tgt, val_causal, lm)
+ src, dec_in, dec_tgt, lm, es, ds = vb
+ src = jnp.asarray(src, dtype=jnp.int32)
+ dec_in = jnp.asarray(dec_in, dtype=jnp.int32)
+ dec_tgt = jnp.asarray(dec_tgt, dtype=jnp.int32)
+ lm = jnp.asarray(lm, dtype=jnp.float32)
+ es = jnp.asarray(es, dtype=jnp.int32)
+ ds = jnp.asarray(ds, dtype=jnp.int32)
+ vl, vt = val_loss_fn(eval_params, src, dec_in, dec_tgt, lm, es, ds)
full_loss += float(vl); full_toks += float(vt)
- vl, vt = val_loss_fn(q_params, src, dec_in, dec_tgt, val_causal, lm)
+ vl, vt = val_loss_fn(q_params, src, dec_in, dec_tgt, lm, es, ds)
q_loss += float(vl); q_toks += float(vt)
for f, fn in mat_vl_fns.items():
- vl, vt = fn(eval_params, src, dec_in, dec_tgt, val_causal, lm)
+ vl, vt = fn(eval_params, src, dec_in, dec_tgt, lm, es, ds)
mat_accum[f][0] += float(vl)
mat_accum[f][1] += float(vt)
@@ -861,6 +892,7 @@ def train(args):
[ex["tools"] for ex in _chunk],
max_gen_len=eval_gen_len,
max_enc_len=args.max_enc_len,
+ constrained=False,
))
_gen_elapsed = time.perf_counter() - _gen_t0
_gen_total_toks = sum(len(tokenizer.encode(p)) for p in all_preds)