Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku839adf3ce703
diff --git a/src/cli.py b/src/cli.pyindex 126ecbd..b3313bc 100644--- a/src/cli.py+++ b/src/cli.py@@ -15,15 +15,15 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--checkpoint", type=str, default=None)- p.add_argument("--epochs", type=int, default=5)- p.add_argument("--batch-size", type=int, default=64)- p.add_argument("--lr", type=float, default=4.5e-4)+ p.add_argument("--epochs", type=int, default=10)+ p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=768)+ p.add_argument("--d-model", type=int, default=512)p.add_argument("--num-heads", type=int, default=16)p.add_argument("--num-kv-heads", type=int, default=8)- p.add_argument("--num-layers", type=int, default=4)- p.add_argument("--num-dec-layers", type=int, default=4)+ p.add_argument("--num-layers", type=int, default=6)+ p.add_argument("--num-dec-layers", type=int, default=6)p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)p.add_argument("--max-samples", type=int, default=None)@@ -43,11 +43,9 @@ def main():p.add_argument("--prune-end-frac", type=float, default=0.67,help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])- p.add_argument("--num-memory-slots", type=int, default=256)+ p.add_argument("--num-memory-slots", type=int, default=128)p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")- p.add_argument("--mat-shared-input", action=argparse.BooleanOptionalAction, default=True,- help="Each unique input is repeated across all mat widths (default: True)")p.add_argument("--dropout", type=float, default=0.1,help="Dropout rate for residual connections (default: 0.1)")p.add_argument("--curriculum", action="store_true",diff --git a/src/data.py b/src/data.pyindex 33861f2..40bba44 100644--- a/src/data.py+++ b/src/data.py@@ -27,20 +27,6 @@ _DISK_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")_SHM_UNIFIED_DIR = os.path.join("/dev/shm", "needle_data", "tool_calls_unified")-def _pick_unified_dir():- """Use /dev/shm for unified dataset when RAM is plentiful, else disk."""- # Prefer shm if it exists there already or if we have enough RAM- if os.path.isdir(_SHM_UNIFIED_DIR) and any(- f.endswith(".arrow") for f in os.listdir(_SHM_UNIFIED_DIR)- ):- return _SHM_UNIFIED_DIR- if _shm_available():- return _SHM_UNIFIED_DIR- return _DISK_UNIFIED_DIR---LOCAL_UNIFIED_DIR = _pick_unified_dir()-_MIN_SHM_BYTES = 200 * 1024**3@@ -56,6 +42,20 @@ def _shm_available():return False+def _pick_unified_dir():+ """Use /dev/shm for unified dataset when RAM is plentiful, else disk."""+ if os.path.isdir(_SHM_UNIFIED_DIR) and any(+ f.endswith(".arrow") for f in os.listdir(_SHM_UNIFIED_DIR)+ ):+ return _SHM_UNIFIED_DIR+ if _shm_available():+ return _SHM_UNIFIED_DIR+ return _DISK_UNIFIED_DIR+++LOCAL_UNIFIED_DIR = _pick_unified_dir()++def _pick_cache_dir():"""Use /dev/shm (tmpfs/RAM) when available and large enough, else disk."""if _shm_available():diff --git a/src/train.py b/src/train.pyindex 9731246..bcacfc6 100644--- a/src/train.py+++ b/src/train.py@@ -441,10 +441,7 @@ def train(args):rng = jax.random.PRNGKey(args.seed)rng, init_rng = jax.random.split(rng)- mat_shared_input = getattr(args, "mat_shared_input", False)- unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size-- unique_batch_size = (unique_batch_size // num_devices) * num_devices+ unique_batch_size = (effective_batch_size // num_devices) * num_devicestext_batches_per_epoch = count_batches(len(enc_inputs), unique_batch_size)num_batches = text_batches_per_epochtotal_steps = num_batches * args.epochs@@ -492,19 +489,14 @@ def train(args):(num_devices, 1, args.max_dec_len, args.max_dec_len),)- per_width = args.batch_size // n_widths if n_widths > 1 else args.batch_size- mat_per_device = per_width * n_widths # actual per-device batch (may < args.batch_size)- text_ffn_mask = _make_ffn_mask(mat_per_device, config.d_ff, _MAT_FF_WIDTHS)+ text_ffn_mask = _make_ffn_mask(args.batch_size, config.d_ff, _MAT_FF_WIDTHS)text_ffn_mask = jnp.broadcast_to(text_ffn_mask[None, :, :],- (num_devices, mat_per_device, config.d_ff),+ (num_devices, args.batch_size, config.d_ff),)if n_widths > 1:print(f" Mat factors {n_widths} (full + {', '.join(str(f)+'x' for f in _MAT_FACTORS)})")- if mat_shared_input:- print(f" Mat mode shared input ({per_width}/dev x {n_widths} repeats)")- else:- print(f" Mat mode unique input ({args.batch_size}/dev, random width)")+ print(f" Mat mode unique input ({args.batch_size}/dev, split by width)")adam_schedule = _wsd_schedule(scaled_lr, total_steps, warmup_steps)muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)@@ -547,19 +539,10 @@ def train(args):src, tgt_in, tgt_out, lm = next(text_batch_iter)- if n_widths > 1 and mat_shared_input:- def _tile_for_mat(arr):- s = arr.reshape(num_devices, per_width, *arr.shape[1:])- return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))- src_b = _tile_for_mat(src)- tgt_in_b = _tile_for_mat(tgt_in)- tgt_out_b = _tile_for_mat(tgt_out)- lm_b = _tile_for_mat(lm)- else:- src_b = shard_batch(src, num_devices)- tgt_in_b = shard_batch(tgt_in, num_devices)- tgt_out_b = shard_batch(tgt_out, num_devices)- lm_b = shard_batch(lm, num_devices)+ src_b = shard_batch(src, num_devices)+ tgt_in_b = shard_batch(tgt_in, num_devices)+ tgt_out_b = shard_batch(tgt_out, num_devices)+ lm_b = shard_batch(lm, num_devices)rng, text_rng = jax.random.split(rng)text_rngs = jax.random.split(text_rng, num_devices)@@ -725,8 +708,8 @@ def train(args):# Collect all eval examples, then batch-generate everything at onceimport json as _json_mod- tc_with_n = 25- tc_without_n = 5+ tc_with_n = 85+ tc_without_n = 15tc_rng = np.random.RandomState(epoch + 42)tc_pool = tc_rng.permutation(len(val_kept))tc_with, tc_without = [], []