Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku839adf3ce703
diff --git a/src/cli.py b/src/cli.py
index 126ecbd..b3313bc 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -15,15 +15,15 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--checkpoint", type=str, default=None)
- p.add_argument("--epochs", type=int, default=5)
- p.add_argument("--batch-size", type=int, default=64)
- p.add_argument("--lr", type=float, default=4.5e-4)
+ p.add_argument("--epochs", type=int, default=10)
+ p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=768)
+ p.add_argument("--d-model", type=int, default=512)
p.add_argument("--num-heads", type=int, default=16)
p.add_argument("--num-kv-heads", type=int, default=8)
- p.add_argument("--num-layers", type=int, default=4)
- p.add_argument("--num-dec-layers", type=int, default=4)
+ p.add_argument("--num-layers", type=int, default=6)
+ p.add_argument("--num-dec-layers", type=int, default=6)
p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
p.add_argument("--max-samples", type=int, default=None)
@@ -43,11 +43,9 @@ def main():
p.add_argument("--prune-end-frac", type=float, default=0.67,
help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
- p.add_argument("--num-memory-slots", type=int, default=256)
+ p.add_argument("--num-memory-slots", type=int, default=128)
p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],
help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")
- p.add_argument("--mat-shared-input", action=argparse.BooleanOptionalAction, default=True,
- help="Each unique input is repeated across all mat widths (default: True)")
p.add_argument("--dropout", type=float, default=0.1,
help="Dropout rate for residual connections (default: 0.1)")
p.add_argument("--curriculum", action="store_true",
diff --git a/src/data.py b/src/data.py
index 33861f2..40bba44 100644
--- a/src/data.py
+++ b/src/data.py
@@ -27,20 +27,6 @@ _DISK_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
_SHM_UNIFIED_DIR = os.path.join("/dev/shm", "needle_data", "tool_calls_unified")
-def _pick_unified_dir():
- """Use /dev/shm for unified dataset when RAM is plentiful, else disk."""
- # Prefer shm if it exists there already or if we have enough RAM
- if os.path.isdir(_SHM_UNIFIED_DIR) and any(
- f.endswith(".arrow") for f in os.listdir(_SHM_UNIFIED_DIR)
- ):
- return _SHM_UNIFIED_DIR
- if _shm_available():
- return _SHM_UNIFIED_DIR
- return _DISK_UNIFIED_DIR
-
-
-LOCAL_UNIFIED_DIR = _pick_unified_dir()
-
_MIN_SHM_BYTES = 200 * 1024**3
@@ -56,6 +42,20 @@ def _shm_available():
return False
+def _pick_unified_dir():
+ """Use /dev/shm for unified dataset when RAM is plentiful, else disk."""
+ if os.path.isdir(_SHM_UNIFIED_DIR) and any(
+ f.endswith(".arrow") for f in os.listdir(_SHM_UNIFIED_DIR)
+ ):
+ return _SHM_UNIFIED_DIR
+ if _shm_available():
+ return _SHM_UNIFIED_DIR
+ return _DISK_UNIFIED_DIR
+
+
+LOCAL_UNIFIED_DIR = _pick_unified_dir()
+
+
def _pick_cache_dir():
"""Use /dev/shm (tmpfs/RAM) when available and large enough, else disk."""
if _shm_available():
diff --git a/src/train.py b/src/train.py
index 9731246..bcacfc6 100644
--- a/src/train.py
+++ b/src/train.py
@@ -441,10 +441,7 @@ def train(args):
rng = jax.random.PRNGKey(args.seed)
rng, init_rng = jax.random.split(rng)
- mat_shared_input = getattr(args, "mat_shared_input", False)
- unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size
-
- unique_batch_size = (unique_batch_size // num_devices) * num_devices
+ unique_batch_size = (effective_batch_size // num_devices) * num_devices
text_batches_per_epoch = count_batches(len(enc_inputs), unique_batch_size)
num_batches = text_batches_per_epoch
total_steps = num_batches * args.epochs
@@ -492,19 +489,14 @@ def train(args):
(num_devices, 1, args.max_dec_len, args.max_dec_len),
)
- per_width = args.batch_size // n_widths if n_widths > 1 else args.batch_size
- mat_per_device = per_width * n_widths # actual per-device batch (may < args.batch_size)
- text_ffn_mask = _make_ffn_mask(mat_per_device, config.d_ff, _MAT_FF_WIDTHS)
+ text_ffn_mask = _make_ffn_mask(args.batch_size, config.d_ff, _MAT_FF_WIDTHS)
text_ffn_mask = jnp.broadcast_to(
text_ffn_mask[None, :, :],
- (num_devices, mat_per_device, config.d_ff),
+ (num_devices, args.batch_size, config.d_ff),
)
if n_widths > 1:
print(f" Mat factors {n_widths} (full + {', '.join(str(f)+'x' for f in _MAT_FACTORS)})")
- if mat_shared_input:
- print(f" Mat mode shared input ({per_width}/dev x {n_widths} repeats)")
- else:
- print(f" Mat mode unique input ({args.batch_size}/dev, random width)")
+ print(f" Mat mode unique input ({args.batch_size}/dev, split by width)")
adam_schedule = _wsd_schedule(scaled_lr, total_steps, warmup_steps)
muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)
@@ -547,19 +539,10 @@ def train(args):
src, tgt_in, tgt_out, lm = next(text_batch_iter)
- if n_widths > 1 and mat_shared_input:
- def _tile_for_mat(arr):
- s = arr.reshape(num_devices, per_width, *arr.shape[1:])
- return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))
- src_b = _tile_for_mat(src)
- tgt_in_b = _tile_for_mat(tgt_in)
- tgt_out_b = _tile_for_mat(tgt_out)
- lm_b = _tile_for_mat(lm)
- else:
- src_b = shard_batch(src, num_devices)
- tgt_in_b = shard_batch(tgt_in, num_devices)
- tgt_out_b = shard_batch(tgt_out, num_devices)
- lm_b = shard_batch(lm, num_devices)
+ src_b = shard_batch(src, num_devices)
+ tgt_in_b = shard_batch(tgt_in, num_devices)
+ tgt_out_b = shard_batch(tgt_out, num_devices)
+ lm_b = shard_batch(lm, num_devices)
rng, text_rng = jax.random.split(rng)
text_rngs = jax.random.split(text_rng, num_devices)
@@ -725,8 +708,8 @@ def train(args):
# Collect all eval examples, then batch-generate everything at once
import json as _json_mod
- tc_with_n = 25
- tc_without_n = 5
+ tc_with_n = 85
+ tc_without_n = 15
tc_rng = np.random.RandomState(epoch + 42)
tc_pool = tc_rng.permutation(len(val_kept))
tc_with, tc_without = [], []