Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

add pretrain support

HenryNdubuaku49adaa7690b2
diff --git a/src/cli.py b/src/cli.py
index 2299092..2d5c840 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -106,7 +106,12 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--name", type=str, default="baseline",
help="Experiment name for checkpoints and wandb (default: baseline)")
- p.add_argument("--checkpoint", type=str, default=None)
+ p.add_argument("--checkpoint", type=str, default=None,
+ help="Full resume: adopts checkpoint's config and step counter")
+ p.add_argument("--init-from", type=str, default=None,
+ help="Initialize params from a pretrained base on HF "
+ "(e.g. needle_base.pkl). Uses CLI config; partial-loads "
+ "matching params, random-inits the rest.")
p.add_argument("--epochs", type=int, default=1)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
diff --git a/src/distributed.py b/src/distributed.py
index 5f1f26c..4d2f9d2 100644
--- a/src/distributed.py
+++ b/src/distributed.py
@@ -1,11 +1,112 @@
import os
+import pickle
import threading
import jax
+import jax.numpy as jnp
import numpy as np
_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"
+def _download_checkpoint(name_or_path):
+ """Return a local path to a checkpoint, downloading from HF if needed.
+
+ If `name_or_path` already exists on disk, it's returned unchanged.
+ Otherwise treats the basename as a file in `Cactus-Compute/checkpoints`.
+ """
+ if os.path.exists(name_or_path):
+ return name_or_path
+ from huggingface_hub import hf_hub_download
+ local_dir = os.path.dirname(name_or_path) or "checkpoints"
+ os.makedirs(local_dir, exist_ok=True)
+ print(f" Downloading {os.path.basename(name_or_path)} from {_HF_CHECKPOINT_REPO}...", flush=True)
+ local_path = hf_hub_download(
+ repo_id=_HF_CHECKPOINT_REPO,
+ filename=os.path.basename(name_or_path),
+ repo_type="model",
+ local_dir=local_dir,
+ )
+ print(f" Downloaded to {local_path}", flush=True)
+ return local_path
+
+
+def _flatten_params(tree, prefix=()):
+ """Flatten a nested dict of params into {tuple-path: leaf}."""
+ out = {}
+ if isinstance(tree, dict):
+ for k, v in tree.items():
+ out.update(_flatten_params(v, prefix + (k,)))
+ else:
+ out[prefix] = tree
+ return out
+
+
+def _unflatten_params(flat):
+ """Inverse of _flatten_params: rebuild a nested dict."""
+ root = {}
+ for path, leaf in flat.items():
+ node = root
+ for k in path[:-1]:
+ node = node.setdefault(k, {})
+ node[path[-1]] = leaf
+ return root
+
+
+def partial_load_params(init_params, loaded_params):
+ """Merge `loaded_params` into `init_params` by matching path + shape.
+
+ For every leaf in `init_params`:
+ - if the same path exists in `loaded_params` and shapes agree,
+ replace it with the loaded value (cast to the init leaf's dtype);
+ - otherwise keep the init leaf (random init).
+
+ Extra leaves in `loaded_params` (not present in `init_params`) are ignored.
+
+ Returns (merged_params, stats_dict).
+ """
+ init_flat = _flatten_params(init_params)
+ loaded_flat = _flatten_params(loaded_params)
+
+ merged = {}
+ n_loaded = 0
+ n_random = 0
+ shape_mismatches = []
+ missing_in_ckpt = []
+ for path, init_leaf in init_flat.items():
+ loaded_leaf = loaded_flat.get(path)
+ if loaded_leaf is None:
+ merged[path] = init_leaf
+ n_random += 1
+ missing_in_ckpt.append(path)
+ continue
+ if tuple(loaded_leaf.shape) != tuple(init_leaf.shape):
+ merged[path] = init_leaf
+ n_random += 1
+ shape_mismatches.append((path, tuple(loaded_leaf.shape), tuple(init_leaf.shape)))
+ continue
+ merged[path] = jnp.asarray(loaded_leaf, dtype=init_leaf.dtype)
+ n_loaded += 1
+
+ extra_in_ckpt = [p for p in loaded_flat.keys() if p not in init_flat]
+
+ stats = {
+ "loaded": n_loaded,
+ "random_init": n_random,
+ "shape_mismatches": shape_mismatches,
+ "missing_in_ckpt": missing_in_ckpt,
+ "extra_in_ckpt": extra_in_ckpt,
+ }
+ return _unflatten_params(merged), stats
+
+
+def load_pretrained_params(path_or_name):
+ """Download (if needed) + pickle.load a checkpoint, return its params dict."""
+ local = _download_checkpoint(path_or_name)
+ with open(local, "rb") as f:
+ data = pickle.load(f)
+ return data["params"], data.get("config", None), local
+
+
def _replicate(tree):
"""Replicate a pytree across all local devices (multi-host safe)."""
devices = jax.local_devices()
diff --git a/src/train.py b/src/train.py
index ec35be1..e104840 100644
--- a/src/train.py
+++ b/src/train.py
@@ -23,7 +23,10 @@ from .model import (
make_cross_packing_mask,
)
from .optim import create_train_state, _wsd_schedule
-from .distributed import _replicate, _unreplicate, shard_batch, _upload_checkpoint
+from .distributed import (
+ _replicate, _unreplicate, shard_batch, _upload_checkpoint,
+ load_pretrained_params, partial_load_params,
+)
from .quantize import (
_quantize_params, _cubic_sparsity_schedule, _make_prune_mask,
)
@@ -364,6 +367,25 @@ def train(args):
state = state.replace(params=ckpt_params)
print(f" Loaded checkpoint params into train state")
+ init_from = getattr(args, "init_from", None)
+ if init_from and not resume_checkpoint:
+ print(f"\n Initializing from pretrained base: {init_from}")
+ loaded_params, _loaded_cfg, local_path = load_pretrained_params(init_from)
+ merged, stats = partial_load_params(state.params, loaded_params)
+ state = state.replace(params=merged)
+ total = stats["loaded"] + stats["random_init"]
+ print(f" [init-from] {stats['loaded']}/{total} leaves loaded from {os.path.basename(local_path)}, "
+ f"{stats['random_init']} kept at random init")
+ if stats["shape_mismatches"]:
+ print(f" [init-from] {len(stats['shape_mismatches'])} shape mismatch(es):")
+ for path, loaded_shape, init_shape in stats["shape_mismatches"][:8]:
+ print(f" {'/'.join(path)}: ckpt={loaded_shape} vs init={init_shape}")
+ if len(stats["shape_mismatches"]) > 8:
+ print(f" ... ({len(stats['shape_mismatches']) - 8} more)")
+ if stats["extra_in_ckpt"]:
+ print(f" [init-from] {len(stats['extra_in_ckpt'])} extra leaf(s) in ckpt (ignored)")
+ del loaded_params, merged
+
state = _replicate(state)
param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))