Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
add pretrain support
HenryNdubuaku49adaa7690b2
diff --git a/src/cli.py b/src/cli.pyindex 2299092..2d5c840 100644--- a/src/cli.py+++ b/src/cli.py@@ -106,7 +106,12 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--name", type=str, default="baseline",help="Experiment name for checkpoints and wandb (default: baseline)")- p.add_argument("--checkpoint", type=str, default=None)+ p.add_argument("--checkpoint", type=str, default=None,+ help="Full resume: adopts checkpoint's config and step counter")+ p.add_argument("--init-from", type=str, default=None,+ help="Initialize params from a pretrained base on HF "+ "(e.g. needle_base.pkl). Uses CLI config; partial-loads "+ "matching params, random-inits the rest.")p.add_argument("--epochs", type=int, default=1)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)diff --git a/src/distributed.py b/src/distributed.pyindex 5f1f26c..4d2f9d2 100644--- a/src/distributed.py+++ b/src/distributed.py@@ -1,11 +1,112 @@import os+import pickleimport threadingimport jax+import jax.numpy as jnpimport numpy as np_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"+def _download_checkpoint(name_or_path):+ """Return a local path to a checkpoint, downloading from HF if needed.++ If `name_or_path` already exists on disk, it's returned unchanged.+ Otherwise treats the basename as a file in `Cactus-Compute/checkpoints`.+ """+ if os.path.exists(name_or_path):+ return name_or_path+ from huggingface_hub import hf_hub_download+ local_dir = os.path.dirname(name_or_path) or "checkpoints"+ os.makedirs(local_dir, exist_ok=True)+ print(f" Downloading {os.path.basename(name_or_path)} from {_HF_CHECKPOINT_REPO}...", flush=True)+ local_path = hf_hub_download(+ repo_id=_HF_CHECKPOINT_REPO,+ filename=os.path.basename(name_or_path),+ repo_type="model",+ local_dir=local_dir,+ )+ print(f" Downloaded to {local_path}", flush=True)+ return local_path+++def _flatten_params(tree, prefix=()):+ """Flatten a nested dict of params into {tuple-path: leaf}."""+ out = {}+ if isinstance(tree, dict):+ for k, v in tree.items():+ out.update(_flatten_params(v, prefix + (k,)))+ else:+ out[prefix] = tree+ return out+++def _unflatten_params(flat):+ """Inverse of _flatten_params: rebuild a nested dict."""+ root = {}+ for path, leaf in flat.items():+ node = root+ for k in path[:-1]:+ node = node.setdefault(k, {})+ node[path[-1]] = leaf+ return root+++def partial_load_params(init_params, loaded_params):+ """Merge `loaded_params` into `init_params` by matching path + shape.++ For every leaf in `init_params`:+ - if the same path exists in `loaded_params` and shapes agree,+ replace it with the loaded value (cast to the init leaf's dtype);+ - otherwise keep the init leaf (random init).++ Extra leaves in `loaded_params` (not present in `init_params`) are ignored.++ Returns (merged_params, stats_dict).+ """+ init_flat = _flatten_params(init_params)+ loaded_flat = _flatten_params(loaded_params)++ merged = {}+ n_loaded = 0+ n_random = 0+ shape_mismatches = []+ missing_in_ckpt = []+ for path, init_leaf in init_flat.items():+ loaded_leaf = loaded_flat.get(path)+ if loaded_leaf is None:+ merged[path] = init_leaf+ n_random += 1+ missing_in_ckpt.append(path)+ continue+ if tuple(loaded_leaf.shape) != tuple(init_leaf.shape):+ merged[path] = init_leaf+ n_random += 1+ shape_mismatches.append((path, tuple(loaded_leaf.shape), tuple(init_leaf.shape)))+ continue+ merged[path] = jnp.asarray(loaded_leaf, dtype=init_leaf.dtype)+ n_loaded += 1++ extra_in_ckpt = [p for p in loaded_flat.keys() if p not in init_flat]++ stats = {+ "loaded": n_loaded,+ "random_init": n_random,+ "shape_mismatches": shape_mismatches,+ "missing_in_ckpt": missing_in_ckpt,+ "extra_in_ckpt": extra_in_ckpt,+ }+ return _unflatten_params(merged), stats+++def load_pretrained_params(path_or_name):+ """Download (if needed) + pickle.load a checkpoint, return its params dict."""+ local = _download_checkpoint(path_or_name)+ with open(local, "rb") as f:+ data = pickle.load(f)+ return data["params"], data.get("config", None), local++def _replicate(tree):"""Replicate a pytree across all local devices (multi-host safe)."""devices = jax.local_devices()diff --git a/src/train.py b/src/train.pyindex ec35be1..e104840 100644--- a/src/train.py+++ b/src/train.py@@ -23,7 +23,10 @@ from .model import (make_cross_packing_mask,)from .optim import create_train_state, _wsd_schedule-from .distributed import _replicate, _unreplicate, shard_batch, _upload_checkpoint+from .distributed import (+ _replicate, _unreplicate, shard_batch, _upload_checkpoint,+ load_pretrained_params, partial_load_params,+)from .quantize import (_quantize_params, _cubic_sparsity_schedule, _make_prune_mask,)@@ -364,6 +367,25 @@ def train(args):state = state.replace(params=ckpt_params)print(f" Loaded checkpoint params into train state")+ init_from = getattr(args, "init_from", None)+ if init_from and not resume_checkpoint:+ print(f"\n Initializing from pretrained base: {init_from}")+ loaded_params, _loaded_cfg, local_path = load_pretrained_params(init_from)+ merged, stats = partial_load_params(state.params, loaded_params)+ state = state.replace(params=merged)+ total = stats["loaded"] + stats["random_init"]+ print(f" [init-from] {stats['loaded']}/{total} leaves loaded from {os.path.basename(local_path)}, "+ f"{stats['random_init']} kept at random init")+ if stats["shape_mismatches"]:+ print(f" [init-from] {len(stats['shape_mismatches'])} shape mismatch(es):")+ for path, loaded_shape, init_shape in stats["shape_mismatches"][:8]:+ print(f" {'/'.join(path)}: ckpt={loaded_shape} vs init={init_shape}")+ if len(stats["shape_mismatches"]) > 8:+ print(f" ... ({len(stats['shape_mismatches']) - 8} more)")+ if stats["extra_in_ckpt"]:+ print(f" [init-from] {len(stats['extra_in_ckpt'])} extra leaf(s) in ckpt (ignored)")+ del loaded_params, merged+state = _replicate(state)param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))