Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
streamline
HenryNdubuakud59ceac47291
diff --git a/src/pretrain.py b/src/pretrain.pyindex 7cee4be..94d852d 100644--- a/src/pretrain.py+++ b/src/pretrain.py@@ -47,7 +47,9 @@ def _stream_batches(tokenizer, batch_size, max_enc_len, max_dec_len, seed=42):from datasets import load_datasetds = load_dataset(_HF_PRETRAIN_REPO, split="train", streaming=True)- ds = ds.shuffle(seed=seed, buffer_size=10000)+ # Small shuffle buffer so first batch yields quickly on resume.+ # Dataset is already split across 500 shards so it's already partially shuffled.+ ds = ds.shuffle(seed=seed, buffer_size=1000)tools_sep_id = TOOLS_IDeos_id = EOS_ID@@ -191,20 +193,50 @@ def pretrain(args):print(f"\n[2/3] Loading tokenizer...")tokenizer = get_tokenizer()- config = TransformerConfig(- d_model=args.d_model,- num_heads=args.num_heads,- num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,- num_encoder_layers=args.num_layers,- num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),- d_ff=getattr(args, "d_ff", None) or args.d_model * 4,- max_seq_len=max(args.max_enc_len, args.max_dec_len),- dtype=args.dtype,- activation=getattr(args, "activation", "swiglu"),- no_feedforward=getattr(args, "no_feedforward", True),- contrastive_dim=getattr(args, "contrastive_dim", 128),- enable_speech=False,- )+ # If resuming, download checkpoint from HF if not local, then load config+ resume_checkpoint_arg = getattr(args, "checkpoint", None)+ _preloaded_ckpt = None+ if resume_checkpoint_arg:+ if not os.path.exists(resume_checkpoint_arg):+ print(f" Checkpoint not found locally, downloading from HF...", flush=True)+ try:+ from huggingface_hub import hf_hub_download+ local_dir = os.path.dirname(resume_checkpoint_arg) or "checkpoints"+ os.makedirs(local_dir, exist_ok=True)+ local_path = hf_hub_download(+ repo_id="Cactus-Compute/checkpoints",+ filename=os.path.basename(resume_checkpoint_arg),+ repo_type="model",+ local_dir=local_dir,+ )+ resume_checkpoint_arg = local_path+ print(f" Downloaded to {local_path}", flush=True)+ except Exception as e:+ print(f" ERROR downloading checkpoint: {e}", flush=True)+ return++ print(f" Pre-loading config from {resume_checkpoint_arg}", flush=True)+ with open(resume_checkpoint_arg, "rb") as f:+ _preloaded_ckpt = pickle.load(f)+ config = TransformerConfig(**_preloaded_ckpt["config"])+ print(f" Config from ckpt: d={config.d_model}, "+ f"heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}",+ flush=True)+ else:+ config = TransformerConfig(+ d_model=args.d_model,+ num_heads=args.num_heads,+ num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,+ num_encoder_layers=args.num_layers,+ num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),+ d_ff=getattr(args, "d_ff", None) or args.d_model * 4,+ max_seq_len=max(args.max_enc_len, args.max_dec_len),+ dtype=args.dtype,+ activation=getattr(args, "activation", "swiglu"),+ no_feedforward=getattr(args, "no_feedforward", True),+ contrastive_dim=getattr(args, "contrastive_dim", 128),+ enable_speech=False,+ )effective_batch_size = args.batch_size * num_devicesglobal_batch_size = effective_batch_size * num_hosts@@ -223,35 +255,40 @@ def pretrain(args):decay_ratio = getattr(args, "decay_ratio", 0.05)resume_step = 0- resume_checkpoint = getattr(args, "checkpoint", None)+ resume_checkpoint = resume_checkpoint_arg+ ckpt_data = _preloaded_ckptprint(f"\n[3/3] Initializing model...")rng = jax.random.PRNGKey(args.seed)rng, init_rng = jax.random.split(rng)state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)- if resume_checkpoint:- print(f" Resuming from {resume_checkpoint}")- with open(resume_checkpoint, "rb") as f:- ckpt_data = pickle.load(f)+ if resume_checkpoint and ckpt_data is not None:+ print(f" Resuming from {resume_checkpoint}", flush=True)ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])state = state.replace(params=ckpt_params)del ckpt_params- # Use --resume-step override, else read from checkpoint, else 0+ print(f" Loaded checkpoint params", flush=True)+ # Use --resume-step override, else read from checkpoint, else 0+ if ckpt_data is not None:manual_step = getattr(args, "resume_step", None)if manual_step is not None:resume_step = manual_stepelse:resume_step = ckpt_data.get("pretrain_step", 0)if resume_step > 0:- print(f" Resuming from step {resume_step}")+ print(f" Resuming from step {resume_step}", flush=True)+ print(f" Replicating state across {num_devices} local devices...", flush=True)state = _replicate(state)+ print(f" Replicated.", flush=True)param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))+ print(f" Params: {param_count:,}", flush=True)global _p_pretrain_step_p_pretrain_step = _make_p_pretrain_step()+ print(f" pmap ready.", flush=True)save_every = getattr(args, "save_every", 1000)checkpoint_dir = getattr(args, "checkpoint_dir", "checkpoints")@@ -282,10 +319,13 @@ def pretrain(args):global_step = resume_stepfor epoch in range(args.epochs):+ # Use a different seed on resume to get fresh data ordering+ # (avoids re-seeing exactly the same examples we saw before the crash)+ stream_seed = args.seed + epoch + resume_stepbatch_stream = _PrefetchStream(lambda: _stream_batches(tokenizer, global_batch_size,args.max_enc_len, args.max_dec_len,- seed=args.seed + epoch),+ seed=stream_seed),prefetch=8,)@@ -293,15 +333,6 @@ def pretrain(args):total=estimated_steps, initial=resume_step,disable=not is_main)- # Skip ahead if resuming- if resume_step > 0:- if is_main:- print(f" Skipping {resume_step} batches to resume...")- for i, _ in enumerate(batch_stream):- if i + 1 >= resume_step:- break- resume_step = 0 # only skip once-for batch in batch_stream:if max_steps and global_step >= max_steps:breakdiff --git a/src/tpu.py b/src/tpu.pyindex e3e5d3a..3907747 100644--- a/src/tpu.py+++ b/src/tpu.py@@ -568,7 +568,8 @@ def _tpu_run_command(args, needle_cmd="train"):extra = getattr(args, "train_args", [])if extra and extra[0] == "--":extra = extra[1:]- train_cmd = f"cd ~/needle && .venv/bin/needle {needle_cmd}"++ train_cmd = f"cd ~/needle && PYTHONUNBUFFERED=1 .venv/bin/needle {needle_cmd}"if extra:train_cmd += " " + " ".join(extra)