Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge main into this branch and resolve conflicts

Nandakishora14ea5f5f013
diff --cc laya/agent.py
index bb687820,f3ca21d1..ebb95808
--- a/laya/agent.py
+++ b/laya/agent.py
@@@ -245,8 -261,13 +262,9 @@@ class Agent(HookRegistry)
self.model_id = model_id_or_path
from safetensors.torch import load_file
- try:
- from transformers.initialization import no_init_weights
- except ImportError: # Transformers 4.x
- from transformers.modeling_utils import no_init_weights
model_dir = model_id_or_path
+ self.revision: Optional[str] = None
if not os.path.exists(model_dir):
if model_id_or_path.startswith(("/", "./", "../")) or os.path.isabs(model_id_or_path):
raise FileNotFoundError(
diff --cc laya/common.py
index 6566a26c,aac9507f..91f702a5
--- a/laya/common.py
+++ b/laya/common.py
@@@ -240,34 -206,21 +240,39 @@@ def _apply_rope_config(ecfg) -> None
setattr(ecfg, attr, float(theta))
+def _no_init_weights():
+ """`no_init_weights` lives in different modules across transformers versions."""
+ try:
+ from transformers.initialization import no_init_weights
+ except ImportError: # transformers 4.x
+ from transformers.modeling_utils import no_init_weights
+ return no_init_weights()
+
+
- def build_model(cfg: Dict, encoder_dir: Optional[str] = None, pretrained: bool = True) -> DecisionModel:
+ def build_model(cfg: Dict, encoder_dir: Optional[str] = None, pretrained: bool = True,
+ revision: Optional[str] = None) -> DecisionModel:
+ """Build the decision model described by `cfg`.
+
+ With `pretrained=False`, or when `encoder_dir` holds a saved encoder config, nothing is
+ downloaded and **no parameter is initialised**: the caller is expected to load a checkpoint
+ into the result with `load_state_dict(..., strict=True)` immediately. Skipping initialisation
+ keeps `load()` from spending time on, or consuming RNG for, weights it is about to overwrite.
+ """
from transformers import AutoConfig, AutoModel
+ head_layers, n_act = cfg.get("head_layers", 2), len(cfg.get("act_costs", {})) + 1
if not pretrained or (encoder_dir and os.path.exists(encoder_dir)):
ecfg = AutoConfig.from_pretrained(encoder_dir or cfg["encoder"])
_apply_rope_config(ecfg)
- enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
- else:
- # Training-time Hub load of the base encoder; allow pinning it like the checkpoints.
- kw = {"attn_implementation": "sdpa"}
- if revision:
- kw["revision"] = revision
- enc = AutoModel.from_pretrained(cfg["encoder"], **kw)
- return DecisionModel(enc, cfg.get("head_layers", 2), len(cfg.get("act_costs", {})) + 1)
+ with _no_init_weights():
+ enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
+ return DecisionModel(enc, head_layers, n_act, no_init=True)
- enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")
++ # Training-time Hub load of the base encoder; allow pinning it like the checkpoints.
++ kw = {"attn_implementation": "sdpa"}
++ if revision:
++ kw["revision"] = revision
++ enc = AutoModel.from_pretrained(cfg["encoder"], **kw)
+ return DecisionModel(enc, head_layers, n_act)
def proper_reward(