Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Skip redundant initialization when loading checkpoints (#195)

Sophiefea4aa2f1fe1
diff --git a/laya/agent.py b/laya/agent.py
index 15e2eb56..39553a74 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -115,6 +115,10 @@ class Agent:
"""
from safetensors.torch import load_file
from transformers import AutoTokenizer
+ try:
+ from transformers.initialization import no_init_weights
+ except ImportError: # Transformers 4.x
+ from transformers.modeling_utils import no_init_weights
model_dir = model_id_or_path
if not os.path.exists(model_dir):
@@ -185,7 +189,10 @@ class Agent:
self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))
enc_dir = os.path.join(model_dir, "encoder")
- self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None)
+ # The checkpoint supplies every parameter; skip random/base-model weights.
+ with no_init_weights():
+ self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None,
+ pretrained=False)
# Load weights and verify architectural compatibility
weights = load_file(weights_path)
diff --git a/laya/common.py b/laya/common.py
index fb900053..2d030002 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -136,11 +136,11 @@ class DecisionModel(nn.Module):
return logits, act_logits
-def build_model(cfg: Dict, encoder_dir: Optional[str] = None) -> DecisionModel:
+def build_model(cfg: Dict, encoder_dir: Optional[str] = None, pretrained: bool = True) -> DecisionModel:
from transformers import AutoConfig, AutoModel
- if encoder_dir and os.path.exists(encoder_dir):
- ecfg = AutoConfig.from_pretrained(encoder_dir)
+ if not pretrained or (encoder_dir and os.path.exists(encoder_dir)):
+ ecfg = AutoConfig.from_pretrained(encoder_dir or cfg["encoder"])
enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
else:
enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")
diff --git a/tests/test_download.py b/tests/test_download.py
index 34942eaf..4a34b3e7 100644
--- a/tests/test_download.py
+++ b/tests/test_download.py
@@ -17,7 +17,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import torch # noqa: E402
from huggingface_hub.utils import filter_repo_objects # noqa: E402
-from safetensors.torch import save_file # noqa: E402
+from safetensors.torch import load_file, save_file # noqa: E402
from tokenizers import Tokenizer # noqa: E402
from tokenizers.models import WordLevel # noqa: E402
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
@@ -113,6 +113,29 @@ class DownloadTests(unittest.TestCase):
self.assertEqual(agent.predict("hello", self.questions), self.expected)
download.assert_not_called()
+ def test_loading_skips_initialization_and_preserves_weights(self):
+ weights = load_file(self.repo / "model.safetensors")
+ for bundled_encoder in (True, False):
+ with self.subTest(bundled_encoder=bundled_encoder), tempfile.TemporaryDirectory() as destination:
+ path = Path(destination)
+ for filename in self.runtime_files:
+ if not bundled_encoder and filename.startswith("encoder/"):
+ continue
+ target = path / filename
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(self.repo / filename, target)
+ cfg_path = path / "rl_agent_config.json"
+ cfg = json.loads(cfg_path.read_text())
+ cfg["encoder"] = str(self.repo / "encoder")
+ cfg_path.write_text(json.dumps(cfg))
+ rng = torch.get_rng_state()
+ with patch("transformers.AutoModel.from_pretrained", side_effect=AssertionError("Unused base weights")):
+ agent = load(str(path), device="cpu")
+ self.assertTrue(torch.equal(torch.get_rng_state(), rng), "Loading initialized random weights")
+ for name, value in agent.model.state_dict().items():
+ torch.testing.assert_close(value, weights[name], rtol=0, atol=0)
+ self.assertEqual(agent.predict("hello", self.questions), self.expected)
+
if __name__ == "__main__":
torch.set_num_threads(1)