Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

20.0 KB
Raw Blame
"""Portability regressions: checkpoints saved by transformers 5, and non-CUDA devices.

Three failures found while getting Laya to run on a machine that is not the one it was written
on, all silent or fatal depending on the platform:

1. transformers 5 records ModernBERT's RoPE bases as
   `rope_parameters = {"full_attention": {...}, "sliding_attention": {...}}`. transformers 4.x
   (the only line that runs on macOS Intel, where PyTorch stops at 2.2.2) does not read that key
   and falls back to its own defaults -- global 160000, local 10000. English and typed-decisions
   happen to match those defaults; mmBERT does not (both of its bases are 160000), so it ran with
   the wrong RoPE base instead of failing loudly.

2. `torch.autocast(device_type=self.device.type, enabled=False)` raises on devices torch has no
   autocast backend for, even when disabled: 'User specified an unsupported autocast device_type
   mps'. Laya only ever enables autocast on CUDA, but it still entered the context on every call,
   so `predict()` died outright on the MPS device torch selects on Apple/AMD machines.

3. `open(path)` and `Path.read_text()` with no `encoding=` take `locale.getpreferredencoding()`,
   which is cp1252 on the `tests (windows)` runner. The pages the suites police -- README.md, the
   docs, the examples -- are UTF-8 and carry characters cp1252 has no mapping for, so a test that
   reads one of them dies at the read with `UnicodeDecodeError: 'charmap' codec can't decode byte
   0x81` instead of failing one assertion. It has cost the runner twice: once at a metadata gate
   that read README.md, and once before that at the batch-shape gate, whose fix is recorded in
   tests/test_mcp.py's comment at its README read. Section 4 below gates the convention.
"""
import ast
import glob
import os
import sys
from contextlib import nullcontext
from types import SimpleNamespace
from unittest import mock

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import torch  # noqa: E402

from laya import agent as _agent  # noqa: E402
from laya.common import _apply_rope_config  # noqa: E402

PASS, FAIL = [], []


def check(name, got, want):
    if got == want:
        PASS.append(name)
    else:
        FAIL.append("%s:\n     got  %r\n     want %r" % (name, got, want))


def check_true(name, cond, detail=""):
    if cond:
        PASS.append(name)
    else:
        FAIL.append("%s %s" % (name, detail))


# ------------------------------------------------ 1. transformers-5 RoPE config -> transformers 4.x
def cfg_with(rope_parameters, **kwargs):
    """A stand-in for a ModernBertConfig carrying both the 5.x and the 4.x view of RoPE."""
    base = dict(global_rope_theta=160000.0, local_rope_theta=10000.0)   # 4.x defaults
    base.update(kwargs)
    return SimpleNamespace(rope_parameters=rope_parameters, **base)


# the mmBERT case: both bases are 160000, so the 4.x default local theta (10000) is wrong
mmbert = cfg_with({"full_attention": {"rope_theta": 160000, "rope_type": "default"},
                   "sliding_attention": {"rope_theta": 160000, "rope_type": "default"}})
_apply_rope_config(mmbert)
check("rope/mmbert global theta", mmbert.global_rope_theta, 160000.0)
check("rope/mmbert local theta corrected", mmbert.local_rope_theta, 160000.0)

# ModernBERT-large / typed-decisions: 160000 / 10000 -- already the 4.x defaults, must not move
modernbert = cfg_with({"full_attention": {"rope_theta": 160000.0, "rope_type": "default"},
                       "sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}})
_apply_rope_config(modernbert)
check("rope/modernbert global theta unchanged", modernbert.global_rope_theta, 160000.0)
check("rope/modernbert local theta unchanged", modernbert.local_rope_theta, 10000.0)

# a config that carries no rope_parameters at all (every 4.x-era checkpoint) must be left alone
native = cfg_with(None, global_rope_theta=1234.0, local_rope_theta=5678.0)
_apply_rope_config(native)
check("rope/absent rope_parameters is a no-op", (native.global_rope_theta, native.local_rope_theta),
      (1234.0, 5678.0))

# transformers 5 keeps the thetas only inside rope_parameters; the old attributes are gone, so
# there is nothing to write and the call must not raise
v5_only = SimpleNamespace(rope_parameters={"full_attention": {"rope_theta": 160000},
                                           "sliding_attention": {"rope_theta": 160000}})
try:
    _apply_rope_config(v5_only)
    check_true("rope/transformers-5 config is a no-op", not hasattr(v5_only, "global_rope_theta"))
except Exception as e:
    FAIL.append("rope/transformers-5 config raised: %r" % e)

# a flat rope_parameters mapping is also accepted
flat = cfg_with({"rope_theta": 10000.0, "rope_type": "default"})
_apply_rope_config(flat)
check("rope/flat rope_parameters", (flat.global_rope_theta, flat.local_rope_theta), (10000.0, 10000.0))


# ------------------------------------------------ 2. autocast is never entered off CUDA
def _autocast_that_rejects_non_cuda(entered):
    def fake(*args, **kwargs):
        entered.append(kwargs.get("device_type"))
        device_type = kwargs.get("device_type")
        if device_type != "cuda":
            raise RuntimeError("User specified an unsupported autocast device_type %r" % device_type)
        return nullcontext()
    return fake


# A disabled context must never enter torch.autocast: on a torch build with no MPS autocast
# backend, entering it raises. The gate (Agent._amp_enabled_for) decides `enabled`.
for name in ("cpu", "mps", "xpu"):
    entered = []
    with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):
        try:
            ctx = _agent._amp_context(SimpleNamespace(type=name), torch.float32, False)
            with ctx:
                pass
            check_true("amp/%s never enters autocast when disabled" % name, entered == [],
                       "entered %s" % entered)
        except RuntimeError as e:
            FAIL.append("amp/%s raised %s" % (name, e))

check_true("amp/disabled is a no-op",
           isinstance(_agent._amp_context(SimpleNamespace(type="mps"), torch.float16, False), nullcontext))

# CUDA still gets mixed precision, with the dtype the model was configured for
entered = []
with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):
    with _agent._amp_context(SimpleNamespace(type="cuda"), torch.float16, True):
        pass
check("amp/cuda uses autocast", entered, ["cuda"])

# the old unconditional call is what broke MPS -- make sure it cannot come back
import inspect  # noqa: E402

_src = inspect.getsource(_agent.Agent._infer)
check_true("amp/infer goes through _amp_context", "with _amp_context(self.device, self.dtype, enabled)" in _src)
check_true("amp/no unconditional autocast in the forward pass",
           "torch.autocast(device_type=self.device.type" not in _src)


# ------------------------------------------------ 3. a failed GPU placement falls back to CPU
# `system_one` promises to survive a device that cannot hold the work: on a memory error it moves
# to CPU and re-runs. That path is easy to break and never fires on a machine with no GPU, so it
# is driven here with a stand-in model that fails the first forward pass the way CUDA does.
class _FakeTok:
    cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
    mask_token = "[M]"

    def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
        ids = [10 + (ord(c) % 40) for c in text]
        if truncation and max_length:
            ids = ids[:max_length]
        return {"input_ids": ids}


class _FailsOnce(torch.nn.Module):
    """Raises a CUDA-style OOM on the first call, then answers."""

    def __init__(self, message):
        super().__init__()
        self.message, self.calls = message, 0
        self.dummy = torch.nn.Parameter(torch.zeros(1))

    def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):
        self.calls += 1
        if self.calls == 1:
            raise RuntimeError(self.message)
        logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1]))
        logits[:, 0] = 1.0
        return logits, torch.tensor([[1.0, 0.0]])


class _RestoreFails(_FailsOnce):
    """OOMs once like _FailsOnce, but also refuses to move back to the accelerator afterwards,
    so `_restore_runtime`'s `model.to(device)` fails and the agent must stay on CPU."""
    def to(self, *args, **kwargs):
        target = args[0] if args else kwargs.get("device")
        dt = getattr(target, "type", None) or (target if isinstance(target, str) else None)
        if dt == "mps":
            raise RuntimeError("CUDA out of memory: model no longer fits after the retry")
        return super().to(*args, **kwargs)


def _bare_agent(model):
    agent = _agent.Agent.__new__(_agent.Agent)     # no weights: exercise system_one only
    agent.device = torch.device("mps")             # any non-CPU device enters the fallback branch
    agent.dtype = torch.float32
    agent.cfg = {"max_len": 64, "head_max_len": 32}
    agent.temperature = [1.0, 1.0, 1.0]
    agent.temperature_by_options = {}
    agent.tok = _FakeTok()
    agent.model = model
    return agent


QUESTIONS = {"q": {"type": "choice", "instructions": "Pick one",
                   "criteria": {"a": "first option", "b": "second option"}}}

# Placement on the stand-in device is faked so the scenario also runs where torch has no
# such backend compiled in (the CPU-only wheels CI installs): batch `.to(device)` calls
# targeting the fake device become no-ops and the stand-in model supplies the failure,
# exactly as the real device would.
_real_tensor_to = torch.Tensor.to


def _to_that_ignores_mps(self, *args, **kwargs):
    target = args[0] if args else kwargs.get("device")
    if (isinstance(target, torch.device) and target.type == "mps") or target == "mps":
        return self
    return _real_tensor_to(self, *args, **kwargs)


with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):
    agent = _bare_agent(_FailsOnce("CUDA out of memory. Tried to allocate 2.00 GiB"))
    try:
        result = agent.predict({"body": "some state"}, QUESTIONS)
        check("fallback/answers after a memory failure", result["answers"]["q"]["choice"], "a")
        check("fallback/forward pass retried once", agent.model.calls, 2)
        # the demotion is scoped to the failed request (#344): one oversized call must not
        # leave every later call on a ~10-15x slower CPU path for the life of the process
        check("fallback/device restored after the retry", agent.device.type, "mps")
        check("fallback/a later request is answered without a new demotion",
              agent.predict({"body": "another state"}, QUESTIONS)["answers"]["q"]["choice"], "a")
        check("fallback/later request needs no extra forward", agent.model.calls, 3)
        check("fallback/device still the original after a later call", agent.device.type, "mps")
    except Exception as e:  # noqa: BLE001
        FAIL.append("fallback/memory failure was not survived: %s: %s" % (type(e).__name__, e))

    # a non-memory RuntimeError must still propagate: silently swallowing real bugs is worse
    # than the crash it would hide
    agent = _bare_agent(_FailsOnce("shape mismatch in attention"))
    try:
        agent.predict({"body": "some state"}, QUESTIONS)
        FAIL.append("fallback/non-memory error propagates (nothing raised)")
    except RuntimeError:
        PASS.append("fallback/non-memory error propagates")
    except Exception as e:  # noqa: BLE001
        FAIL.append("fallback/non-memory error raised %s instead of RuntimeError" % type(e).__name__)

    # a RuntimeError that merely mentions cuda is not an OOM (#344): it used to demote the
    # agent to CPU permanently and return silently-CPU results
    agent = _bare_agent(_FailsOnce("CUDA error: device-side assert triggered"))
    try:
        agent.predict({"body": "some state"}, QUESTIONS)
        FAIL.append("fallback/cuda-worded non-memory error propagates (nothing raised)")
    except RuntimeError:
        PASS.append("fallback/cuda-worded non-memory error propagates")
    except Exception as e:  # noqa: BLE001
        FAIL.append("fallback/cuda-worded non-memory error raised %s instead of RuntimeError"
                    % type(e).__name__)

    # restore fails: the model no longer fits the accelerator after the CPU retry, so it stays
    # demoted rather than crashing a request that already succeeded
    agent = _bare_agent(_RestoreFails("CUDA out of memory. Tried to allocate 2.00 GiB"))
    try:
        result = agent.predict({"body": "some state"}, QUESTIONS)
        check("fallback/restore-failure still answers the request", result["answers"]["q"]["choice"], "a")
        check("fallback/restore-failure stays on cpu", agent.device.type, "cpu")
        check("fallback/restore-failure: later call runs on cpu without crashing",
              agent.predict({"body": "another state"}, QUESTIONS)["answers"]["q"]["choice"], "a")
        check("fallback/restore-failure: no extra forward beyond retry + later call", agent.model.calls, 3)
    except Exception as e:  # noqa: BLE001
        FAIL.append("fallback/restore-failure not survived: %s: %s" % (type(e).__name__, e))

    # one unsupported autocast op retries that request and leaves AMP on. Three misses in a
    # row disable it, so a build without the op does not pay for two forwards forever (#351).
    # CPU so the MPS row gate does not hide the branch.
    class _AutocastMisses(torch.nn.Module):
        def __init__(self):
            super().__init__()
            self.dummy = torch.nn.Parameter(torch.zeros(1))
            self.calls = 0

        def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):
            self.calls += 1
            if self.calls in (1, 3, 5):
                raise RuntimeError("User specified an unsupported autocast device_type cpu")
            logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1]))
            logits[:, 0] = 1.0
            return logits, torch.tensor([[1.0, 0.0]])

    agent = _bare_agent(_AutocastMisses())
    agent.device = torch.device("cpu")
    agent.amp_enabled = True
    agent.dtype = torch.float16
    try:
        result = agent.predict({"body": "some state"}, QUESTIONS)
        check("autocast/answers after one miss", result["answers"]["q"]["choice"], "a")
        check("autocast/one miss retries once", agent.model.calls, 2)
        check("autocast/one miss keeps amp", agent.amp_enabled, True)
        check("autocast/one miss keeps dtype", agent.dtype, torch.float16)
        agent.predict({"body": "another state"}, QUESTIONS)
        check("autocast/two misses keep amp", agent.amp_enabled, True)
        agent.predict({"body": "third state"}, QUESTIONS)
        check("autocast/third miss disables amp", agent.amp_enabled, False)
        check("autocast/third miss drops dtype", agent.dtype, torch.float32)
        check("autocast/three misses are six forwards", agent.model.calls, 6)
        agent.predict({"body": "later state"}, QUESTIONS)
        check("autocast/later request is one forward", agent.model.calls, 7)
    except Exception as e:  # noqa: BLE001
        FAIL.append("autocast/streak was not survived: %s: %s" % (type(e).__name__, e))


# ------------------------------------------------ 4. a repo read must pin its encoding, not take the
# locale's. See the module docstring: on `tests (windows)` the locale codec is cp1252, so reading a
# UTF-8 page raises at the read and the whole suite exits 1 rather than failing one assertion.
TEXT_SUFFIXES = (".md", ".py", ".nix", ".txt", ".yml", ".yaml", ".json", ".toml")
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def _repo_reads(src):
    """The reads of a repo file `src` carries, as `(lineno, call name, path expression, pinned)`.

    Writes are out of scope: this is about decoding, which only a read does. A read is in scope when
    its path expression names a file that exists in the repo, or builds from the module's own repo
    root (`ROOT`, `parents[`, `dirname(`, `__file__`). The second arm is deliberately blunt --
    `open(os.path.join(ROOT, rel))` is how both crashes were written, and `rel` cannot be resolved
    without running the module, so a read of that shape is asked to pin its encoding rather than
    proven dangerous.
    """
    out = []
    for node in ast.walk(ast.parse(src)):
        if not isinstance(node, ast.Call):
            continue
        fn = node.func
        name = fn.attr if isinstance(fn, ast.Attribute) else (fn.id if isinstance(fn, ast.Name) else "")
        if name == "open" and node.args:
            target = node.args[0]
        elif name == "read_text" and isinstance(fn, ast.Attribute):
            # `Path.read_text()` carries its path on the object, not in `args`.
            target = fn.value
        else:
            continue
        modes = [ast.unparse(k.value) for k in node.keywords if k.arg == "mode"]
        if name == "open" and len(node.args) > 1:
            modes.append(ast.unparse(node.args[1]))
        if any("w" in m or "a" in m or "b" in m for m in modes):
            continue
        arg = ast.unparse(target)
        literals = [n.value for n in ast.walk(target)
                    if isinstance(n, ast.Constant) and isinstance(n.value, str)]
        names_repo_file = any(l.endswith(TEXT_SUFFIXES) and os.path.exists(os.path.join(REPO, l))
                              for l in literals)
        from_root = any(tok in arg for tok in ("ROOT", "parents[", "dirname(", "__file__"))
        if names_repo_file or from_root:
            out.append((node.lineno, name, arg, any(k.arg == "encoding" for k in node.keywords)))
    return out


reads_by_suite = {}
for _path in sorted(glob.glob(os.path.join(REPO, "tests", "*.py"))):
    with open(_path, encoding="utf-8") as _fh:
        reads_by_suite["tests/" + os.path.basename(_path)] = _repo_reads(_fh.read())

unpinned = ["%s:%d %s" % (_rel, _lineno, _arg)
            for _rel, _reads in sorted(reads_by_suite.items())
            for _lineno, _name, _arg, _pinned in _reads if not _pinned]
check("encoding/no repo read in tests/ leaves the codec to the locale", unpinned, [])

# The rule must not pass by finding nothing: it has to see the reads that already pin the encoding.
pinned_seen = sum(1 for _reads in reads_by_suite.values() for _, _, _, _p in _reads if _p)
check_true("encoding/the scan sees the reads that already pin it", pinned_seen >= 10,
           "only %d pinned repo reads found, so the scan is not reaching the suites" % pinned_seen)

# And it has to see both call shapes. The first version of this scanner read `node.args`, which a
# `Path.read_text()` does not use -- it carries its path on the object -- so it reported a clean repo
# while tests/test_evals.py still decoded docs/evals.md with the locale codec. A scan that reaches one
# shape only passes the half of the convention it can still see.
shapes_seen = {_name for _reads in reads_by_suite.values() for _, _name, _, _ in _reads}
check_true("encoding/the scan sees both open() and Path.read_text()",
           shapes_seen >= {"open", "read_text"},
           "the scan reached %s only" % sorted(shapes_seen))

# And the premise has to hold: pages the suites read really are UTF-8 that cp1252 cannot decode.
# Without one such page the rule is a style preference, not a crash guard.
_repo_text = []
for _pattern in ("README.md", os.path.join("docs", "**", "*.md"),
                 os.path.join("examples", "*.py"), os.path.join("laya", "*.py")):
    _repo_text.extend(glob.glob(os.path.join(REPO, _pattern), recursive=True))
_undecodable = []
for _page in _repo_text:
    try:
        with open(_page, encoding="cp1252") as _fh:
            _fh.read()
    except UnicodeDecodeError:
        _undecodable.append(os.path.relpath(_page, REPO).replace(os.sep, "/"))
check_true("encoding/some repo page is UTF-8 the Windows locale cannot decode", bool(_undecodable),
           "every repo page decodes as cp1252, so the crash this gate guards has nothing to crash on")
print("encoding: %d repo pages cp1252 cannot decode, %d repo reads in tests/ all pinned"
      % (len(_undecodable), pinned_seen))


print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
    print("  FAIL " + f)
sys.exit(1 if FAIL else 0)