Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

docs(reference): the quantize failure is .device on every published checkpoint

Nandakishor10f3b756c06f
diff --git a/docs/reference/agent.md b/docs/reference/agent.md
index 4c3ef836..cf6fe890 100644
--- a/docs/reference/agent.md
+++ b/docs/reference/agent.md
@@ -62,11 +62,23 @@ fbgemm was faster on that CPU (0.39 s) and still moved the score, to 0.15. Valid
probabilities on your own data.
Do not quantize the whole `agent.model`. That replaces the decision head's `nn.Linear` layers
-with dynamically quantized linears, whose `weight` is a method rather than a tensor. `act_head[0]`
-is one of them, and `laya/common.py::DecisionModel.forward` reads `.dtype` off its `weight` whenever
-autocast is off, so `predict` raises
-`AttributeError: 'function' object has no attribute 'dtype'`. The stack above that Linear --
-`agent.model.head`, an `nn.TransformerEncoder` of `head_layers` `nn.TransformerEncoderLayer`s
-(`laya/common.py::DecisionModel.__init__`), and absent entirely when `head_layers` is 0 -- is what
-"the decision head" usually names, and it is not the module that fails. Either way,
-leave the head in fp32.
+with dynamically quantized linears, whose `weight` is a method rather than a tensor, and two
+different reads of that method can be the one that raises.
+
+On every published checkpoint it is the encoder stack. `agent.model.head` is an
+`nn.TransformerEncoder` of `head_layers` `nn.TransformerEncoderLayer`s
+(`laya/common.py::DecisionModel.__init__`, and `None` when `head_layers` is 0), and each layer's
+sparsity fast path reads `.device` on the quantized weights, so `predict` raises
+`AttributeError: 'function' object has no attribute 'device'`. Reproduced on `laya`,
+`laya-multilingual` and `laya-typed-decisions`, qnnpack engine, from
+`torch/nn/modules/transformer.py`'s `(x.device.type in _supported_device_type)` line.
+
+Whether that fast path is taken is what decides which message you get, so a reader who sees the
+other one has not found a different bug. Torch skips the fast path when `self_attn.num_heads` is
+odd, and the head takes `nhead = max(1, d // 64)`, so a model built from a config whose hidden
+size is under 128 gets `nhead = 1`, survives the stack, and fails one step later in
+`DecisionModel.forward`, which reads `.dtype` on `act_head[0].weight` whenever autocast is off:
+`AttributeError: 'function' object has no attribute 'dtype'`. The published checkpoints are
+`d=768` and `d=1024`, so `nhead` is 12 or 16 and never that case.
+
+Either way, leave the head in fp32.
diff --git a/tests/test_decision_model.py b/tests/test_decision_model.py
index ec55e4b5..ad14434f 100644
--- a/tests/test_decision_model.py
+++ b/tests/test_decision_model.py
@@ -139,26 +139,61 @@ def test_head_is_the_encoder_stack_not_a_single_layer():
assert _tiny_model(head_layers=0).head is None
-def test_the_quantized_head_failure_reads_dtype_on_the_action_head():
- # The page also tells readers which attribute blows up when they quantize the whole model.
- # The two facts it names are the attribute the fast path reads and the module it reads it on,
- # so both come off the source, and the page's quoted message has to agree with them. The
- # failure itself is reproducible -- a quantized `act_head[0]` makes `weight` a method and
- # `predict` raises inside `DecisionModel.forward` -- but reproducing it needs a working
- # quantization engine, which is the very thing that page says depends on the CPU. So this
- # pins the reading, not the engine.
+def _quantized_whole_model_error(model):
+ """The AttributeError text `predict` would raise on a fully quantized `model`, or None."""
+ quantized = torch.ao.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
+ try:
+ with torch.no_grad():
+ quantized(*_inputs(batch=1, seq=8, n_markers=1))
+ except AttributeError as exc:
+ return str(exc)
+ return None
+
+
+def test_the_quantized_whole_model_failure_is_the_one_the_page_quotes():
+ # The page tells readers which attribute blows up when they quantize the whole model, and
+ # there are two answers, so it names both and says which one they will get. Torch skips the
+ # encoder layer's sparsity fast path when `self_attn.num_heads` is odd, and the head takes
+ # `nhead = max(1, d // 64)`: a hidden size under 128 gives `nhead = 1`, survives the stack
+ # and fails on `act_head[0].weight.dtype`, while every published checkpoint (d=768, d=1024
+ # -> nhead 12 or 16) takes the fast path and fails earlier, on `.device`. Reproduced here on
+ # both shapes rather than read off the source, because reading the source is exactly what
+ # made the page claim `.dtype` for checkpoints that raise `.device`.
src = inspect.getsource(DecisionModel.forward)
read = re.search(r"self\.act_head\[0\]\.weight\.(\w+)", src)
assert read is not None, "the fast path no longer reads a weight attribute this way"
assert read.group(1) == "dtype", read.group(1)
assert type(_tiny_model().act_head[0]).__name__ == "Linear"
+
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
with open(os.path.join(root, "docs", "reference", "agent.md"), encoding="utf-8") as handle:
page = handle.read()
named = re.findall(r"'function' object has no attribute '(\w+)'", page)
- assert named == ["dtype"], named
+ assert named == ["device", "dtype"], named
assert re.search(r"head\s+is an `nn\.TransformerEncoderLayer`", page) is None, (
"the page calls the head a single layer again")
+ assert "nhead = max(1, d // 64)" in page, "the page no longer names what picks the message"
+
+ # A quantization engine is what the surrounding section says depends on the CPU, so the
+ # reproduction is skipped rather than failed where there is none.
+ try:
+ torch.backends.quantized.engine = "qnnpack"
+ except RuntimeError:
+ return
+ odd = _tiny_model(head_layers=2)
+ assert odd.head.layers[0].self_attn.num_heads % 2 == 1, "the tiny fixture stopped being odd"
+ try:
+ odd_error = _quantized_whole_model_error(odd)
+ except RuntimeError: # no engine for quantized::linear_prepack
+ return
+ assert odd_error == "'function' object has no attribute 'dtype'", odd_error
+
+ even = _tiny_model(head_layers=2)
+ for layer in even.head.layers: # 2 heads over d=16: even, so the fast path runs
+ layer.self_attn = torch.nn.MultiheadAttention(16, 2, batch_first=True)
+ assert even.head.layers[0].self_attn.num_heads % 2 == 0
+ even_error = _quantized_whole_model_error(even)
+ assert even_error == "'function' object has no attribute 'device'", even_error
if __name__ == "__main__":
@@ -167,6 +202,6 @@ if __name__ == "__main__":
test_multi_option_question_is_unaffected()
test_action_head_explicit_dtype_and_autocast()
test_head_is_the_encoder_stack_not_a_single_layer()
- test_the_quantized_head_failure_reads_dtype_on_the_action_head()
+ test_the_quantized_whole_model_failure_is_the_one_the_page_quotes()
print("all decision model tests passed")