Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

review: narrow lang fallback, docs, and flag before end hooks (#361)

Aashishdf5d655b0ab6
diff --git a/README.md b/README.md
index 54c20cf9..49387955 100644
--- a/README.md
+++ b/README.md
@@ -635,6 +635,22 @@ A threshold is a policy you choose from measured accuracy at that coverage on yo
A threshold also depends on the autocast dtype. On CUDA at compute capability 8 or above the runtime uses the checkpoint's `amp_dtype`, which is bf16 for all three shipped checkpoints. On the fixed set from `benchmarks/parity_fast.py` (60 states, 288 questions per checkpoint, RTX 2000 Ada) bf16 moves a probability by up to 0.073 against the fp32 forward and flips 3 of 864 argmaxes across the three checkpoints; fp16 stays within 0.019 and flips none, at the same latency. `LAYA_CUDA_AMP=fp16` selects fp16 and `LAYA_CUDA_AMP=bf16` selects bf16 (`LAYA_CPU_AMP=bf16` is the CPU counterpart). Fit and measure a threshold in the dtype you serve with.
+### Opt-in abstention: `min_confidence`
+
+`predict`, `predict_batch`, `system_one` and `decide` — on `Agent`, `Router` and `ONNXAgent` — take an opt-in `min_confidence`, off by default. It is a caller-side policy on top of the emitted confidence: every answer whose `answer_confidence` falls below the threshold is flagged `low_confidence: True`, with the raw answer, probabilities and confidence left intact for inspection.
+
+```python
+res = agent.predict(state, questions, min_confidence=0.85)
+ans = res["answers"]["department"]
+
+if ans.get("low_confidence"): # answer_confidence < 0.85
+ escalate_to_human_agent(ans["choice"], reason=f"Low confidence ({ans['answer_confidence']:.2f})")
+else:
+ route_automatically(ans["choice"])
+```
+
+The threshold reads `answer_confidence` (`max(p)`) — the calibrated quantity, invariant to the number of options — never the entropy `confidence`. With `decide(..., min_confidence=...)` a low-confidence field comes back as `None` in the schema output, while `return_details=True` keeps the answer and its confidence. [LangChain `LayaRouter`](docs/langchain.md)'s `confidence_threshold` reads the same value: `answer_confidence` when the answer carries it, `confidence` otherwise. Left unset, `min_confidence` changes nothing.
+
---
## Prediction Hooks
diff --git a/docs/langchain.md b/docs/langchain.md
index a5bd8fd3..d6c709a9 100644
--- a/docs/langchain.md
+++ b/docs/langchain.md
@@ -42,7 +42,7 @@ router = LayaRouter(
"sales_agent": "pricing plans, new contracts, demo requests",
},
instructions="Which specialist agent should answer this user query?",
- confidence_threshold=0.80, # If confidence < 0.80, route to human fallback
+ confidence_threshold=0.80, # If answer_confidence < 0.80, route to the fallback
fallback="human_agent",
state_key="input",
)
@@ -71,6 +71,10 @@ result = app.invoke({"input": "I was billed twice for last month's subscription.
print(result["response"]) # -> "Handling billing..."
```
+`confidence_threshold` reads `answer_confidence`, the calibrated `max(p)` confidence the
+calibration figures describe, when the answer carries it, and falls back to the entropy
+`confidence` otherwise.
+
### Routing with the full conversation
When a graph state contains a `messages` list, Laya uses the newest user
diff --git a/laya/agent.py b/laya/agent.py
index 48043a8e..355fe957 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -946,6 +946,8 @@ class Agent(HookRegistry):
ctx.elapsed_ms = (time.perf_counter() - ctx.started_at) * 1000.0
if ctx.results is not None:
ctx.usage = aggregate_usage(ctx.results)
+ if mc is not None:
+ flag_low_confidence(ctx.results, mc)
try:
dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
@@ -954,8 +956,6 @@ class Agent(HookRegistry):
ctx.error.__context__ = hook_exc
else:
raise
- if mc is not None and ctx.results is not None:
- flag_low_confidence(ctx.results, mc)
return ctx.results
def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
diff --git a/laya/common.py b/laya/common.py
index 2cc7bbbd..91f702a5 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -388,7 +388,6 @@ def clamp_temperature(t, lo: float = TEMP_MIN, hi: float = TEMP_MAX) -> float:
return min(hi, max(lo, t))
-
def amp_dtype(name: Optional[str]) -> torch.dtype:
return torch.bfloat16 if name == "bf16" else torch.float16
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 3c6a9414..7afa4bed 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -263,6 +263,8 @@ class ONNXAgent(HookRegistry):
ctx.elapsed_ms = (time.perf_counter() - ctx.started_at) * 1000.0
if ctx.results is not None:
ctx.usage = aggregate_usage(ctx.results)
+ if mc is not None:
+ flag_low_confidence(ctx.results, mc)
try:
dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
@@ -270,8 +272,6 @@ class ONNXAgent(HookRegistry):
ctx.error.__context__ = hook_exc
else:
raise
- if mc is not None and ctx.results:
- flag_low_confidence(ctx.results, mc)
return ctx.results[0]
def _infer(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
diff --git a/laya/router.py b/laya/router.py
index ccfacda2..bf553714 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -606,7 +606,6 @@ class Router(HookRegistry):
result = agent.system_one(ctx.states[0], ctx.questions, lang=effective_lang, **overrides)
except TypeError as e:
if "unexpected keyword argument 'lang'" in str(e):
- overrides.pop("lang", None)
result = agent.system_one(ctx.states[0], ctx.questions, **overrides)
else:
raise
@@ -631,6 +630,8 @@ class Router(HookRegistry):
ctx.elapsed_ms = (time.perf_counter() - ctx.started_at) * 1000.0
if ctx.results is not None:
ctx.usage = aggregate_usage(ctx.results)
+ if mc is not None:
+ flag_low_confidence(ctx.results, mc)
try:
dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
@@ -638,8 +639,6 @@ class Router(HookRegistry):
ctx.error.__context__ = hook_exc
else:
raise
- if mc is not None and ctx.results:
- flag_low_confidence(ctx.results, mc)
return ctx.results[0]
def decide(self, state: Union[str, dict, list], schema: Any = None, *,
@@ -876,12 +875,18 @@ class Router(HookRegistry):
lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
exc.__context__ = hook_exc
+ for ctx in started:
+ if mc is not None and ctx.results:
+ flag_low_confidence(ctx.results, mc)
try:
self._end_contexts(active, started, raise_errors, timeout)
except BaseException as hook_exc:
exc.__context__ = hook_exc
raise
+ for ctx in started:
+ if mc is not None and ctx.results:
+ flag_low_confidence(ctx.results, mc)
self._end_contexts(active, started, raise_errors, timeout)
for i, ctx in zip(indices, started):
results[i] = ctx.results[0]
@@ -891,10 +896,7 @@ class Router(HookRegistry):
if any(result is None for result in results):
raise RuntimeError("internal error: batch execution did not produce every result")
- final = [result for result in results if result is not None]
- if mc is not None:
- flag_low_confidence(final, mc)
- return final
+ return [result for result in results if result is not None]
predict_many = predict_batch
diff --git a/tests/test_batch.py b/tests/test_batch.py
index 9d4d3b63..47b3b845 100644
--- a/tests/test_batch.py
+++ b/tests/test_batch.py
@@ -271,6 +271,40 @@ with patch.object(_agent, "confidence_from_probs", wraps=_agent.confidence_from_
check("decode/noul-only answers skip entropy", entropy.call_count, 0)
+# --------------------------------------------------------------- min_confidence abstention (#361)
+# The flag is set before `on_predict_end` runs, so an audit hook sees exactly what the caller gets.
+def make_confidence_fake():
+ fake = make_fake()
+
+ def decode(logits, act, items, ids, internal, offset):
+ return {qid: {"type": "choice", "choice": qid,
+ "answer_confidence": 0.90 if qid == "a" else 0.40}
+ for qid in ids}
+
+ fake._decode_answers = decode
+ return fake
+
+
+seen = []
+gated = make_confidence_fake().predict_batch(
+ ["s0", "s1"], QUESTIONS, min_confidence=0.5,
+ on_predict_end=lambda ctx: seen.append(
+ [{q: a.get("low_confidence", False) for q, a in r["answers"].items()} for r in ctx.results]))
+check("min_confidence/flags only answers below the threshold",
+ [[r["answers"][q].get("low_confidence", False) for q in ("a", "b")] for r in gated],
+ [[False, True], [False, True]])
+check("min_confidence/end hook sees the same flags", seen, [[{"a": False, "b": True}] * 2])
+check("min_confidence/raw answer and confidence intact",
+ (gated[0]["answers"]["a"]["answer_confidence"], "low_confidence" in gated[0]["answers"]["a"]),
+ (0.90, False))
+check("min_confidence/default leaves results untouched",
+ "low_confidence" in make_confidence_fake().predict_batch(["s0"], QUESTIONS)[0]["answers"]["b"], False)
+check_raises("min_confidence/out-of-range rejected", ValueError,
+ lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=1.5))
+check_raises("min_confidence/bool rejected", ValueError,
+ lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=True))
+
+
# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL:
diff --git a/tests/test_router_batch.py b/tests/test_router_batch.py
index afd521f5..21628bce 100644
--- a/tests/test_router_batch.py
+++ b/tests/test_router_batch.py
@@ -353,3 +353,39 @@ def test_router_predict_and_predict_batch_min_confidence(monkeypatch):
router.predict("hello", {"q1": {}}, min_confidence=True)
with pytest.raises(ValueError):
router.predict_batch([request("hello", model="english")], min_confidence=-0.1)
+
+
+def test_end_hooks_see_the_low_confidence_flag(monkeypatch):
+ """`on_predict_end` sees `low_confidence` on both Router paths (#361 review)."""
+ import laya.agent
+
+ class Agent:
+ def __init__(self, repo, *, device, token, subfolder):
+ pass
+
+ def system_one(self, state, questions, **overrides):
+ return {
+ "model": "fake",
+ "answers": {
+ "q1": {"type": "choice", "choice": "yes", "answer_confidence": 0.95},
+ "q2": {"type": "choice", "choice": "no", "answer_confidence": 0.60},
+ },
+ }
+
+ def predict_batch(self, states, questions, batch_size=None, **overrides):
+ return [self.system_one(s, questions, **overrides) for s in states]
+
+ monkeypatch.setattr(laya.agent, "Agent", Agent)
+ seen = []
+
+ class Record:
+ def on_predict_end(self, ctx):
+ seen.append({q: a.get("low_confidence", False) for q, a in ctx.results[0]["answers"].items()})
+
+ router = Router(max_loaded=1, default="english", hooks=[Record()])
+
+ router.predict("hello", {"q1": {}}, model="english", min_confidence=0.80)
+ assert seen[-1] == {"q1": False, "q2": True}
+
+ router.predict_batch([request("hello", model="english")], min_confidence=0.80)
+ assert seen[-1] == {"q1": False, "q2": True}