Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

40.8 KB
Raw Blame
"""Unit tests for Laya LlamaIndex integration.

Tests verify single/multi-selector logic, confidence threshold fallback gating,
query router dispatch, and LlamaIndex schema conventions without requiring
model downloads, GPU, or external services.
"""
import asyncio
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from laya.integrations.llamaindex import (
    LayaLowConfidenceError,
    LayaMultiSelector,
    LayaQueryRouter,
    LayaSingleSelector,
    QueryBundle,
    ToolMetadata,
    _extract_query_str,
    _format_choices_criteria,
)

PASS, FAIL = [], []


def check(name, got, want):
    if got == want:
        PASS.append(name)
    else:
        FAIL.append(f"{name}:\n     got  {got!r}\n     want {want!r}")


def check_true(name, cond, detail=""):
    if cond:
        PASS.append(name)
    else:
        FAIL.append(f"{name} {detail}")


# --------------------------------------------------------------- Mock Agent
class MockLayaAgent:
    """Mock agent returning deterministic responses for testing."""

    def __init__(self, response_fn):
        self.response_fn = response_fn

    def predict(self, state, questions, **kwargs):
        return self.response_fn(state, questions)


# --------------------------------------------------------------- 1. Query & Criteria Formatting
check("extract/str", _extract_query_str("What is the revenue?"), "What is the revenue?")
check(
    "extract/query_bundle",
    _extract_query_str(QueryBundle(query_str="How do I reset password?")),
    "How do I reset password?",
)


class CustomQueryObj:
    def __init__(self, text):
        self.query_str = text


check(
    "extract/custom_query_obj",
    _extract_query_str(CustomQueryObj("custom text")),
    "custom text",
)

# Test criteria formatting from ToolMetadata
tools = [
    ToolMetadata(name="vector_store", description="Search financial documents"),
    ToolMetadata(name="sql_db", description="Query relational tables"),
    {"name": "summary_tool", "description": "High level summarization"},
    "Simple string description tool",
]
criteria = _format_choices_criteria(tools)
check(
    "criteria/vector_store",
    criteria["choice_0"],
    "vector_store: Search financial documents",
)
check(
    "criteria/sql_db",
    criteria["choice_1"],
    "sql_db: Query relational tables",
)
check(
    "criteria/summary_tool",
    criteria["choice_2"],
    "summary_tool: High level summarization",
)
check(
    "criteria/string_tool",
    criteria["choice_3"],
    "Simple string description tool",
)


# --------------------------------------------------------------- 2. LayaSingleSelector
def mock_single_selector_response(state, questions):
    text = str(state).lower()
    if "sql" in text or "table" in text or "database" in text:
        chosen = "choice_1"
        conf = 0.94
    elif "lowconf" in text:
        chosen = "choice_0"
        conf = 0.42
    else:
        chosen = "choice_0"
        conf = 0.91

    return {
        "model": "mock-single",
        "answers": {
            "selector": {
                "choice": chosen,
                "answer_confidence": conf,
                "confidence": conf,
            }
        },
    }


mock_agent = MockLayaAgent(mock_single_selector_response)
single_selector = LayaSingleSelector(agent=mock_agent)

# Standard selection for choice_0
res0 = single_selector.select(tools, "Search for quarterly 10-K report")
check("single/index_0", res0.selections[0].index, 0)
check_true(
    "single/reason_0",
    "Selected 'vector_store' via Laya System 1 decision" in res0.selections[0].reason,
)

# Standard selection for choice_1 (sql)
res1 = single_selector.select(tools, QueryBundle(query_str="Query the SQL database for customers"))
check("single/index_1", res1.selections[0].index, 1)
check_true(
    "single/reason_1",
    "Selected 'sql_db' via Laya System 1 decision" in res1.selections[0].reason,
)

# Confidence threshold with fallback
gated_selector = LayaSingleSelector(
    agent=mock_agent,
    confidence_threshold=0.80,
    fallback_index=2,
)
res_fallback = gated_selector.select(tools, "lowconf query here")
check("single/fallback_index", res_fallback.selections[0].index, 2)
check_true(
    "single/fallback_reason",
    "Selected fallback index 2" in res_fallback.selections[0].reason,
)

# Confidence threshold with raise_on_low_confidence
raising_selector = LayaSingleSelector(
    agent=mock_agent,
    confidence_threshold=0.80,
    raise_on_low_confidence=True,
)
try:
    raising_selector.select(tools, "lowconf query here")
    check("single/raise_error", False, True)
except LayaLowConfidenceError as e:
    check("single/raise_error", True, True)
    check("single/error_conf", e.confidence, 0.42)
    check("single/error_thresh", e.threshold, 0.80)

# Empty choices validation
try:
    single_selector.select([], "any query")
    check("single/empty_choices", False, True)
except ValueError:
    check("single/empty_choices", True, True)


# Async single selector
async def run_async_single():
    res_async = await single_selector.aselect(tools, "Search the SQL database")
    check("single/async_index", res_async.selections[0].index, 1)


asyncio.run(run_async_single())


# --------------------------------------------------------------- 3. LayaMultiSelector
def mock_multi_selector_response(state, questions):
    text = str(state).lower()
    if "compare" in text or "both" in text:
        # High probability for both vector (0) and sql (1)
        probs = {"choice_0": 0.55, "choice_1": 0.40, "choice_2": 0.05}
    elif "none_qualify" in text:
        # All below threshold
        probs = {"choice_0": 0.15, "choice_1": 0.10, "choice_2": 0.05}
    else:
        probs = {"choice_0": 0.85, "choice_1": 0.10, "choice_2": 0.05}

    return {
        "model": "mock-multi",
        "answers": {
            "selector": {
                "choice": max(probs, key=probs.get),
                "probabilities": probs,
                "answer_confidence": 0.85,
            }
        },
    }


mock_multi_agent = MockLayaAgent(mock_multi_selector_response)
multi_selector = LayaMultiSelector(
    agent=mock_multi_agent,
    probability_threshold=0.30,
)

# Multi-select returns multiple tools
res_multi = multi_selector.select(tools[:3], "Compare both vector docs and SQL records")
check("multi/count", len(res_multi.selections), 2)
check("multi/idx_0", res_multi.selections[0].index, 0)
check("multi/idx_1", res_multi.selections[1].index, 1)

# Multi-select with max_outputs limit
limited_selector = LayaMultiSelector(
    agent=mock_multi_agent,
    probability_threshold=0.30,
    max_outputs=1,
)
res_limited = limited_selector.select(tools[:3], "Compare both vector docs and SQL records")
check("multi/limited_count", len(res_limited.selections), 1)
check("multi/limited_idx", res_limited.selections[0].index, 0)

# Multi-select fallback to top option when none meet threshold
res_none = multi_selector.select(tools[:3], "none_qualify search")
check("multi/fallback_top_count", len(res_none.selections), 1)
check("multi/fallback_top_idx", res_none.selections[0].index, 0)


# Async multi-selector
async def run_async_multi():
    res_async = await multi_selector.aselect(tools[:3], "Compare both sources")
    check("multi/async_count", len(res_async.selections), 2)


asyncio.run(run_async_multi())


# --------------------------------------------------------------- 4. LayaQueryRouter
class MockQueryEngine:
    def __init__(self, name):
        self.name = name

    def query(self, q):
        return f"Result from {self.name} for: {q}"

    async def aquery(self, q):
        return f"Async result from {self.name} for: {q}"


engines = {
    "vector": MockQueryEngine("VectorEngine"),
    "sql": MockQueryEngine("SqlEngine"),
    "summary": MockQueryEngine("SummaryEngine"),
}


def mock_router_dispatcher(state, questions):
    text = str(state).lower()
    if "sql" in text or "table" in text:
        chosen = "sql"
        conf = 0.96
    elif "summary" in text:
        chosen = "summary"
        conf = 0.91
    elif "lowconf" in text:
        chosen = "summary"
        conf = 0.35
    else:
        chosen = "vector"
        conf = 0.89

    return {
        "model": "mock-router",
        "answers": {
            "route": {
                "choice": chosen,
                "answer_confidence": conf,
                "confidence": conf,
            }
        },
    }


router_agent = MockLayaAgent(mock_router_dispatcher)
laya_qr = LayaQueryRouter(
    query_engines=engines,
    descriptions={
        "vector": "Semantic search over documentation",
        "sql": "Relational SQL queries",
        "summary": "Document summarization",
    },
    agent=router_agent,
    confidence_threshold=0.70,
    fallback_key="vector",
)

# Route direct
check("router/route_sql", laya_qr.route("Run a query on the SQL table"), "sql")
check("router/route_summary", laya_qr.route("Provide an executive summary"), "summary")
check("router/route_vector", laya_qr.route("How does authentication work?"), "vector")

# Confidence fallback
check("router/route_fallback", laya_qr.route("lowconf request"), "vector")

# Execute query()
res_q = laya_qr.query("Run a query on the SQL table")
check("router/query_exec", res_q, "Result from SqlEngine for: Run a query on the SQL table")


# Async query
async def run_async_router():
    res_aq = await laya_qr.aquery("Provide an executive summary")
    check("router/aquery_exec", res_aq, "Async result from SummaryEngine for: Provide an executive summary")


asyncio.run(run_async_router())


# Callable engine support
callable_engines = {
    "calc": lambda q: f"Calculated: {q}",
    "search": lambda q: f"Searched: {q}",
}
callable_qr = LayaQueryRouter(
    query_engines=callable_engines,
    agent=MockLayaAgent(
        lambda s, q: {"answers": {"route": {"choice": "calc", "answer_confidence": 0.99}}}
    ),
)
check("router/callable", callable_qr.query("2 + 2"), "Calculated: 2 + 2")


# --------------------------------------------------------------- 5. Remote HTTP Execution (Mocked)
from unittest.mock import MagicMock, patch


class DummyHTTPResponse:
    def __init__(self, data_dict):
        self.data = json.dumps(data_dict).encode("utf-8")

    def read(self):
        return self.data

    def __enter__(self):
        return self

    def __exit__(self, *args):
        pass


import json

remote_response = {
    "model": "laya-multilingual",
    "answers": {
        "selector": {
            "choice": "choice_1",
            "answer_confidence": 0.98,
            "confidence": 0.98,
        }
    },
}

with patch("urllib.request.build_opener") as mock_build_opener:
    mock_opener = MagicMock()
    mock_opener.open.return_value = DummyHTTPResponse(remote_response)
    mock_build_opener.return_value = mock_opener

    remote_selector = LayaSingleSelector(
        base_url="http://localhost:8000",
        api_key="sk-test-token",
    )
    res_remote = remote_selector.select(tools, "Query via remote server")
    check("remote/index", res_remote.selections[0].index, 1)

    # Check request headers and URL
    call_args = mock_opener.open.call_args
    req = call_args[0][0]
    check("remote/url", req.full_url, "http://localhost:8000/v1/systemone")
    check("remote/auth", req.headers.get("Authorization"), "Bearer sk-test-token")


# --------------------------------------------------------------- 5b. Non-Object Remote Response
#
# A remote laya-serve body is arbitrary JSON: a gateway, a mis-pointed `base_url`, or a proxy error
# page can answer with a top-level array, string, number or `true`/`null`. `json.loads` accepts each
# of those and the consumers read the result with `res.get("answers", …)` (the selector and route
# lookups), so before the guard a non-object leaked `AttributeError: 'list' object has no attribute
# 'get'` past the fail-closed path instead of the `RuntimeError` every other remote failure raises
# here. `_call_remote` and the public `select()` are both driven so a regression at either layer is
# caught.
from laya.integrations.llamaindex import _call_remote as _llama_call_remote

_LLAMA_NON_OBJECT_BODIES = [
    ([], "list"),
    ([{"answers": {"selector": {"choice": "choice_1"}}}], "list-of-objects"),
    ("unauthorized", "str"),
    (None, "null"),
    (123, "int"),
    (True, "bool"),
]
for _body, _label in _LLAMA_NON_OBJECT_BODIES:
    with patch("urllib.request.build_opener") as _m:
        _m.return_value.open.return_value = DummyHTTPResponse(_body)
        try:
            _llama_call_remote("http://localhost:8000", "s", {"q": {}})
            check_true("nonobj/%s/_call_remote raises" % _label, False, "no error raised")
        except RuntimeError as exc:
            check_true("nonobj/%s/malformed" % _label, "Malformed decision response" in str(exc))
            check_true("nonobj/%s/names object" % _label, "expected a JSON object" in str(exc))
        except Exception as exc:
            check_true("nonobj/%s/_call_remote raises" % _label, False, type(exc).__name__)

        try:
            LayaSingleSelector(base_url="http://localhost:8000", api_key="k").select(
                tools, "Query via remote server")
            check_true("nonobj/%s/select raises" % _label, False, "no error raised")
        except RuntimeError as exc:
            check_true("nonobj/%s/select malformed" % _label,
                       "Malformed decision response" in str(exc))
        except Exception as exc:
            check_true("nonobj/%s/select surfaces RuntimeError" % _label, False, type(exc).__name__)

# Positive control: a well-formed object body still selects -- the guard does not over-reject.
with patch("urllib.request.build_opener") as _m:
    _m.return_value.open.return_value = DummyHTTPResponse(remote_response)
    check("nonobj/object still selects",
          LayaSingleSelector(base_url="http://localhost:8000").select(
              tools, "Query via remote server").selections[0].index, 1)


# --------------------------------------------------------------- 5c. Non-Object `answers` Field
#
# Section 5 makes `_call_remote` refuse a top-level array, string, number or `true`/`null`. One level
# deeper is the same arbitrary JSON: a gateway can answer `{"answers": []}`. `res.get("answers", {})`
# hands back the list -- the key is present, so the `{}` default never fires -- and the lookup then
# crashes on `'list' object has no attribute 'get'`, which is not the fail-closed path. Three call
# sites read it that way (the single selector, the multi selector and the query router); the two with
# different answer keys are driven here, and the multi selector shares the single one's line. A body
# without `answers`, or with `answers` as an object (even an empty one), is a no-decision response
# rather than a malformed one, and has to keep selecting and routing exactly as it does today.
from laya.integrations.llamaindex import _call_remote as _llama_call_remote

_LLAMA_BAD_ANSWERS = [
    (["boom"], "list"),
    ("boom", "str"),
    (3, "int"),
    (True, "bool"),
    (None, "null"),
]
for _answers, _label in _LLAMA_BAD_ANSWERS:
    with patch("urllib.request.build_opener") as _m:
        _m.return_value.open.return_value = DummyHTTPResponse({"answers": _answers})
        try:
            _llama_call_remote("http://localhost:8000", "s", {"q": {}})
            check_true("badans/%s/_call_remote raises" % _label, False, "no error raised")
        except RuntimeError as _exc:
            check_true("badans/%s/names answers" % _label,
                       "'answers' must be a JSON object" in str(_exc), str(_exc))
        except Exception as _exc:
            check_true("badans/%s/_call_remote raises" % _label, False, type(_exc).__name__)

        for _entry, _run in (
            ("select", lambda: LayaSingleSelector(base_url="http://localhost:8000",
                                                  api_key="k").select(
                                                     tools, "Query via remote server")),
            ("route", lambda: LayaQueryRouter(query_engines=engines, base_url="http://localhost:8000",
                                              api_key="k").route("Query via remote server")),
        ):
            try:
                _run()
                check_true("badans/%s/%s raises" % (_label, _entry), False, "no error raised")
            except RuntimeError:
                check_true("badans/%s/%s raises" % (_label, _entry), True)
            except Exception as _exc:
                check_true("badans/%s/%s raises" % (_label, _entry), False, type(_exc).__name__)

# The refusal names the offending type -- with `answers` nested inside an otherwise valid body, that
# is the only thing a caller reading a log line can act on.
with patch("urllib.request.build_opener") as _m:
    _m.return_value.open.return_value = DummyHTTPResponse({"answers": ["boom"]})
    try:
        _llama_call_remote("http://localhost:8000", "s", {"q": {}})
        check_true("badans/names the type", False, "no error raised")
    except RuntimeError as _exc:
        check_true("badans/names the type", "got list" in str(_exc), str(_exc))

# Controls: the two no-decision shapes still select and still route.
for _body, _label in (({"model": "laya-multilingual"}, "answers absent"),
                      ({"answers": {}}, "answers empty")):
    with patch("urllib.request.build_opener") as _m:
        _m.return_value.open.return_value = DummyHTTPResponse(_body)
        check("badans/%s still selects" % _label,
              LayaSingleSelector(base_url="http://localhost:8000").select(
                  tools, "Query via remote server").selections[0].index, 0)
        check("badans/%s still routes" % _label,
              LayaQueryRouter(query_engines=engines, base_url="http://localhost:8000").route(
                  "Query via remote server"), next(iter(engines)))


# --------------------------------------------------------------- 6. Per-call decision controls
#
# `laya.integrations.langchain` has forwarded `max_len` / `head_max_len` and the five hook
# arguments since #530 / #532; this surface forwarded only `model`, so a RAG pipeline choosing
# between more tools than the default head budget fits could not widen its own window. The rule
# now lives in one module the three integrations import, and the lists below are read out of it
# rather than written out again here -- a control added to `._controls` without reaching this file
# fails here. Three classes, three `_execute_decision` call sites, so every surface is driven.
import inspect
from laya.agent import Agent
from laya.integrations import _controls
from laya.integrations import crewai as crewai_module
from laya.integrations import langchain as langchain_module
from laya.integrations import llamaindex as llamaindex_module
from laya.router import Router

CONTROLS = (tuple(_controls.PREDICT_CONTROLS) + tuple(_controls.DECISION_CONTROLS)
            + tuple(_controls.HOOK_CONTROLS))
ROUTER_HINTS = tuple(_controls.ROUTER_ONLY_CONTROLS)


def _params(fn):
    return set(inspect.signature(fn).parameters)


check("controls/budget tuple names budget_kwargs",
      set(_params(_controls.budget_kwargs)), set(_controls.PREDICT_CONTROLS))
check("controls/decision tuple names decision_kwargs",
      set(_params(_controls.decision_kwargs)), set(_controls.DECISION_CONTROLS))
check("controls/hook tuple names hook_kwargs",
      set(_params(_controls.hook_kwargs)), set(_controls.HOOK_CONTROLS))
check("controls/routing tuple names router_kwargs",
      set(_params(_controls.router_kwargs)), set(_controls.ROUTER_ONLY_CONTROLS))

# Everything called a control here is an argument a real runner accepts, or a selector would fail
# with a TypeError deep inside core instead of at the call site.
_agent_params = _params(Agent.system_one)
_router_params = _params(Router.predict)
for _c in CONTROLS:
    check_true("controls/%s accepted by Agent" % _c, _c in _agent_params)
    check_true("controls/%s accepted by Router.predict" % _c, _c in _router_params)

# The two routing hints are the reverse case, and it is read from core rather than written down:
# a Router has a routing step to read them in and an Agent does not, so the wrapper's rule --
# forward at one, refuse at the other -- is only correct while these two lines hold.
for _c in ROUTER_HINTS:
    check_true("routing/%s accepted by Router.predict" % _c, _c in _router_params)
    check_true("routing/%s rejected by Agent.predict" % _c, _c not in _agent_params)

for cls in (LayaSingleSelector, LayaMultiSelector, LayaQueryRouter):
    for _c in CONTROLS + ROUTER_HINTS:
        check_true("controls/%s takes %s" % (cls.__name__, _c), _c in _params(cls.__init__))
check("controls/_execute_decision takes every control",
      set(_params(llamaindex_module._execute_decision)) - {"state", "questions", "agent",
                                                           "base_url", "api_key", "model"},
      set(CONTROLS) | set(ROUTER_HINTS))

# The three wrappers end at the same runner call, so they must accept the same controls.
for _mod in (langchain_module, crewai_module):
    check("controls/%s agrees with llamaindex" % _mod.__name__.rsplit(".", 1)[-1],
          set(_params(_mod._execute_decision)),
          set(_params(llamaindex_module._execute_decision)))


class RecordingAgent:
    """A runner that keeps the kwargs of every call, the way a pipeline would have to be debugged."""

    device = "cpu"

    def __init__(self):
        self.calls = []

    def predict(self, state, questions, **kwargs):
        self.calls.append(kwargs)
        return {
            "answers": {
                "selector": {"choice": "choice_0", "confidence": 0.9, "answer_confidence": 0.9,
                             "probabilities": {"choice_0": 0.9, "choice_1": 0.4}},
                "route": {"choice": "sql", "confidence": 0.9, "answer_confidence": 0.9},
            },
            "routing": {"model": "english", "repo": None, "reason": "explicit model"},
        }


ALL_CONTROLS = {"max_len": 1024, "head_max_len": 512, "lang": "fr", "min_confidence": 0.4,
                "hooks": ["H"], "on_predict_start": "S",
                "on_predict_end": "E", "hooks_raise": True, "hooks_timeout": 0.5}

# One entry per class that can take a decision here: (label, class, constructor extras, one call).
SURFACES = [
    ("single", LayaSingleSelector, {},
     lambda s: s.select(tools, "Run a query on the SQL table")),
    ("multi", LayaMultiSelector, {},
     lambda s: s.select(tools, "Run a query on the SQL table")),
    ("query_router", LayaQueryRouter, {"query_engines": engines},
     lambda s: s.route("Run a query on the SQL table")),
]

HOOK_SAMPLES = (("hooks", [object()]), ("on_predict_start", object()),
                ("on_predict_end", object()), ("hooks_raise", False), ("hooks_timeout", 0.5))


def local_kwargs(cls, extra, run, **controls):
    """Run one decision on a fresh surface, or report the surface's own refusal.

    The broad except is deliberate: a wrapper that cannot even be built or that drops an attribute
    has to show up as a named mismatch, not as a traceback that hides every later check.
    """
    agent = RecordingAgent()
    try:
        run(cls(agent=agent, **dict(extra, **controls)))
    except Exception as exc:
        return "%s: %s" % (type(exc).__name__, exc)
    return agent.calls[0]


def remote_body(cls, extra, run, **controls):
    """POST through the real urllib path with the opener mocked, and return the JSON body sent."""
    with patch("urllib.request.build_opener") as mock_build_opener:
        mock_opener = MagicMock()
        mock_opener.open.return_value = DummyHTTPResponse(remote_response)
        mock_build_opener.return_value = mock_opener
        run(cls(base_url="http://localhost:8000", **dict(extra, **controls)))
        return json.loads(mock_opener.open.call_args[0][0].data)


for _label, _cls, _extra, _run in SURFACES:
    check("controls/%s with nothing set sends nothing" % _label,
          local_kwargs(_cls, _extra, _run), {})
    check("controls/%s forwards every control" % _label,
          local_kwargs(_cls, _extra, _run, **ALL_CONTROLS), ALL_CONTROLS)
    check("controls/%s forwards one budget alone" % _label,
          local_kwargs(_cls, _extra, _run, head_max_len=256), {"head_max_len": 256})
    check("controls/%s forwards lang alone" % _label,
          local_kwargs(_cls, _extra, _run, lang="fr"), {"lang": "fr"})
    check("controls/%s forwards min_confidence alone" % _label,
          local_kwargs(_cls, _extra, _run, min_confidence=0.4), {"min_confidence": 0.4})
    # 0 and [] are decisions, not absences: truthiness tests here would drop them.
    check("controls/%s keeps falsy values" % _label,
          local_kwargs(_cls, _extra, _run, head_max_len=0, hooks=[], hooks_raise=False,
                       min_confidence=0.0),
          {"head_max_len": 0, "hooks": [], "hooks_raise": False, "min_confidence": 0.0})
    check("controls/%s alongside model" % _label,
          local_kwargs(_cls, _extra, _run, model="laya-multilingual", max_len=1024),
          {"model": "laya-multilingual", "max_len": 1024})
    check("controls/%s remote body carries both budgets" % _label,
          {k: v for k, v in remote_body(_cls, _extra, _run, max_len=1024,
                                        head_max_len=384).items()
           if k in _controls.PREDICT_CONTROLS},
          {"max_len": 1024, "head_max_len": 384})
    check("controls/%s remote body omits unset budgets" % _label,
          [k for k in remote_body(_cls, _extra, _run) if k in _controls.PREDICT_CONTROLS], [])
    check("controls/%s remote body keeps a zero" % _label,
          remote_body(_cls, _extra, _run, head_max_len=0).get("head_max_len"), 0)
    # `lang` / `min_confidence` are laya-serve `BODY_CONTROLS`, so the same override reaches the
    # remote node -- and an unset one stays out of the body rather than shadowing the deployment.
    check("controls/%s remote body carries the decision controls" % _label,
          {k: v for k, v in remote_body(_cls, _extra, _run, lang="es", min_confidence=0.3).items()
           if k in _controls.DECISION_CONTROLS},
          {"lang": "es", "min_confidence": 0.3})
    check("controls/%s remote body omits unset decision controls" % _label,
          [k for k in remote_body(_cls, _extra, _run) if k in _controls.DECISION_CONTROLS], [])
    check("controls/%s remote body keeps min_confidence=0.0" % _label,
          remote_body(_cls, _extra, _run, min_confidence=0.0).get("min_confidence"), 0.0)
    # A hook is a Python callable that runs inside `predict`; a serve node cannot receive one.
    for _c, _sample in HOOK_SAMPLES:
        try:
            remote_body(_cls, _extra, _run, **{_c: _sample})
            check_true("controls/%s remote refuses %s" % (_label, _c), False, "no error raised")
        except ValueError as exc:
            check_true("controls/%s remote refuses %s" % (_label, _c),
                       _c in str(exc) and "laya-serve" in str(exc))
        except Exception as exc:
            check_true("controls/%s remote refuses %s" % (_label, _c), False, type(exc).__name__)

# The caller's own hook objects have to arrive, not a copy or a re-wrapped stand-in. Read with
# `.get()` so a surface that drops them reports a named failure instead of a KeyError that hides
# the rest of the run.
_sentinel_hooks = [RecordingAgent()]
_ident_agent = RecordingAgent()
LayaSingleSelector(agent=_ident_agent, hooks=_sentinel_hooks).select(
    tools, "Run a query on the SQL table")
_seen_hooks = _ident_agent.calls[0].get("hooks")
check_true("controls/forwards the caller's objects",
           _seen_hooks is _sentinel_hooks and _seen_hooks[0] is _sentinel_hooks[0], repr(_seen_hooks))


# --------------------------------------------------------------- Confidence source
# Every gate in this module reads the same number: the calibrated `answer_confidence`
# core gates on, never the entropy `confidence` (see `laya.confidence`). The two are on
# different scales, so they must not be interchangeable here.


class DisagreeingSelectorAgent(RecordingAgent):
    """Answers whose two confidence fields disagree, on purpose."""

    def __init__(self, response_fn=None):
        super().__init__()
        self.response_fn = response_fn

    def predict(self, state, questions, **kwargs):
        self.calls.append(kwargs)
        if self.response_fn is not None:
            return self.response_fn(state, questions)
        return super().predict(state, questions, **kwargs)


def _disagree(choice_answer):
    def response(state, questions):
        return {"model": "m",
                "answers": {"selector": dict({"choice": "choice_0"}, **choice_answer),
                            "route": {"choice": "sql", "confidence": 0.9, "answer_confidence": 0.9}}}
    return DisagreeingSelectorAgent(response)


# Below the calibrated threshold but above the entropy one: gates on the calibrated number.
low_agent = _disagree({"confidence": 0.95, "answer_confidence": 0.4})
low_sel = LayaSingleSelector(agent=low_agent, confidence_threshold=0.80, fallback_index=1).select(
    tools, "anything")
check("gate/reads calibrated not entropy",
      low_sel.selections[0].index if hasattr(low_sel, "selections") else low_sel, 1)
try:
    LayaSingleSelector(agent=_disagree({"confidence": 0.95, "answer_confidence": 0.4}),
                       confidence_threshold=0.80,
                       raise_on_low_confidence=True).select(tools, "anything")
    check_true("gate/raises on the calibrated number", False, "no error raised")
except LayaLowConfidenceError as err:
    check("gate/error carries the calibrated number", err.confidence, 0.4)

# An answer with no usable confidence keeps the old behaviour: treated as fully confident.
def _silent_selector(state, questions):
    return {"model": "m",
            "answers": {"selector": {"choice": "choice_2"},
                        "route": {"choice": "sql", "confidence": 0.9, "answer_confidence": 0.9}}}


kept = LayaSingleSelector(agent=DisagreeingSelectorAgent(_silent_selector)).select(tools, "anything")
check("gate/missing confidence still passes",
      kept.selections[0].index if hasattr(kept, "selections") else kept, 2)
# ------------------------------------------------- 6b. The two Router-only routing hints
class RoutingRunner:
    """A Router-shaped runner: it declares the arguments core's `Router.predict` declares.

    Explicit parameters rather than `**kwargs`, because the point of this section is the *name* a
    control arrives under -- a `**kwargs` sink records anything, including a control that arrived
    misspelled or in the wrong slot.
    """

    device = "cpu"

    def __init__(self):
        self.seen = {}

    def predict(self, state, questions, model=None, task=None, lang=None, lang_guess=None,
                max_len=None, head_max_len=None):
        self.seen = {k: v for k, v in locals().items()
                     if v is not None and k not in ("self", "state", "questions")}
        return dict(ROUTING_ANSWER)


class NonRoutingRunner(RoutingRunner):
    """An Agent-shaped runner: the budgets yes, either routing hint never."""

    def predict(self, state, questions, max_len=None, head_max_len=None):
        return RoutingRunner.predict(self, state, questions, max_len=max_len,
                                     head_max_len=head_max_len)


# The canned answers every selector here reads back out of one decision.
ROUTING_ANSWER = RecordingAgent().predict(None, None)

# A mock that drifts from core would make this section agree with itself and with nothing real, so
# each declared name has to be one the entry point it imitates accepts.
check_true("routing/RoutingRunner imitates accepted arguments only",
           (_params(RoutingRunner.predict) - {"state", "questions"}) <= _router_params,
           repr(_params(RoutingRunner.predict)))
check_true("routing/NonRoutingRunner imitates accepted arguments only",
           (_params(NonRoutingRunner.predict) - {"state", "questions"}) <= _agent_params,
           repr(_params(NonRoutingRunner.predict)))


def routing_kwargs(cls, extra, run, runner_cls, **hints):
    """One decision through `cls`, and whatever the runner says it was handed."""
    runner = runner_cls()
    run(cls(agent=runner, **dict(extra, **hints)))
    return runner.seen


for _label, _cls, _extra, _run in SURFACES:
    check("routing/%s default sends nothing" % _label,
          routing_kwargs(_cls, _extra, _run, RoutingRunner), {})
    check("routing/%s forwards both hints" % _label,
          routing_kwargs(_cls, _extra, _run, RoutingRunner, task="typed_decisions",
                         lang_guess="de"),
          {"task": "typed_decisions", "lang_guess": "de"})
    check("routing/%s forwards one hint alone" % _label,
          routing_kwargs(_cls, _extra, _run, RoutingRunner, lang_guess="de"), {"lang_guess": "de"})
    # A blank hint is core's documented "no usable hint, fall through to detection", so it is a
    # value the caller chose and must survive the trip.
    check("routing/%s keeps an empty lang_guess" % _label,
          routing_kwargs(_cls, _extra, _run, RoutingRunner, lang_guess=""), {"lang_guess": ""})
    check("routing/%s beside model and budget" % _label,
          routing_kwargs(_cls, _extra, _run, RoutingRunner, model="laya-multilingual",
                         max_len=1024, task="typed_decisions"),
          {"model": "laya-multilingual", "max_len": 1024, "task": "typed_decisions"})

    # A runner with no routing step must not be handed a routing hint: core would raise a
    # `TypeError` from inside `predict`, past every hook, on the way to an answer nobody asked for.
    for _c, _sample in zip(ROUTER_HINTS, ("typed_decisions", "de")):
        try:
            routing_kwargs(_cls, _extra, _run, NonRoutingRunner, **{_c: _sample})
            check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c),
                       False, "no error raised")
        except ValueError as exc:
            check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c), True)
            check_true("routing/%s %s names itself" % (_label, _c), _c in str(exc))
            check_true("routing/%s %s names the call that refused" % (_label, _c),
                       "NonRoutingRunner.predict" in str(exc))
            check_true("routing/%s %s says what to do" % (_label, _c), "Router" in str(exc))
        except Exception as exc:
            check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c),
                       False, type(exc).__name__)

    # Both hints ride in the laya-serve body -- `BODY_CONTROLS` there -- and a blank one stays.
    check("routing/%s remote body carries both hints" % _label,
          {k: v for k, v in remote_body(_cls, _extra, _run, task="typed_decisions",
                                        lang_guess="de").items() if k in ROUTER_HINTS},
          {"task": "typed_decisions", "lang_guess": "de"})
    check("routing/%s remote body omits unset hints" % _label,
          [k for k in remote_body(_cls, _extra, _run) if k in ROUTER_HINTS], [])
    check("routing/%s remote body keeps an empty lang_guess" % _label,
          remote_body(_cls, _extra, _run, lang_guess="").get("lang_guess"), "")
    # A `lang_guess` may be a callable taking the state, and only the code string has a wire form;
    # serve answers 422 for a callable, so naming it here beats an HTTP status after the request.
    try:
        remote_body(_cls, _extra, _run, lang_guess=lambda state: "de")
        check_true("routing/%s remote refuses a callable lang_guess" % _label, False,
                   "no error raised")
    except ValueError as exc:
        check_true("routing/%s remote refuses a callable lang_guess" % _label,
                   "lang_guess" in str(exc) and "laya-serve" in str(exc))
    except Exception as exc:
        check_true("routing/%s remote refuses a callable lang_guess" % _label, False,
                   type(exc).__name__)


# --------------------------------------------------------------- Results Summary
print(f"PASS: {len(PASS)}")

# ------------------------------------------------------ 9c. Malformed NESTED decision fields
#
# One level below the non-object refusal above. The selectors read
# `res.get("answers", {}).get(...)`, use `choice` with `.startswith`, and the multi-selector sorts
# and thresholds `probabilities`, so a string probability raised TypeError from `sorted` and a
# `true` ranked first as 1.0 -- a malformed body could win a selection instead of being refused.
with patch("urllib.request.build_opener") as _m:
    _m.return_value.open.return_value = DummyHTTPResponse({"answers": []})
    for _what, _call in (
        ("single", lambda: LayaSingleSelector(base_url="http://localhost:8000").select(
            tools, "Query via remote server")),
        ("multi", lambda: LayaMultiSelector(base_url="http://localhost:8000",
                                            probability_threshold=0.3).select(
            tools, "Compare both vector docs")),
        ("router", lambda: LayaQueryRouter(
            query_engines=engines,
            descriptions={"vector": "Semantic search", "sql": "Relational SQL"},
            base_url="http://localhost:8000").route("Run a query")),
    ):
        try:
            _call()
            check_true("nested/answers-list/%s raises" % _what, False, "no error raised")
        except RuntimeError as exc:
            check_true("nested/answers-list/%s malformed" % _what,
                       "Malformed decision response" in str(exc))
        except Exception as exc:
            check_true("nested/answers-list/%s is RuntimeError" % _what, False, type(exc).__name__)

_LLAMA_NESTED = [
    ({"answers": {"selector": "choice_0"}}, "single", "answer-str"),
    ({"answers": {"selector": {"choice": ["choice_0"]}}}, "single", "choice-list"),
    ({"answers": {"selector": {"choice": "choice_0", "probabilities": {"choice_0": "90%"}}}},
     "multi", "prob-str"),
    ({"answers": {"selector": {"choice": "choice_0", "probabilities": {"choice_0": True}}}},
     "multi", "prob-bool"),
    ({"answers": {"route": {"choice": [1]}}}, "router", "route-choice-list"),
]
for _body, _surface, _label in _LLAMA_NESTED:
    _surfaces = {
        "single": lambda: LayaSingleSelector(base_url="http://localhost:8000").select(
            tools, "Query via remote server"),
        "multi": lambda: LayaMultiSelector(base_url="http://localhost:8000",
                                           probability_threshold=0.3).select(
            tools, "Compare both vector docs"),
        "router": lambda: LayaQueryRouter(
            query_engines=engines,
            descriptions={"vector": "Semantic search", "sql": "Relational SQL"},
            base_url="http://localhost:8000").route("Run a query"),
    }
    with patch("urllib.request.build_opener") as _m:
        _m.return_value.open.return_value = DummyHTTPResponse(_body)
        try:
            _surfaces[_surface]()
            check_true("nested/%s raises" % _label, False, "no error raised")
        except RuntimeError as exc:
            check_true("nested/%s malformed" % _label, "Malformed decision response" in str(exc))
        except Exception as exc:
            check_true("nested/%s is RuntimeError" % _label, False, type(exc).__name__)

# A `choice` may name an index the caller's list does not have. `chosen_key` is remote JSON and
# `choices` is the caller's, and `choice_N` was parsed straight into `choices[N]`: `choice_9` raised
# a bare IndexError out of the selection and `choice_-1` wrapped to the LAST tool -- a selection no
# caller asked for and nothing in the decision names. `LayaQueryRouter.route` in this module already
# treats a name outside its engines as no match, so the single selector now checks the range too.
_LLAMA_CHOICE_INDEX = ["choice_4", "choice_9", "choice_-1", "choice_-99"]
for _choice in _LLAMA_CHOICE_INDEX:
    with patch("urllib.request.build_opener") as _m:
        _m.return_value.open.return_value = DummyHTTPResponse(
            {"answers": {"selector": {"choice": _choice, "answer_confidence": 0.95}}})
        try:
            _r = LayaSingleSelector(base_url="http://localhost:8000").select(
                tools, "Query via remote server")
            check_true("choice-index/%s/select raises" % _choice, False,
                       "no error raised (selected index %r)" % (_r.selections[0].index,))
        except RuntimeError as exc:
            check_true("choice-index/%s/select malformed" % _choice,
                       "Malformed decision response" in str(exc))
            check_true("choice-index/%s/select names the choice" % _choice, _choice in str(exc))
            check_true("choice-index/%s/select names the count" % _choice, "4 choices" in str(exc))
        except Exception as exc:
            check_true("choice-index/%s/select is RuntimeError" % _choice, False, type(exc).__name__)

# The last real index still selects: the guard is a range check, not a rejection of the highest one.
with patch("urllib.request.build_opener") as _m:
    _m.return_value.open.return_value = DummyHTTPResponse(
        {"answers": {"selector": {"choice": "choice_3", "answer_confidence": 0.95}}})
    check("choice-index/last choice still selects",
          LayaSingleSelector(base_url="http://localhost:8000").select(
              tools, "Query via remote server").selections[0].index, 3)

print(f"FAIL: {len(FAIL)}")
for f in FAIL:
    print(f"  - {f}")

if FAIL:
    sys.exit(1)