Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #523 from Bruce-Yii/feat/mcp-shortlist-lang-embed-cache

Nandakishorb1412db0c254
diff --cc laya/mcp/tools.py
index 36e57ae0,afb31620..f14190d0
--- a/laya/mcp/tools.py
+++ b/laya/mcp/tools.py
@@@ -219,17 -196,29 +219,40 @@@ def _decision_to_dict(decision: Any) -
}
+def laya_route(state: Any, questions: Any, *, router: Any = None) -> dict:
+ """Routing decision only: no forward pass."""
+ state_d = validate_state(state)
+ questions_d = validate_questions(questions)
+ if router is None:
+ raise ToolError("models_not_ready", "Router is not loaded")
+ if not hasattr(router, "route"):
+ raise ToolError("internal_error", "router has no route() method")
+ return _decision_to_dict(router.route(state_d, questions_d))
+
+
+ # Where the reusable shortlist embedder lives. Kept private and on the agent, so the
+ # cache's lifetime is the answering checkpoint's rather than this process's.
+ _EMBED_CACHE_ATTR = "_laya_mcp_shortlist_embed_fn"
+
+
+ def _detected_language(detection: Any) -> str | None:
+ """The language a route decision detected, or None when it reported none.
+
+ Accepts either shape ``Router.route`` returns -- a ``RouteDecision`` (a dict)
+ or a plain object -- and tolerates a decision that carries no detection at all,
+ an explicit-model decision being the normal case.
+ """
+ if detection is None:
+ return None
+ if isinstance(detection, dict):
+ lang = detection.get("language")
+ else:
+ lang = getattr(detection, "language", None)
+ if isinstance(lang, str) and lang:
+ return lang
+ return None
+
+
def _resident_or_load(router: Any, name: str) -> Any:
"""The resident agent for checkpoint ``name``, loading on demand when possible.
@@@ -286,8 -307,23 +341,11 @@@ def laya_shortlist
raise ToolError("models_not_ready", "Router is not loaded (auto mode)")
if not hasattr(router, "route"):
raise ToolError("internal_error", "router has no route() method")
- routing = _decision_to_dict(router.route(state_d, questions_d))
+ decision = router.route(state_d, questions_d)
- if isinstance(decision, dict):
- routed = decision.get("model")
- detection = decision.get("detection")
- routing = {
- "model": routed,
- "repo": decision.get("repo"),
- "reason": decision.get("reason"),
- }
- else:
- routed = getattr(decision, "model", None)
- detection = getattr(decision, "detection", None)
- routing = {
- "model": routed,
- "repo": getattr(decision, "repo", None),
- "reason": getattr(decision, "reason", None),
- }
++ routing = _decision_to_dict(decision)
+ routed = routing["model"]
++ detection = (decision.get("detection") if isinstance(decision, dict)
++ else getattr(decision, "detection", None))
if not isinstance(routed, str) or not routed:
raise ToolError("internal_error", "router.route returned no model")
predict_target = router
diff --cc tests/test_mcp.py
index de0743a2,5804d751..c6f4d568
--- a/tests/test_mcp.py
+++ b/tests/test_mcp.py
@@@ -564,272 -505,156 +564,422 @@@ def test_shortlist()
and out["routing"]["reason"] == "explicit model", repr(out["routing"]))
+# --- batch tools (mocked router, no weights) ----------------------------------
+
+class BatchRouter(FakeRouter):
+ """Records each batch call; returns one result per request, input order."""
+
+ def __init__(self):
+ self.predict_batch_calls = []
+ self.route_batch_calls = []
+
+ def _answer_for(self, request):
+ answers = {}
+ for name, spec in request["questions"].items():
+ if spec["type"] == "choice":
+ answers[name] = {"choice": "billing", "confidence": 0.9}
+ elif spec["type"] == "score":
+ answers[name] = {"score": 1.5, "confidence": 0.8}
+ else:
+ answers[name] = {"noul": 0.7, "confidence": 0.9}
+ return {"answers": answers,
+ "routing": {"model": request.get("model") or "english",
+ "repo": "fake/laya", "reason": "batch route"}}
+
+ def predict_batch(self, requests, batch_size=None):
+ self.predict_batch_calls.append((list(requests), batch_size))
+ return [self._answer_for(request) for request in requests]
+
+ def route_batch(self, requests):
+ self.route_batch_calls.append(list(requests))
+ return [{"model": request.get("model") or "english", "repo": "fake/laya",
+ "reason": "batch route"} for request in requests]
+
+
+class ShortRouter:
+ """Router whose batches misreport their own size."""
+
+ def predict_batch(self, requests, batch_size=None):
+ return []
+
+ def route_batch(self, requests):
+ return []
+
+
+BATCH_REQUESTS = [
+ {"state": {"body": "refund please"}, "questions": QUESTIONS},
+ {"state": {"body": "please refund invoice 2"}, "questions": QUESTIONS,
+ "model": "english", "task": "massive", "lang": "en"},
+ {"state": {"body": "mera account double charge hua"},
+ "questions": {"triage": {"type": "noul", "instructions": "Leaving?"}},
+ "lang": "hi"},
+]
+
+
+def test_batch_validation():
+ for bad in (None, {}, "x", {"state": STATE, "questions": QUESTIONS}):
+ expect_tool_error("batch/requests_bad_%r" % (type(bad).__name__,),
+ lambda b=bad: validate_batch_requests(b), "invalid_request")
+ expect_tool_error("batch/requests_empty",
+ lambda: validate_batch_requests([]), "invalid_request")
+ expect_tool_error("batch/item_not_object",
+ lambda: validate_batch_requests(["x"]), "invalid_request")
+ expect_tool_error("batch/item_missing_state",
+ lambda: validate_batch_requests([{"questions": QUESTIONS}]), "invalid_state")
+ expect_tool_error("batch/item_missing_questions",
+ lambda: validate_batch_requests([{"state": STATE}]), "invalid_questions")
+ expect_tool_error("batch/item_bad_questions",
+ lambda: validate_batch_requests([{"state": STATE, "questions": {}}]),
+ "invalid_questions")
+ expect_tool_error("batch/item_bad_model",
+ lambda: validate_batch_requests(
+ [{"state": STATE, "questions": QUESTIONS, "model": "gpt4"}]),
+ "invalid_model")
+ for key, value in (("task", 3), ("lang", ""), ("lang", [])):
+ expect_tool_error("batch/item_bad_%s" % key,
+ lambda k=key, v=value: validate_batch_requests(
+ [{"state": STATE, "questions": QUESTIONS, k: v}]),
+ "invalid_request")
+
+ out = validate_batch_requests(BATCH_REQUESTS)
+ ok("batch/validation_preserves_order", [item["state"] for item in out]
+ == [request["state"] for request in BATCH_REQUESTS])
+ ok("batch/validation_keeps_overrides",
+ out[1].get("model") == "english" and out[1].get("task") == "massive"
+ and out[1].get("lang") == "en" and out[2].get("lang") == "hi", repr(out[1]))
+ # "auto" is the same thing as absent: Router.route resolves the checkpoint.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "model": "auto"}])
+ ok("batch/validation_auto_dropped", "model" not in out[0], repr(out[0]))
+ # Keys the Router would never expect are dropped, not forwarded.
+ out = validate_batch_requests([{"state": STATE, "questions": QUESTIONS, "temperature": 0}])
+ ok("batch/validation_unknown_key_dropped", set(out[0]) == {"state", "questions"}, repr(out[0]))
+
+
+def test_batch_predict():
+ expect_tool_error("batch/predict_no_router",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=None), "models_not_ready")
+ expect_tool_error("batch/predict_no_method",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=FakeRouter()),
+ "internal_error")
+ for bad_size in (True, 0, -2, 2.5, "3"):
+ expect_tool_error("batch/predict_bad_size_%r" % (bad_size,),
+ lambda s=bad_size: laya_predict_batch(
+ BATCH_REQUESTS, batch_size=s, router=BatchRouter()),
+ "invalid_batch_size")
+ # Validation runs before the router is touched: one bad item, no partial batch.
+ router = BatchRouter()
+ expect_tool_error("batch/predict_validates_first",
+ lambda: laya_predict_batch([{"state": STATE}, {"state": STATE, "questions": QUESTIONS}],
+ router=router),
+ "invalid_questions")
+ ok("batch/predict_not_called_on_bad_input", router.predict_batch_calls == [])
+
+ router = BatchRouter()
+ out = laya_predict_batch(BATCH_REQUESTS, batch_size=8, router=router)
+ ok("batch/predict_one_call", len(router.predict_batch_calls) == 1,
+ repr(len(router.predict_batch_calls)))
+ forwarded, size = router.predict_batch_calls[0]
+ ok("batch/predict_size_forwarded", size == 8, repr(size))
+ ok("batch/predict_items_forwarded", [item["state"] for item in forwarded]
+ == [request["state"] for request in BATCH_REQUESTS])
+
+ ok("batch/predict_keys", set(out) == {"requests", "model_counts",
+ "total_latency_ms", "per_request_latency_ms"}, repr(sorted(out)))
+ ok("batch/predict_input_order", [entry["answers"].get("triage", {}).get("noul")
+ for entry in out["requests"]] == [None, None, 0.7],
+ repr(out["requests"]))
+ ok("batch/predict_department", out["requests"][0]["answers"]["department"]["choice"] == "billing")
+ ok("batch/predict_routing", out["requests"][1]["routing"]["model"] == "english")
+ ok("batch/predict_device_resident", out["requests"][0].get("device") == "cpu",
+ repr(out["requests"][0].get("device")))
+ ok("batch/predict_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))
+ ok("batch/predict_latency", out["total_latency_ms"] >= 0
+ and abs(out["per_request_latency_ms"] - out["total_latency_ms"] / 3) < 0.01,
+ repr(out["per_request_latency_ms"]))
+
+ # batch_size unset must not be forwarded as None (strict old stubs included).
+ router = BatchRouter()
+ laya_predict_batch(BATCH_REQUESTS, router=router)
+ ok("batch/predict_size_default", router.predict_batch_calls[0][1] is None)
+
+ expect_tool_error("batch/predict_count_mismatch",
+ lambda: laya_predict_batch(BATCH_REQUESTS, router=ShortRouter()),
+ "internal_error")
+
+
+def test_batch_route():
+ expect_tool_error("batch/route_no_router",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=None), "models_not_ready")
+ expect_tool_error("batch/route_no_method",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=FakeRouter()),
+ "internal_error")
+ router = BatchRouter()
+ out = laya_route_batch(BATCH_REQUESTS, router=router)
+ ok("batch/route_one_call", len(router.route_batch_calls) == 1)
+ ok("batch/route_decisions", len(out["decisions"]) == 3
+ and set(out["decisions"][0]) == {"model", "repo", "reason"}, repr(out["decisions"][0]))
+ ok("batch/route_counts", out["model_counts"] == {"english": 3}, repr(out["model_counts"]))
+ # Route-only never predicts.
+ ok("batch/route_no_forward", router.predict_batch_calls == [])
+ expect_tool_error("batch/route_count_mismatch",
+ lambda: laya_route_batch(BATCH_REQUESTS, router=ShortRouter()),
+ "internal_error")
+
+
+# --- decide tool (mocked router, no weights) ----------------------------------
+
+DECIDE_SCHEMA = {
+ "type": "object",
+ "properties": {
+ "department": {"enum": ["billing", "support", "sales"],
+ "description": "Which team should handle this ticket?"},
+ "urgency": {"type": "integer", "minimum": 0, "maximum": 2},
+ "needs_human": {"type": "boolean"},
+ },
+}
+
+
+class TypeEchoRouter(FakeRouter):
+ """FakeRouter that also answers with the real payload shape: the `type`
+ key and a probabilities dict keyed by option/level, like system_one does,
+ so laya.structured's per-field probabilities are exercised."""
+
+ def predict(self, state, questions, **kwargs):
+ out = super().predict(state, questions, **kwargs)
+ for name, spec in questions.items():
+ answer = out["answers"][name]
+ answer["type"] = spec["type"]
+ if spec["type"] == "score":
+ answer["probabilities"] = {"0": 0.1, "1": 0.2, "2": 0.7}
+ return out
+
+
+def test_decide():
+ import laya
+
+ router = TypeEchoRouter()
+ out = laya_decide(STATE, DECIDE_SCHEMA, router=router)
+ ok("decide/keys", set(out) >= {"values", "confidence", "probabilities", "routing", "latency_ms"},
+ repr(sorted(out)))
+ ok("decide/values_choice", out["values"]["department"] == "billing", repr(out["values"]))
+ ok("decide/values_score_level", out["values"]["urgency"] == 2, repr(out["values"]))
+ ok("decide/values_bool", out["values"]["needs_human"] is True, repr(out["values"]))
+ ok("decide/confidence", set(out["confidence"]) == {"department", "urgency", "needs_human"}
+ and out["confidence"]["department"] == 0.94, repr(out["confidence"]))
+ ok("decide/probs_noul_pair", set(out["probabilities"]["needs_human"]) == {"false", "true"}
+ and out["probabilities"]["needs_human"]["true"] == 0.892,
+ repr(out["probabilities"]["needs_human"]))
+ ok("decide/probs_score_offset", set(out["probabilities"]["urgency"]) >= {"0", "1", "2"},
+ repr(out["probabilities"]["urgency"]))
+ ok("decide/score_level_from_probs", out["values"]["urgency"] == 2, repr(out["values"]))
+ ok("decide/device", out.get("device") == "cpu", repr(out.get("device")))
+ ok("decide/routing", out["routing"]["model"] == "english")
+
+ # Same values the core decide() returns for the same runner: the tool is a
+ # thin MCP surface over the documented schema projection, not a second one.
+ core = laya.decide(router, STATE, schema=DECIDE_SCHEMA)
+ ok("decide/parity_with_core", out["values"] == core, "tool=%r core=%r" % (out["values"], core))
+
+ # Score projection follows the argmax of the distribution, offset by minimum.
+ class LowUrgencyRouter(TypeEchoRouter):
+ def predict(self, state, questions, **kwargs):
+ out = super().predict(state, questions, **kwargs)
+ out["answers"]["urgency"] = {"type": "score", "score": 2.0, "confidence": 0.6,
+ "probabilities": {"0": 0.6, "1": 0.3, "2": 0.1}}
+ return out
+
+ out = laya_decide(STATE, DECIDE_SCHEMA, router=LowUrgencyRouter())
+ ok("decide/score_argmax", out["values"]["urgency"] == 0, repr(out["values"]["urgency"]))
+
+ expect_tool_error("decide/bad_state", lambda: laya_decide({}, DECIDE_SCHEMA, router=router),
+ "invalid_state")
+ for bad, why in (({"type": "object"}, "no properties"),
+ ({"type": "object", "properties": {"x": {"type": "string"}}}, "free string"),
+ ({"type": "object", "properties": {"x": {"type": "array"}}}, "array"),
+ ({}, "empty"), ([], "not an object")):
+ expect_tool_error("decide/bad_schema_%s" % why,
+ lambda b=bad: laya_decide(STATE, b, router=router), "invalid_schema")
+ expect_tool_error("decide/bad_model",
+ lambda: laya_decide(STATE, DECIDE_SCHEMA, model="gpt4", router=router),
+ "invalid_model")
+ expect_tool_error("decide/no_router",
+ lambda: laya_decide(STATE, DECIDE_SCHEMA), "models_not_ready")
+ # Explicit model with a direct agent: answered by the agent, no Router needed.
+ # A bare Agent payload carries no routing, which is when the tool synthesises
+ # the "explicit model" routing entry (laya_predict does the same).
+ class BareAgent:
+ device = "cpu"
+
+ def predict(self, state, questions, **kwargs):
+ answers = {}
+ for name, spec in questions.items():
+ if spec["type"] == "choice":
+ answers[name] = {"type": "choice", "choice": "billing", "confidence": 0.9,
+ "probabilities": {"billing": 0.9, "support": 0.1}}
+ elif spec["type"] == "score":
+ answers[name] = {"type": "score", "score": 1.0, "confidence": 0.6,
+ "probabilities": {"0": 0.2, "1": 0.6, "2": 0.2}}
+ else:
+ answers[name] = {"type": "noul", "noul": 0.2, "confidence": 0.8}
+ return {"answers": answers}
+
+ out = laya_decide(STATE, DECIDE_SCHEMA, model="english", agent=BareAgent())
+ ok("decide/direct_agent", out["values"]["department"] == "billing"
+ and out["routing"] == {"model": "english", "repo": None, "reason": "explicit model"},
+ repr(out["routing"]))
+ ok("decide/direct_agent_device", out.get("device") == "cpu", repr(out.get("device")))
+
+
+ def test_shortlist_lang_parity():
+ """Auto mode must forward the routed language, so per-language temperatures
+ behave as they do in laya_predict. Passing an explicit model= makes
+ Router.predict re-route with that model, and an explicit-model decision
+ carries no detection -- so the language is lost unless shortlist passes it on.
+ """
+ class DetectingRouter(ShortlistRouter):
+ def route(self, state, questions):
+ return {
+ "model": "multilingual",
+ "repo": "fake/repo",
+ "reason": "Latin script but language looks like 'de'",
+ "detection": {"language": "de", "script": "Latin"},
+ }
+
+ router = DetectingRouter({"multilingual": ShortlistAgent()}, routed="multilingual")
+ out = laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=router, embed_fn=_tie_embed)
+ ok("shortlist_lang/auto_forwards_lang", (router.seen_kwargs or {}).get("lang") == "de",
+ repr(router.seen_kwargs))
+ ok("shortlist_lang/auto_still_explicit_model",
+ (router.seen_kwargs or {}).get("model") == "multilingual", repr(router.seen_kwargs))
+ ok("shortlist_lang/auto_routes_once",
+ out["routing"]["model"] == "multilingual", repr(out["routing"]))
+
+ # A decision without a usable language must not invent one.
+ plain = ShortlistRouter({"english": ShortlistAgent()}, routed="english")
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, router=plain, embed_fn=_tie_embed)
+ ok("shortlist_lang/no_detection_adds_nothing", plain.seen_kwargs == {"model": "english"},
+ repr(plain.seen_kwargs))
+
+ # Explicit-model mode is unchanged: no routing, so no language to forward.
+ agent = ShortlistDirectAgent()
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="multilingual", agent=agent,
+ embed_fn=_tie_embed)
+ ok("shortlist_lang/explicit_model_untouched", agent.seen is not None)
+
+
+ def test_shortlist_embed_cache():
+ """A fixed option list must not be re-embedded on every call.
+
+ Uses the answering checkpoint's own encoder (no injected embed_fn) through a
+ weight-free fake tokenizer/encoder, so the number of tokenizer batches is a
+ direct count of embedding work.
+ """
+ import torch
+
+ class _Out:
+ def __init__(self, hidden):
+ self.last_hidden_state = hidden
+
+ class CountingEncoder:
+ def __init__(self):
+ self.calls = 0
+
+ def __call__(self, input_ids=None, attention_mask=None):
+ self.calls += 1
+ ids = input_ids.float()
+ return _Out(torch.stack([ids, torch.ones_like(ids)], dim=-1))
+
+ class CacheTok:
+ def __init__(self):
+ self.batches = []
+
+ def __call__(self, texts, padding=True, truncation=True, max_length=256, return_tensors="pt"):
+ self.batches.append(list(texts))
+ rows = [[(ord(ch) % 5) + 1 for ch in t][:max_length] or [1] for t in texts]
+ width = max(len(r) for r in rows)
+ ids, mask = [], []
+ for row in rows:
+ pad = width - len(row)
+ ids.append(row + [0] * pad)
+ mask.append([1] * len(row) + [0] * pad)
+ return {"input_ids": torch.tensor(ids, dtype=torch.long),
+ "attention_mask": torch.tensor(mask, dtype=torch.long)}
+
+ def make_agent():
+ enc = CountingEncoder()
+ tok = CacheTok()
+ agent = ShortlistAgent()
+ agent.tok = tok
+ agent.encoder = enc
+ agent.model = type("M", (), {"encoder": enc})()
+ agent.device = torch.device("cpu")
+ return agent
+
+ # Same checkpoint, same option list: the second call re-embeds only the query.
+ # Count embedded texts, not tokenizer batches: the whole set fits in one batch
+ # either way, so batch count cannot tell the two cases apart.
+ agent = make_agent()
+ router = ShortlistRouter({"english": agent})
+
+ def texts_since(mark):
+ return sum(len(b) for b in agent.tok.batches[mark:])
+
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=router)
+ first_texts = texts_since(0)
+ ok("shortlist_cache/first_call_embeds_query_and_options", first_texts == 6,
+ "texts=%d batches=%r" % (first_texts, agent.tok.batches))
+
+ # An identical repeat call embeds nothing at all: the query text is the same
+ # too, and cached_embed_fn matches on exact strings.
+ mark = len(agent.tok.batches)
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=router)
+ ok("shortlist_cache/identical_repeat_embeds_nothing", texts_since(mark) == 0,
+ "texts=%d batches=%r" % (texts_since(mark), agent.tok.batches[mark:]))
+
+ # A different state is a new query, so a repeat with that state embeds the
+ # query alone and reuses the five cached option rows.
+ mark = len(agent.tok.batches)
+ laya_shortlist({"text": "a completely different customer message"},
+ SHORTLIST_QUESTIONS, k=2, model="english", router=router)
+ ok("shortlist_cache/new_state_still_embedded", texts_since(mark) == 1,
+ "texts=%d batches=%r" % (texts_since(mark), agent.tok.batches[mark:]))
+
+ # A different answering checkpoint must not share another one's embeddings.
+ other = make_agent()
+ other_router = ShortlistRouter({"multilingual": other})
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="multilingual", router=other_router)
+ ok("shortlist_cache/other_checkpoint_embeds_its_own",
+ sum(len(b) for b in other.tok.batches) == 6,
+ "texts=%d" % sum(len(b) for b in other.tok.batches))
+ ok("shortlist_cache/other_checkpoint_encoder_used", other.encoder.calls == 1,
+ "calls=%d" % other.encoder.calls)
+
+ # Replacing the agent (a reload) starts a fresh cache rather than reusing rows.
+ reloaded = make_agent()
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english",
+ router=ShortlistRouter({"english": reloaded}))
+ ok("shortlist_cache/reloaded_agent_re_embeds",
+ sum(len(b) for b in reloaded.tok.batches) == 6,
+ "texts=%d" % sum(len(b) for b in reloaded.tok.batches))
+ ok("shortlist_cache/other_checkpoint_still_cached",
+ other.encoder.calls == 1, "calls=%d" % other.encoder.calls)
+
+ # An injected embed_fn stays the caller's own, uncached.
+ calls = []
+
+ def counting_embed(texts):
+ calls.append(list(texts))
+ return [[0.0, 0.0] for _ in texts]
+
+ inj_router = ShortlistRouter({"english": make_agent()})
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=inj_router,
+ embed_fn=counting_embed)
+ laya_shortlist(STATE, SHORTLIST_QUESTIONS, k=2, model="english", router=inj_router,
+ embed_fn=counting_embed)
+ ok("shortlist_cache/injected_embed_not_wrapped", len(calls) == 2,
+ "calls=%d" % len(calls))
+
+
def test_timeout_removed():
# The per-call timeout was removed: a ThreadPoolExecutor shutdown waits for
# the work anyway, and MCP clients apply their own request timeout. The tool
@@@ -894,10 -713,8 +1044,12 @@@ test_schema(
test_shape()
test_question_forwarding()
test_shortlist()
+test_batch_validation()
+test_batch_predict()
+test_batch_route()
+test_decide()
+ test_shortlist_lang_parity()
+ test_shortlist_embed_cache()
test_timeout_removed()
test_models_from_env()
test_server_registration()