Skip to content

Self-hosted AI workspace.

Python

test(runtime): pin negative capability wording, test-only

Léo5fd911488231
diff --git a/tests/test_runtime_behavior_regressions.py b/tests/test_runtime_behavior_regressions.py
new file mode 100644
index 000000000..389f8dfdf
--- /dev/null
+++ b/tests/test_runtime_behavior_regressions.py
@@ -0,0 +1,121 @@
+"""Deterministic regression coverage for runtime behaviours, test-only.
+
+These drive ``stream_agent_loop`` with a fake model so the assertions are about
+what the runtime offers and does, not about what a model happens to answer.
+Nothing here imports benchmark fixtures or allowlists, and nothing here touches
+production runtime code: the lane exists so the implementation side can move
+without losing the behaviours underneath it.
+
+The fake-model pattern is the one already used by tests/test_tool_policy.py:
+patch ``stream_llm_with_fallback`` and inspect the ``tools`` kwarg the loop
+hands it, which is the runtime's decision about what the turn may do.
+"""
+
+import asyncio
+import json
+
+import pytest
+
+import src.agent_loop as al
+
+
+def _collect(gen):
+ async def _run():
+ return [c async for c in gen]
+
+ return asyncio.run(_run())
+
+
+def _delta_chunk(text):
+ payload = {"choices": [{"delta": {"content": text}}]}
+ return f"data: {json.dumps(payload)}\n\n"
+
+
+def _schema_names(tools):
+ return {
+ tool.get("function", {}).get("name") or tool.get("name")
+ for tool in (tools or [])
+ }
+
+
+def _patch_loop_basics(monkeypatch):
+ monkeypatch.setattr(al, "get_setting", lambda key, default=None: default, raising=False)
+ monkeypatch.setattr(al, "get_mcp_manager", lambda: None, raising=False)
+ monkeypatch.setattr(al, "estimate_tokens", lambda *a, **k: 10, raising=False)
+
+
+def _run_turn(monkeypatch, messages, **kwargs):
+ """Drive one agent turn and return the tool sets offered to the model."""
+ _patch_loop_basics(monkeypatch)
+ offered = []
+
+ async def _fake_stream(_candidates, _messages, **kw):
+ offered.append(kw.get("tools"))
+ yield _delta_chunk("ok")
+ yield "data: [DONE]\n\n"
+
+ monkeypatch.setattr(al, "stream_llm_with_fallback", _fake_stream, raising=False)
+ _collect(
+ al.stream_agent_loop(
+ "http://local.test/v1",
+ "moonshotai/kimi-k3",
+ messages,
+ max_rounds=kwargs.pop("max_rounds", 1),
+ relevant_tools=kwargs.pop("relevant_tools", {"ask_user", "update_plan"}),
+ owner=kwargs.pop("owner", "sft_alex_creator"),
+ **kwargs,
+ )
+ )
+ return offered
+
+
+# ── negative capability wording ─────────────────────────────────────────────
+# A turn that says not to search must not be handed the search tools. The
+# failure this guards is a model that obeys the wording while the runtime
+# contradicted it by offering the tool anyway.
+#
+# Measured on lab@c499c01b: the detection is partial. "don't search online" is
+# caught; "Do not search the web" and "No web search please" are not, and the
+# web tools are offered for both. The two that do not hold yet are marked
+# xfail(strict=True), so they document the target, run on every suite, and fail
+# loudly the moment the behaviour lands. Delete the marker then.
+
+HELD = ["Answer from memory only, don't search online."]
+
+NOT_HELD_YET = [
+ "Summarise what you already know. Do not search the web.",
+ "No web search please, just tell me what you know about Python decorators.",
+]
+
+
[email protected]("phrasing", HELD)
+def test_negative_web_wording_withholds_the_web_tools(monkeypatch, phrasing):
+ offered = _run_turn(monkeypatch, [{"role": "user", "content": phrasing}])
+
+ names = _schema_names(offered[0])
+ assert "web_search" not in names, f"web_search offered despite: {phrasing!r}"
+ assert "web_fetch" not in names, f"web_fetch offered despite: {phrasing!r}"
+
+
+ strict=True,
+ reason="negative web wording is only partially detected on lab@c499c01b; "
+ "these phrasings still get the web tools offered",
+)
[email protected]("phrasing", NOT_HELD_YET)
+def test_negative_web_wording_withholds_the_web_tools_unhandled(monkeypatch, phrasing):
+ offered = _run_turn(monkeypatch, [{"role": "user", "content": phrasing}])
+
+ names = _schema_names(offered[0])
+ assert "web_search" not in names, f"web_search offered despite: {phrasing!r}"
+ assert "web_fetch" not in names, f"web_fetch offered despite: {phrasing!r}"
+
+
+def test_plain_web_request_still_offers_search(monkeypatch):
+ """The guard above must not become a blanket removal of the web tools."""
+ offered = _run_turn(
+ monkeypatch,
+ [{"role": "user", "content": "Search the web for the latest Python release."}],
+ )
+
+ assert "web_search" in _schema_names(offered[0])