Self-hosted AI workspace.
Python
test(web): repair negative security witnesses
Alexandre Teixeiraa52c65715079
diff --git a/tests/test_runtime_behavior_regressions.py b/tests/test_runtime_behavior_regressions.pyindex 1b4d28b2d..ed860dc23 100644--- a/tests/test_runtime_behavior_regressions.py+++ b/tests/test_runtime_behavior_regressions.py@@ -29,7 +29,8 @@ def _collect(gen):def _delta_chunk(text):- payload = {"choices": [{"delta": {"content": text}}]}+ # stream_llm_with_fallback exposes normalized SSE, not provider wire JSON.+ payload = {"delta": text}return f"data: {json.dumps(payload)}\n\n"@@ -57,7 +58,7 @@ def _run_turn(monkeypatch, messages, **kwargs):yield "data: [DONE]\n\n"monkeypatch.setattr(al, "stream_llm_with_fallback", _fake_stream, raising=False)- _collect(+ chunks = _collect(al.stream_agent_loop("http://local.test/v1","moonshotai/kimi-k3",@@ -68,7 +69,7 @@ def _run_turn(monkeypatch, messages, **kwargs):**kwargs,))- return offered+ return offered, chunks@@ -100,60 +101,71 @@ def _contract(offered=("ask_user", "update_plan", "manage_notes"),# failure this guards is a model that obeys the wording while the runtime# contradicted it by offering the tool anyway.#-# SCOPE, and it matters: these cover the inferred path, where the turn has no-# explicit web toggle and the runtime decides from intent. Measured on-# lab@c499c01b, detection there is partial: "don't search online" suppresses-# the intent and the web tools are withheld; "Do not search the web" and "No-# web search please" do not, and the tools are offered.-#-# When the user explicitly enables web for the turn, wording does not withhold-# anything: confirmed end to end against a local Qwen3.5-9B Q4_K_M, where all-# three phrasings were offered web_search, web_fetch and private_browser. That-# may well be correct, an explicit toggle beating an inferred negative, so it-# is recorded here rather than asserted either way.-#-# The two inferred-path cases that do not hold are xfail(strict=True): they-# document the target, run on every suite, and fail the moment the behaviour-# lands. Delete the marker then.+# These cover inferred intent with no explicit web toggle or supplied contract.+# On the frozen Wave 3 base all three negatives are recognized. Two are still+# misclassified as web-dependent turns and short-circuit to "web disabled",+# preventing the requested answer from memory. Only that precise failure is+# expected below; unrelated exceptions must fail normally.HELD = ["Answer from memory only, don't search online."]-NOT_HELD_YET = [+BLOCKED_MEMORY_ONLY = ["Summarise what you already know. Do not search the web.","No web search please, just tell me what you know about Python decorators.",][email protected]("phrasing", HELD)-def test_negative_web_wording_withholds_the_web_tools(monkeypatch, phrasing):- offered = _run_turn(monkeypatch, [{"role": "user", "content": phrasing}])+class _MemoryOnlyTurnBlocked(AssertionError):+ """The real loop blocked a memory-only answer as requiring web access."""++def _assert_negative_web_turn(monkeypatch, phrasing):+ offered, chunks = _run_turn(monkeypatch, [{"role": "user", "content": phrasing}])+ events = [json.loads(chunk[6:]) for chunk in chunks+ if chunk.startswith("data: ") and chunk.strip() != "data: [DONE]"]+ finals = [event for event in events if event.get("type") == "final_response"]+ if not offered and finals == [{+ "type": "final_response",+ "content": "Web access is disabled for this turn. Enable web search and resend the request.",+ }]:+ raise _MemoryOnlyTurnBlocked(+ f"Memory-only turn was blocked before any model call: {phrasing!r}; "+ f"actual response: {finals[0]['content']}"+ )++ assert len(offered) == 1, f"Expected one memory-only model call; events: {events!r}"names = _schema_names(offered[0])assert "web_search" not in names, f"web_search offered despite: {phrasing!r}"assert "web_fetch" not in names, f"web_fetch offered despite: {phrasing!r}"+ assert any(event.get("delta") == "ok" for event in events), events+ assert chunks[-1] == "data: [DONE]\n\n"++[email protected]("phrasing", HELD)+def test_negative_web_wording_withholds_the_web_tools(monkeypatch, phrasing):+ _assert_negative_web_turn(monkeypatch, phrasing)@pytest.mark.xfail(strict=True,- reason="negative web wording is only partially detected on lab@c499c01b; "- "these phrasings still get the web tools offered",+ raises=_MemoryOnlyTurnBlocked,+ reason="negative web wording is recognized but the memory-only turn is "+ "misclassified as web-dependent and blocked before the model call; "+ "production intent/short-circuit correction is outside test isolation",)[email protected]("phrasing", NOT_HELD_YET)[email protected]("phrasing", BLOCKED_MEMORY_ONLY)def test_negative_web_wording_withholds_the_web_tools_unhandled(monkeypatch, phrasing):- offered = _run_turn(monkeypatch, [{"role": "user", "content": phrasing}])-- names = _schema_names(offered[0])- assert "web_search" not in names, f"web_search offered despite: {phrasing!r}"- assert "web_fetch" not in names, f"web_fetch offered despite: {phrasing!r}"+ _assert_negative_web_turn(monkeypatch, phrasing)def test_plain_web_request_still_offers_search(monkeypatch):"""The guard above must not become a blanket removal of the web tools."""- offered = _run_turn(+ offered, chunks = _run_turn(monkeypatch,[{"role": "user", "content": "Search the web for the latest Python release."}],)+ assert len(offered) == 1, chunksassert "web_search" in _schema_names(offered[0])