Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(model-router): retain backend identity per signed probe attempt (#8174)

Mike Bradley4c96f320c475
diff --git a/ods/docs/MODEL-SWITCHBOARD.md b/ods/docs/MODEL-SWITCHBOARD.md
index ecc5868f8..c526e8f09 100644
--- a/ods/docs/MODEL-SWITCHBOARD.md
+++ b/ods/docs/MODEL-SWITCHBOARD.md
@@ -251,6 +251,10 @@ For an owned latency investigation, the same internal bearer can opt one signed
The existing internal GET then includes `upstreamAttempts`, even while an attempt is pending or after an upstream timeout. Every actual upstream send has its own sequence, router request ID, attempt number, route identity, monotonic start, elapsed milliseconds, HTTP status and transport outcome. Tool-protocol repair is a separate attempt. A complete transport does not establish task success, tool execution or backend inference quiescence. Streaming elapsed time includes downstream consumption; it is not a model-only generation measurement. Hashing preparation time is recorded separately.
+Each attempt also has an additive `responseModel` field: the backend model string observed by the existing response parser, bounded to 256 characters, or `null` when unproven. It is never filled from the selected route or rewritten public alias. Pending, failed HTTP/transport, malformed or model-less responses remain unknown; streams also require a terminal event, consistent model identity and no observed error. This identity belongs to that request and attempt even when a later request with the same marker (for example, an automatic title) replaces the outer latest-response record. Consumers must treat an absent field from an older router as unknown as well. This does not classify auxiliary requests or prove that no user turn was replayed; use the request IDs and source-bound message fingerprints alongside application history. Capture limits, expiry and access controls are unchanged.
+
+The diagnostic failure check is shared by JSON, ordinary SSE and buffered tool SSE. A non-null `error`, a `failed`/`incomplete`/`cancelled` status, or an `error`/`response.failed`/`response.incomplete`/`response.cancelled` type in the parsed payload or its nested `response` keeps identity unknown, even if terminal framing follows. `error: null` alone is not a failure. Buffered assembly retains only this boolean verdict before discarding unrelated fields; existing forwarding, rejection and repair behavior is unchanged. A `complete` attempt status describes transport completion and can therefore coexist with an unknown identity after an explicit application failure.
+
Each attempt fingerprints the exact forwarded body bytes and ordered message/tool components, with lengths and per-message role, content and tool-call fingerprints. Components use canonical JSON with sorted object keys and preserved array order; the body uses its exact wire serialization. Digests are HMAC-SHA256 under a domain-separated per-probe key, allowing comparison within that probe without persisting bodies or permitting cross-probe plaintext lookup. Components over the 2 MiB body or 256 message/tool bounds report unavailable; no payload text, tool names, secrets or exception text are retained. These byte counts and hashes are not token spans, nor evidence that a backend reused its cache. Capture failures never change or retry an inference request.
The `offeredTools` boundary is `router-forwarded-tools-not-execution-proof`.
diff --git a/ods/extensions/services/model-router/app/completed_tool_sse.py b/ods/extensions/services/model-router/app/completed_tool_sse.py
index b211f6271..ba3970024 100644
--- a/ods/extensions/services/model-router/app/completed_tool_sse.py
+++ b/ods/extensions/services/model-router/app/completed_tool_sse.py
@@ -10,6 +10,22 @@ from typing import Any
_MAX_BYTES = 16 * 1024 * 1024
+def response_reports_failure(payload: dict[str, Any]) -> bool:
+ """Shared diagnostic verdict for parsed Chat/Responses JSON and SSE.
+
+ A null error is normal on successful Responses objects. An explicit failure
+ wins over a completion marker; this verdict does not change forwarding.
+ """
+ for value in (payload, payload.get("response")):
+ if isinstance(value, dict) and (
+ value.get("error") is not None
+ or value.get("status") in ("failed", "incomplete", "cancelled")
+ or value.get("type") in ("error", "response.failed", "response.incomplete", "response.cancelled")
+ ):
+ return True
+ return False
+
+
# Only source-authored reasons may reach logs or the response. Never derive a
# diagnostic from str(exc), backend fields, prompts, tool names or arguments.
_REJECTION_CODES = {
@@ -169,6 +185,7 @@ def assemble_chat_completion_sse(
expected_model: str,
*,
max_bytes: int = _MAX_BYTES,
+ response_metadata: dict[str, bool] | None = None,
) -> dict:
if not isinstance(raw, (bytes, bytearray)):
raise _err("raw must be bytes")
@@ -207,6 +224,10 @@ def assemble_chat_completion_sse(
raise _err("malformed json") from e
if not isinstance(obj, dict):
raise _err("event not object")
+ if response_metadata is not None:
+ # Reuse this parse before assembly discards non-Chat fields. Only a
+ # constant-size verdict escapes; neither payload nor output changes.
+ response_metadata["failed"] = response_metadata.get("failed", False) or response_reports_failure(obj)
if "error" in obj and obj["error"] is not None:
raise _err("top-level error")
saw_any = True
diff --git a/ods/extensions/services/model-router/app/main.py b/ods/extensions/services/model-router/app/main.py
index b7fbce562..30ce5aecd 100644
--- a/ods/extensions/services/model-router/app/main.py
+++ b/ods/extensions/services/model-router/app/main.py
@@ -44,7 +44,7 @@ from jsonschema.exceptions import SchemaError, ValidationError
from fastapi import FastAPI, Request, Response
from fastapi.responses import JSONResponse, StreamingResponse
from .completed_tool_sse import (
- CompletionStreamIdentityError, assemble_chat_completion_sse,
+ CompletionStreamIdentityError, assemble_chat_completion_sse, response_reports_failure,
completion_stream_rejection_code,
)
from .probe_attempts import ProbeAttempts
@@ -368,6 +368,13 @@ def _finish_probe_attempt(handle, status, http_status=None):
pass
+def _observe_probe_response_model(handle, model):
+ try:
+ _probe_attempts.observe_response_model(handle, model)
+ except Exception:
+ pass # Optional diagnostics must never alter inference.
+
+
class _TelemetrySink:
"""Best-effort telemetry transport that never blocks model responses."""
@@ -1267,6 +1274,7 @@ class _SSERewriter:
self.identity_matches = True
self.response_model: str | None = None
self.response_model_consistent = True
+ self.response_error = False
self.usage = {
"input_tokens": 0,
"output_tokens": 0,
@@ -1281,6 +1289,7 @@ class _SSERewriter:
) -> None:
self.completed = self.completed or saw_done
for payload in payloads:
+ self.response_error = self.response_error or response_reports_failure(payload)
usage, stop_reason = _usage_from_response(payload)
for key, value in usage.items():
self.usage[key] = max(self.usage[key], value)
@@ -1791,6 +1800,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
}
attempt_handle = None
+ response_metadata = {"failed": False}
next_attempt = 2
try:
forwarded_body = json.dumps(payload).encode("utf-8")
@@ -1833,6 +1843,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
complete = assemble_chat_completion_sse(
raw_stream, route["runtimeModelId"],
max_bytes=MAX_COMPLETED_TOOL_STREAM_BYTES,
+ response_metadata=response_metadata,
)
except CompletionStreamIdentityError as exc:
rejection = completion_stream_rejection_code(exc)
@@ -1918,6 +1929,7 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
attempt_handle = _begin_probe_attempt(
probe_id, request_id, next_attempt, fallback_body, route,
)
+ response_metadata = {"failed": False}
next_attempt += 1
ods_headers["X-ODS-Tool-Stream-Fallback"] = "true"
remaining = UPSTREAM_TIMEOUT_SECONDS - (
@@ -1984,6 +1996,8 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
and 200 <= upstream.status_code < 300
and rewriter.response_model_consistent
):
+ if not rewriter.response_error:
+ _observe_probe_response_model(attempt_handle, rewriter.response_model)
_record_evidence({
**evidence_base,
"status": upstream.status_code,
@@ -2055,6 +2069,8 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
except (ValueError, UnicodeDecodeError):
initial_decision = None
if isinstance(initial_decision, dict):
+ if not response_metadata["failed"] and not response_reports_failure(initial_decision):
+ _observe_probe_response_model(attempt_handle, initial_decision.get("model"))
native_names = _complete_native_envelope_names(initial_decision)
native_invalid = native_names is not None and not _normalize_native_tool_markup(
initial_decision, payload,
@@ -2090,6 +2106,10 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
headers=headers, timeout=remaining,
)
_finish_probe_attempt(repair_handle, "complete", upstream.status_code)
+ # The common response parser must attribute the repaired
+ # response to this send, not to the original tool decision.
+ attempt_handle = repair_handle
+ response_metadata = {"failed": False}
except asyncio.CancelledError:
_finish_probe_attempt(repair_handle, "cancelled")
raise
@@ -2111,6 +2131,8 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
repaired = json.loads(upstream.content.decode("utf-8"))
except (ValueError, UnicodeDecodeError):
repaired = None
+ if isinstance(repaired, dict) and not response_reports_failure(repaired):
+ _observe_probe_response_model(repair_handle, repaired.get("model"))
if (structured_invalid and isinstance(repaired, dict)
and repaired.get("model") != route["runtimeModelId"]):
return JSONResponse({"error": {
@@ -2140,6 +2162,8 @@ async def _forward_inner(request: Request, path: str, payload: dict[str, Any],
parsed = decoded
response_usage, stop_reason = _usage_from_response(parsed)
response_model = parsed.get("model")
+ if not response_metadata["failed"] and not response_reports_failure(parsed):
+ _observe_probe_response_model(attempt_handle, response_model)
if "model" in parsed:
parsed["model"] = requested_alias
_sanitize_choice_content(parsed)
diff --git a/ods/extensions/services/model-router/app/probe_attempts.py b/ods/extensions/services/model-router/app/probe_attempts.py
index b81e67134..06ca9ce46 100644
--- a/ods/extensions/services/model-router/app/probe_attempts.py
+++ b/ods/extensions/services/model-router/app/probe_attempts.py
@@ -63,7 +63,7 @@ class ProbeAttempts:
row = {"requestId": request_id, "attempt": attempt,
"sequence": len(scope["records"]) + 1,
"startedMonotonic": self.clock(), "elapsedMs": None,
- "status": "pending", "httpStatus": None,
+ "status": "pending", "httpStatus": None, "responseModel": None,
"boundary": "actual-router-upstream-send-not-tool-execution",
"encoding": "hmac-sha256;components=json-sort-keys-ascii-v1",
"routeSeq": route["routeSeq"], "routedModel": route["runtimeModelId"],
@@ -126,3 +126,19 @@ class ProbeAttempts:
return {"schemaVersion": 1, "maxAttempts": scope["limit"],
"limitReached": len(scope["records"]) >= scope["limit"],
"attempts": json.loads(json.dumps(scope["records"]))}
+
+ def observe_response_model(self, handle, model):
+ """Attach a parsed backend identity, never a selected route or alias.
+
+ Completion and parsing are separate in the forwarding paths. A null
+ identity remains unknown, including while parsing has not yet finished.
+ The caller must supply only a complete, consistent response's identity.
+ """
+ if handle is None or not isinstance(model, str) or not 0 < len(model) <= 256:
+ return
+ probe_id, scope, row = handle
+ if self._scope(probe_id) is not scope or row not in scope["records"]:
+ return
+ if (row["status"] == "complete" and row["httpStatus"] is not None
+ and 200 <= row["httpStatus"] < 300 and row["responseModel"] is None):
+ row["responseModel"] = model
diff --git a/ods/extensions/services/model-router/tests/test_probe_attempts.py b/ods/extensions/services/model-router/tests/test_probe_attempts.py
index 9ad1ddd7e..f289f51e0 100644
--- a/ods/extensions/services/model-router/tests/test_probe_attempts.py
+++ b/ods/extensions/services/model-router/tests/test_probe_attempts.py
@@ -37,7 +37,7 @@ def payload(probe, stream=False, tools=False):
return result
[email protected]("kind", ["stream", "buffered-tool", "repair"])
[email protected]("kind", ["stream", "buffered-tool", "buffered-json", "thinking", "repair"])
def test_each_upstream_send_matches_wire_and_preserves_attempts(router, kind):
mod, client, write_state, _ = router
write_state()
@@ -67,19 +67,26 @@ def test_each_upstream_send_matches_wire_and_preserves_attempts(router, kind):
for frame in frames) + b"data: [DONE]\n\n"
return httpx.Response(200, content=sse,
headers={"content-type": "text/event-stream"})
- if body["stream"]:
+ if kind == "buffered-json":
+ assert body["stream"] is True
+ elif body["stream"]:
assert kind == "repair"
return httpx.Response(500, json={"error": {
"message": "Invalid diff: now finding less tool calls!"}})
- assert body["stream"] is False
+ else:
+ assert body["stream"] is False
text = ("<tool_call>\n<function=missing_tool>\n<parameter=key>\nx\n</parameter>\n</function>\n</tool_call>"
if kind == "repair" and len(sent) == 2 else "ok")
- return httpx.Response(200, json={"model": "Concrete.gguf", "choices": [
+ model = "Decision-other.gguf" if kind == "repair" and len(sent) == 2 else "Concrete.gguf"
+ return httpx.Response(200, json={"model": model, "choices": [
{"message": {"role": "assistant", "content": text}, "finish_reason": "stop"}]})
asyncio.run(mod.app.state.http.aclose())
mod.app.state.http = httpx.AsyncClient(transport=httpx.MockTransport(upstream))
- response = client.post("/v1/chat/completions", json=payload(probe, True, kind != "stream"))
+ body = payload(probe, True, kind != "stream")
+ if kind == "thinking":
+ body["chat_template_kwargs"] = {"enable_thinking": True}
+ response = client.post("/v1/chat/completions", json=body)
assert response.status_code == 200
rows = observed(client, probe)["attempts"]
assert len(rows) == len(sent) == (3 if kind == "repair" else 1)
@@ -91,6 +98,9 @@ def test_each_upstream_send_matches_wire_and_preserves_attempts(router, kind):
expected_status = "stream-error" if kind == "repair" and row["attempt"] == 1 else "complete"
expected_http = 500 if kind == "repair" and row["attempt"] == 1 else 200
assert row["status"] == expected_status and row["httpStatus"] == expected_http
+ expected_model = (None if expected_http == 500 else
+ "Decision-other.gguf" if kind == "repair" and row["attempt"] == 2 else "Concrete.gguf")
+ assert row["responseModel"] == expected_model
assert row["elapsedMs"] >= 0 and row["capturePrepareMs"] >= 0
actual = json.loads(raw)
for name in ["messages", "tools"]:
@@ -148,6 +158,7 @@ def test_expiry_limit_and_old_completion_cannot_revive_evidence(router):
assert mod._probe_attempts.public(probe, "probe-secret") is None
assert arm(client, probe).status_code == 201
mod._probe_attempts.finish(stale_handle, "complete", 200)
+ mod._probe_attempts.observe_response_model(stale_handle, "Late-other.gguf")
assert observed(client, probe)["attempts"] == []
diff --git a/ods/extensions/services/model-router/tests/test_probe_response_failure_forms.py b/ods/extensions/services/model-router/tests/test_probe_response_failure_forms.py
new file mode 100644
index 000000000..e0ed3f3cf
--- /dev/null
+++ b/ods/extensions/services/model-router/tests/test_probe_response_failure_forms.py
@@ -0,0 +1,97 @@
+"""Explicit Responses failures never become per-attempt success identities."""
+import asyncio
+import json
+import uuid
+
+import httpx
+import pytest
+
+from test_probe_attempts import arm, observed, payload
+from test_router import router as router, _signed_marker
+
+
+def set_upstream(mod, upstream):
+ asyncio.run(mod.app.state.http.aclose())
+ mod.app.state.http = httpx.AsyncClient(transport=httpx.MockTransport(lambda _: upstream))
+
+
[email protected]("status", ["failed", "incomplete", "cancelled"])
+def test_responses_json_failure_status_with_model_stays_unknown(router, status):
+ mod, client, write_state, _ = router
+ write_state()
+ probe = str(uuid.uuid4())
+ arm(client, probe)
+ set_upstream(mod, httpx.Response(200, json={"model": "Concrete.gguf", "status": status}))
+ response = client.post("/v1/responses", json={"model": "ods/current", "input": _signed_marker(probe)})
+ assert response.status_code == 200 and response.json()["status"] == status
+ row, = observed(client, probe)["attempts"]
+ assert row["status"] == "complete" and row["httpStatus"] == 200
+ assert row.get("responseModel") is None
+
+
[email protected]("status", ["failed", "incomplete", "cancelled"])
[email protected]("done", [False, True], ids=["without-done", "with-done"])
+def test_responses_sse_type_only_failure_with_model_stays_unknown(router, status, done):
+ mod, client, write_state, _ = router
+ write_state()
+ probe = str(uuid.uuid4())
+ arm(client, probe)
+ # Compatibility backends may append a Chat-style [DONE] to Responses events.
+ # An explicit failed event must win over that terminal framing marker.
+ event = {"type": "response." + status, "response": {"model": "Concrete.gguf"}}
+ raw = b"data: " + json.dumps(event).encode() + b"\n\n"
+ if done:
+ raw += b"data: [DONE]\n\n"
+ set_upstream(mod, httpx.Response(200, content=raw, headers={"content-type": "text/event-stream"}))
+ response = client.post("/v1/responses", json={"model": "ods/current", "stream": True,
+ "input": _signed_marker(probe)})
+ assert response.status_code == 200 and "response." + status in response.text
+ row, = observed(client, probe)["attempts"]
+ assert row["status"] == "complete"
+ assert row.get("responseModel") is None
+
+
[email protected]("stream", [False, True])
+def test_completed_responses_null_error_is_not_a_failure(router, stream):
+ mod, client, write_state, _ = router
+ write_state()
+ probe = str(uuid.uuid4())
+ arm(client, probe)
+ completed = {"model": "Concrete.gguf", "status": "completed", "error": None}
+ if stream:
+ upstream = httpx.Response(200, content=b"data: " + json.dumps({
+ "type": "response.completed", "response": completed,
+ "error": None}).encode() + b"\n\n", headers={"content-type": "text/event-stream"})
+ else:
+ upstream = httpx.Response(200, json=completed)
+ set_upstream(mod, upstream)
+ response = client.post("/v1/responses", json={"model": "ods/current", "stream": stream,
+ "input": _signed_marker(probe)})
+ assert response.status_code == 200
+ row, = observed(client, probe)["attempts"]
+ assert row.get("responseModel") == "Concrete.gguf"
+
+
[email protected]("failure", [{"error": {"message": "synthetic failure"}},
+ {"type": "response.failed", "response": {"model": "Concrete.gguf"}},
+ {"type": "response.incomplete", "response": {"model": "Concrete.gguf"}}],
+ ids=["chat-error", "responses-failed", "responses-incomplete"])
+def test_buffered_chat_tool_stream_keeps_unproven_identity_unknown(router, failure):
+ mod, client, write_state, _ = router
+ write_state()
+ probe = str(uuid.uuid4())
+ arm(client, probe)
+ first = {"model": "Concrete.gguf", "choices": [{"index": 0,
+ "delta": {"role": "assistant", "content": "synthetic"}, "finish_reason": "stop"}]}
+ raw = b"".join(b"data: " + json.dumps(value).encode() + b"\n\n" for value in [first, failure])
+ set_upstream(mod, httpx.Response(200, content=raw + b"data: [DONE]\n\n",
+ headers={"content-type": "text/event-stream"}))
+ response = client.post("/v1/chat/completions", json=payload(probe, stream=True, tools=True))
+ # Existing forwarding rejects Chat errors, but ignores unfamiliar Responses
+ # lifecycle frames. This patch only changes diagnostic identity attribution.
+ assert response.status_code == (502 if "error" in failure else 200)
+ if "error" in failure:
+ assert response.json()["error"]["type"] == "upstream_invalid_response"
+ row, = observed(client, probe)["attempts"]
+ assert row["status"] == ("stream-error" if "error" in failure else "complete")
+ assert row.get("responseModel") is None
diff --git a/ods/extensions/services/model-router/tests/test_probe_response_models.py b/ods/extensions/services/model-router/tests/test_probe_response_models.py
new file mode 100644
index 000000000..220fb614d
--- /dev/null
+++ b/ods/extensions/services/model-router/tests/test_probe_response_models.py
@@ -0,0 +1,153 @@
+"""Per-send identities survive signed owner/title overlap; no model or sockets."""
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+import json
+import threading
+import uuid
+
+import httpx
+import pytest
+
+from test_probe_attempts import AUTH, arm, expected_digest, observed, payload
+from test_router import router as router, _ChunkedStream
+
+
+def reply(model, stream=False):
+ body = {"model": model, "choices": [{"index": 0,
+ "message": {"role": "assistant", "content": "synthetic response"},
+ "finish_reason": "stop"}]}
+ if not stream:
+ return httpx.Response(200, json=body)
+ body["choices"][0]["delta"] = body["choices"][0].pop("message")
+ return httpx.Response(200, content=b"data: " + json.dumps(body).encode()
+ + b"\n\ndata: [DONE]\n\n", headers={"content-type": "text/event-stream"})
+
+
[email protected]("concurrent", [False, True], ids=["sequential", "overlap"])
[email protected]("owner_stream,title_stream", [(False, False), (True, False),
+ (False, True), (True, True)])
+def test_signed_owner_and_title_keep_distinct_actual_identities(
+ router, concurrent, owner_stream, title_stream):
+ mod, client, write_state, _ = router
+ write_state()
+ probe = str(uuid.uuid4())
+ assert arm(client, probe).status_code == 201
+ owner = payload(probe, owner_stream)
+ title = payload(probe, title_stream)
+ title["messages"][0]["content"] = "Synthetic title instruction"
+ title["response_format"] = {"type": "json_schema", "json_schema": {
+ "name": "session_title", "strict": True, "schema": {"type": "object"}}}
+ started, release = threading.Event(), threading.Event()
+ sent = {}
+
+ async def upstream(request):
+ body = json.loads(request.content)
+ kind = "title" if "response_format" in body else "owner"
+ sent[kind] = request.content
+ if concurrent and kind == "owner":
+ started.set()
+ assert await asyncio.to_thread(release.wait, 5)
+ # Both routes selected Concrete.gguf; title actually answered elsewhere.
+ return reply("Title-other.gguf" if kind == "title" else "Concrete.gguf",
+ body["stream"])
+
+ asyncio.run(mod.app.state.http.aclose())
+ mod.app.state.http = httpx.AsyncClient(transport=httpx.MockTransport(upstream))
+ if concurrent:
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ pending = pool.submit(client.post, "/v1/chat/completions", json=owner)
+ try:
+ assert started.wait(5)
+ responses = {"title": client.post("/v1/chat/completions", json=title)}
+ rows = observed(client, probe)["attempts"]
+ assert rows[0]["status"] == "pending"
+ assert rows[0]["responseModel"] is None
+ assert rows[1]["responseModel"] == "Title-other.gguf"
+ finally:
+ release.set()
+ responses["owner"] = pending.result(timeout=5)
+ else:
+ responses = {kind: client.post("/v1/chat/completions", json=body)
+ for kind, body in [("owner", owner), ("title", title)]}
+ rows = observed(client, probe)["attempts"]
+ assert len(rows) == len(sent) == 2
+ for kind, response in responses.items():
+ assert response.status_code == 200
+ row, = [r for r in rows if r["requestId"] == response.headers["X-ODS-Request-Id"]]
+ assert row["responseModel"] == ("Title-other.gguf" if kind == "title" else "Concrete.gguf")
+ assert row["routedModel"] == "Concrete.gguf" and row["attempt"] == 1
+ assert row["status"] == "complete" and row["httpStatus"] == 200
+ assert row["fingerprints"]["bodyDigest"] == expected_digest(probe, sent[kind])
+ latest = client.get(f"/internal/route-evidence/{probe}", headers=AUTH).json()
+ assert latest["responseModel"] == ("Concrete.gguf" if concurrent else "Title-other.gguf")
+ assert "Synthetic title instruction" not in json.dumps(rows)
+ assert "private user" not in json.dumps(rows)
+
+
[email protected]("kind", ["missing", "non-string", "oversized", "http-error", "json-error",
+ "malformed", "stream-missing", "stream-conflict", "stream-incomplete", "stream-error",
+ "stream-http-error", "stream-inband-error", "stream-response-failed", "timeout"])
+def test_unproven_response_identity_is_not_filled_from_selected_route(router, kind):
+ mod, client, write_state, _ = router
+ write_state()
+ probe = str(uuid.uuid4())
+ arm(client, probe)
+
+ def upstream(request):
+ if kind == "timeout":
+ raise httpx.ReadTimeout("synthetic failure", request=request)
+ if kind.startswith("stream-"):
+ first = b'data: {"model":"Concrete.gguf","choices":[{"delta":{"content":"x"}}]}\n\n'
+ if kind == "stream-missing":
+ first = b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n'
+ chunks = [first]
+ if kind == "stream-conflict":
+ chunks.append(b'data: {"model":"Other.gguf","choices":[]}\n\n')
+ if kind == "stream-inband-error":
+ chunks.append(b'data: {"error":{"message":"synthetic"}}\n\n')
+ if kind == "stream-response-failed":
+ chunks.append(b'data: {"type":"response.failed","response":{"status":"failed"}}\n\n')
+ if kind not in {"stream-incomplete", "stream-error"}:
+ chunks.append(b'data: [DONE]\n\n')
+ return httpx.Response(500 if kind == "stream-http-error" else 200,
+ stream=_ChunkedStream(chunks, httpx.ReadError("synthetic failure")
+ if kind == "stream-error" else None),
+ headers={"content-type": "text/event-stream"})
+ if kind == "malformed":
+ return httpx.Response(200, content=b"invalid json")
+ body = {"choices": []}
+ if kind == "non-string":
+ body["model"] = {"private": "body"}
+ elif kind == "oversized":
+ body["model"] = "x" * 257
+ elif kind in {"http-error", "json-error"}:
+ body = {"model": "Concrete.gguf", "error": {"message": "synthetic"}}
+ return httpx.Response(500 if kind == "http-error" else 200, json=body)
+
+ asyncio.run(mod.app.state.http.aclose())
+ mod.app.state.http = httpx.AsyncClient(transport=httpx.MockTransport(upstream))
+ if kind == "stream-error":
+ with pytest.raises(httpx.ReadError):
+ client.post("/v1/chat/completions", json=payload(probe, True))
+ else:
+ client.post("/v1/chat/completions", json=payload(probe, kind.startswith("stream-")))
+ row, = observed(client, probe)["attempts"]
+ assert row["responseModel"] is None
+ assert row["routedModel"] == "Concrete.gguf"
+ assert row["status"] != "pending"
+
+
+def test_response_identity_survives_only_its_own_capture_lease(router):
+ mod, client, write_state, _ = router
+ write_state()
+ probe = str(uuid.uuid4())
+ now = [1.0]
+ mod._probe_attempts.clock = lambda: now[0]
+ arm(client, probe, ttlSeconds=1)
+ client.post("/v1/chat/completions", json=payload(probe))
+ row, = observed(client, probe)["attempts"]
+ assert row["responseModel"] == "Concrete.gguf"
+ row["responseModel"] = "forged"
+ assert observed(client, probe)["attempts"][0]["responseModel"] == "Concrete.gguf"
+ now[0] = 2.0
+ assert client.get(f"/internal/route-evidence/{probe}", headers=AUTH).json().get("upstreamAttempts") is None
diff --git a/ods/extensions/services/model-router/tests/test_responses_stream_identity.py b/ods/extensions/services/model-router/tests/test_responses_stream_identity.py
index bdb678232..b1f0db92d 100644
--- a/ods/extensions/services/model-router/tests/test_responses_stream_identity.py
+++ b/ods/extensions/services/model-router/tests/test_responses_stream_identity.py
@@ -6,6 +6,7 @@ import uuid
import pytest
from test_router import router as router, _set_stream_upstream, _signed_marker
+from test_probe_attempts import arm
def event(model, kind="response.completed"):
@@ -24,6 +25,7 @@ def test_rewrites_nested_response_identity_and_records_concrete_model(router, al
mod, client, write_state, _ = router
write_state()
probe = str(uuid.uuid4())
+ assert arm(client, probe).status_code == 201
raw = event("Concrete.gguf", "response.created") + event("Concrete.gguf")
_set_stream_upstream(mod, [raw[:63], raw[63:117], raw[117:]])
response = client.post("/v1/responses", json={
@@ -38,6 +40,9 @@ def test_rewrites_nested_response_identity_and_records_concrete_model(router, al
})
assert evidence.status_code == 200
assert evidence.json()["responseModel"] == "Concrete.gguf"
+ row, = evidence.json()["upstreamAttempts"]["attempts"]
+ assert row["responseModel"] == "Concrete.gguf"
+ assert row["requestId"] == response.headers["X-ODS-Request-Id"]
assert mod._inflight == 0