Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): retain safe relay correlation IDs (#2925)
Call White2ffad8302434
diff --git a/core/handlers/model_hub/adapter.py b/core/handlers/model_hub/adapter.pyindex b786b6577..a4c3daace 100644--- a/core/handlers/model_hub/adapter.py+++ b/core/handlers/model_hub/adapter.py@@ -17,6 +17,7 @@ import asynciofrom dataclasses import dataclass, fieldfrom datetime import datetimefrom enum import Enum+import refrom typing import Any, AsyncIterator, Awaitable, Callable, Final, Literal, Mapping, Protocol, Sequencefrom .stream_wire import ProtocolSSEState, ProtocolUsageReport@@ -28,6 +29,55 @@ MODEL_PROTOCOLS = ("anthropic", "openai_responses", "openai_chat", "google")SOURCE_PROTOCOLS = MODEL_PROTOCOLS + ("search",)SEARCH_RESPONSE_MAX_BYTES: Final = 8 * 1024 * 1024SEARCH_TRANSPORT_TIMEOUT_SECONDS: Final = 30.0+# Response correlation is a diagnostic fact, not a header-forwarding surface. These+# are the only upstream response header names allowed to cross the engine boundary.+RESPONSE_CORRELATION_HEADER_NAMES: Final = frozenset({"request-id", "x-request-id"})+RESPONSE_CORRELATION_ID_MAX_LENGTH: Final = 256+RESPONSE_CORRELATION_ID_MAX_ITEMS: Final = len(RESPONSE_CORRELATION_HEADER_NAMES)+_CORRELATION_ID_CREDENTIAL_PATTERN = re.compile(+ r"(?i)(?:bearer|basic)\s+|"+ r"(?:authorization|api[_ -]?key|access[_ -]?token|token|secret|password|"+ r"credential|cookie|session|signature)\s*[:=]|"+ r"(?:sk|rk|pk|sess|token|secret|cred|key)[-_][a-z0-9_-]{8,}|"+ r"(?:AIza[a-z0-9_-]{20,}|github_pat_[a-z0-9_]{20,}|gh[pousr]_[a-z0-9_]{20,})|"+ r"ey[a-z0-9_-]+\.[a-z0-9_-]+\.[a-z0-9_-]+$"+)+++def sanitize_response_correlation_ids(values: object) -> tuple[tuple[str, str], ...]:+ """Keep only bounded, credential-free values from the two trusted headers."""++ pairs = values.items() if isinstance(values, Mapping) else values+ try:+ iterator = iter(pairs) # type: ignore[arg-type]+ except TypeError:+ return ()+ retained: list[tuple[str, str]] = []+ seen: set[str] = set()+ for pair in iterator:+ if not isinstance(pair, (tuple, list)) or len(pair) != 2:+ continue+ raw_name, raw_value = pair+ if not isinstance(raw_name, str) or not isinstance(raw_value, str):+ continue+ name = raw_name.lower()+ if name not in RESPONSE_CORRELATION_HEADER_NAMES or name in seen:+ continue+ if (+ not raw_value+ or len(raw_value) > RESPONSE_CORRELATION_ID_MAX_LENGTH+ or any(not 0x21 <= ord(char) <= 0x7E for char in raw_value)+ or _CORRELATION_ID_CREDENTIAL_PATTERN.search(raw_value)+ ):+ continue+ seen.add(name)+ retained.append((name, raw_value))+ if len(retained) >= RESPONSE_CORRELATION_ID_MAX_ITEMS:+ break+ order = {name: index for index, name in enumerate(sorted(RESPONSE_CORRELATION_HEADER_NAMES))}+ return tuple(sorted(retained, key=lambda item: order[item[0]]))++OBSERVATION_OUTCOMES = ("observed","ambiguous",@@ -228,6 +278,10 @@ class RawCallOutcome:# The search tools whose own completed item a served response carried, for# the positive evidence written at settlement (``vibey-hosted-tools.md`` §3.2).hosted_tools: frozenset[str] = frozenset()+ # The response header name is retained in lowercase so relay ``x-request-id``+ # and provider ``request-id`` values cannot be conflated. The client and+ # provenance writer re-validate this tuple before persistence.+ response_correlation_ids: tuple[tuple[str, str], ...] = ()class ObservationOutcome(str, Enum):diff --git a/core/handlers/model_hub/provenance.py b/core/handlers/model_hub/provenance.pyindex f39f87bcc..770fa9ed8 100644--- a/core/handlers/model_hub/provenance.py+++ b/core/handlers/model_hub/provenance.py@@ -27,7 +27,7 @@ from core.run_settlement import ()from vibe.i18n import t as i18n_t-from .adapter import RawCallOutcome+from .adapter import RawCallOutcome, sanitize_response_correlation_idsfrom .classification import (UPSTREAM_MACHINE_ERROR_CODES,ResolutionDecision,@@ -1032,6 +1032,20 @@ def _terminal_reason(decision: ResolutionDecision) -> str:return "protocol_error"+def _response_correlation_payload(values: object) -> dict[str, list[dict[str, str]]]:+ """Serialize only the adapter's bounded, allowlisted response IDs."""++ ids = sanitize_response_correlation_ids(values)+ if not ids:+ return {}+ return {+ "response_correlation_ids": [+ {"header": header, "value": value}+ for header, value in ids+ ]+ }++def _degrade_persisted_provenance(record: dict) -> dict:degraded = dict(record)attempts = degraded.get("failed_attempts")@@ -2164,6 +2178,7 @@ class TurnCorrelationRegistry:stream_started: bool,force: bool = False,local_error_detail: str | None = None,+ response_correlation_ids: tuple[tuple[str, str], ...] = (),) -> None:if turn_id is None:return@@ -2182,7 +2197,21 @@ class TurnCorrelationRegistry:):returnif reason == "engine_down":- # The engine, not one request, is what went down.+ # The engine, not one request, is what went down. Preserve+ # correlation facts when a later forced finalization repeats+ # this same terminalization without another response.+ previous_ids = ()+ if isinstance(trace.terminal_error, dict) and trace.terminal_error.get("reason") == reason:+ previous = trace.terminal_error.get("response_correlation_ids")+ if isinstance(previous, list):+ previous_ids = tuple(+ (item.get("header"), item.get("value"))+ for item in previous+ if isinstance(item, Mapping)+ )+ correlation_payload = _response_correlation_payload(response_correlation_ids)+ if not correlation_payload:+ correlation_payload = _response_correlation_payload(previous_ids)trace.pending_attempts.clear()trace.served = Nonetrace.model_supply_state = None@@ -2194,6 +2223,7 @@ class TurnCorrelationRegistry:"reason": reason,"stream_started": stream_started,**({"local_error_detail": local_error_detail} if local_error_detail else {}),+ **correlation_payload,}returnif self._peer_request_open(trace, request_id):@@ -2485,6 +2515,15 @@ class TurnCorrelationRegistry:if identity is not None and identity.channel == "hub"else trace.served)+ if payload is not None:+ # A backend_error describes the backend's own report, not the+ # earlier gateway response that the backend rejected. The+ # response-only correlation facts are therefore not valid on+ # this terminal variant and must not cross its schema boundary.+ payload = {+ key: value for key, value in payload.items()+ if key != "response_correlation_ids"+ }if payload is None or payload.get("channel") != "hub":return# The backend rejected the turn's result, not one request's.@@ -2522,6 +2561,7 @@ class TurnCorrelationRegistry:reason="engine_down",stream_started=outcome.stream_started,local_error_detail=format_os_errno(outcome.os_errno),+ response_correlation_ids=outcome.response_correlation_ids,force=True,)return@@ -2534,11 +2574,17 @@ class TurnCorrelationRegistry:returntrace.pending_attempts.pop(request_id, None)if decision.action == "return":- trace.served = identity.payload()+ trace.served = {+ **identity.payload(),+ **_response_correlation_payload(outcome.response_correlation_ids),+ }trace.terminal_error = Nonereturn# The engine client already redacted and bounded this text.- detail = {"upstream_detail": outcome.upstream_detail} if outcome.upstream_detail else {}+ detail = {+ "upstream_detail": outcome.upstream_detail+ } if outcome.upstream_detail else {}+ detail.update(_response_correlation_payload(outcome.response_correlation_ids))logger.warning("Model Hub attempt failed turn=%s source=%s model=%s http_status=%s reason=%s ""stream_started=%s detail=%s",diff --git a/docs/plans/model-hub-contracts/README.md b/docs/plans/model-hub-contracts/README.mdindex 832656664..6ae9c3cae 100644--- a/docs/plans/model-hub-contracts/README.md+++ b/docs/plans/model-hub-contracts/README.md@@ -5,6 +5,9 @@ Status: **contract_version 20**, 2026-10-22: active web-search services (S2b-2).changing the released event. `SEARCH_SERVICES_ACTIVE = True` activates service candidates,`service_vendors`, frozen turn settings and the mixed search chain together with S6 disclosure.The versioned mirrors advance together; persisted TurnProvenance retains v5–v19 and adds v20.+Relay diagnostics in this v20 shape may retain only bounded, credential-free `request-id` and+`x-request-id` response facts on the exact owning attempt; arbitrary headers, bodies and+upstream identity inference remain outside the contract.The downgrade warning in `../web-search-services.md` §12 remains required.It builds on v19 (2026-10-21): dormant web-search-service readers and producerdiff --git a/docs/plans/model-hub-contracts/adapter-interface.py b/docs/plans/model-hub-contracts/adapter-interface.pyindex b786b6577..a4c3daace 100644--- a/docs/plans/model-hub-contracts/adapter-interface.py+++ b/docs/plans/model-hub-contracts/adapter-interface.py@@ -17,6 +17,7 @@ import asynciofrom dataclasses import dataclass, fieldfrom datetime import datetimefrom enum import Enum+import refrom typing import Any, AsyncIterator, Awaitable, Callable, Final, Literal, Mapping, Protocol, Sequencefrom .stream_wire import ProtocolSSEState, ProtocolUsageReport@@ -28,6 +29,55 @@ MODEL_PROTOCOLS = ("anthropic", "openai_responses", "openai_chat", "google")SOURCE_PROTOCOLS = MODEL_PROTOCOLS + ("search",)SEARCH_RESPONSE_MAX_BYTES: Final = 8 * 1024 * 1024SEARCH_TRANSPORT_TIMEOUT_SECONDS: Final = 30.0+# Response correlation is a diagnostic fact, not a header-forwarding surface. These+# are the only upstream response header names allowed to cross the engine boundary.+RESPONSE_CORRELATION_HEADER_NAMES: Final = frozenset({"request-id", "x-request-id"})+RESPONSE_CORRELATION_ID_MAX_LENGTH: Final = 256+RESPONSE_CORRELATION_ID_MAX_ITEMS: Final = len(RESPONSE_CORRELATION_HEADER_NAMES)+_CORRELATION_ID_CREDENTIAL_PATTERN = re.compile(+ r"(?i)(?:bearer|basic)\s+|"+ r"(?:authorization|api[_ -]?key|access[_ -]?token|token|secret|password|"+ r"credential|cookie|session|signature)\s*[:=]|"+ r"(?:sk|rk|pk|sess|token|secret|cred|key)[-_][a-z0-9_-]{8,}|"+ r"(?:AIza[a-z0-9_-]{20,}|github_pat_[a-z0-9_]{20,}|gh[pousr]_[a-z0-9_]{20,})|"+ r"ey[a-z0-9_-]+\.[a-z0-9_-]+\.[a-z0-9_-]+$"+)+++def sanitize_response_correlation_ids(values: object) -> tuple[tuple[str, str], ...]:+ """Keep only bounded, credential-free values from the two trusted headers."""++ pairs = values.items() if isinstance(values, Mapping) else values+ try:+ iterator = iter(pairs) # type: ignore[arg-type]+ except TypeError:+ return ()+ retained: list[tuple[str, str]] = []+ seen: set[str] = set()+ for pair in iterator:+ if not isinstance(pair, (tuple, list)) or len(pair) != 2:+ continue+ raw_name, raw_value = pair+ if not isinstance(raw_name, str) or not isinstance(raw_value, str):+ continue+ name = raw_name.lower()+ if name not in RESPONSE_CORRELATION_HEADER_NAMES or name in seen:+ continue+ if (+ not raw_value+ or len(raw_value) > RESPONSE_CORRELATION_ID_MAX_LENGTH+ or any(not 0x21 <= ord(char) <= 0x7E for char in raw_value)+ or _CORRELATION_ID_CREDENTIAL_PATTERN.search(raw_value)+ ):+ continue+ seen.add(name)+ retained.append((name, raw_value))+ if len(retained) >= RESPONSE_CORRELATION_ID_MAX_ITEMS:+ break+ order = {name: index for index, name in enumerate(sorted(RESPONSE_CORRELATION_HEADER_NAMES))}+ return tuple(sorted(retained, key=lambda item: order[item[0]]))++OBSERVATION_OUTCOMES = ("observed","ambiguous",@@ -228,6 +278,10 @@ class RawCallOutcome:# The search tools whose own completed item a served response carried, for# the positive evidence written at settlement (``vibey-hosted-tools.md`` §3.2).hosted_tools: frozenset[str] = frozenset()+ # The response header name is retained in lowercase so relay ``x-request-id``+ # and provider ``request-id`` values cannot be conflated. The client and+ # provenance writer re-validate this tuple before persistence.+ response_correlation_ids: tuple[tuple[str, str], ...] = ()class ObservationOutcome(str, Enum):diff --git a/docs/plans/model-hub-contracts/api.md b/docs/plans/model-hub-contracts/api.mdindex 4c6e94996..cf882fc11 100644--- a/docs/plans/model-hub-contracts/api.md+++ b/docs/plans/model-hub-contracts/api.md@@ -1466,6 +1466,16 @@ of control and bidi characters, whitespace-collapsed, and bounded to 400 charactThey are omitted when no message was observed and in older records, and are neverread by classification.+`failed_attempts[].response_correlation_ids`, `terminal_error.response_correlation_ids`,+and `served.response_correlation_ids` are independently optional arrays of at most two+`{header, value}` facts from that exact response. The only accepted canonical header+names are `request-id` (provider request identity) and `x-request-id` (relay/request+correlation); values are printable ASCII, bounded to 256 characters, and rejected when+they contain controls or credential-like material. Arbitrary response headers,+request headers, request bodies, and raw response bytes are never persisted. These+facts are diagnostic only and do not affect classification, fallback, retry policy,+model identity, or vendor inference. Historical records may omit them.+`terminal_error.reason: backend_error` is an agent backend's own report of a failedturn that the gateway did not end itself(`docs/plans/model-hub-backend-failure-facts.md`). The shared failure bridge records@@ -1916,6 +1926,7 @@ contract harness and API-boundary tests enforce:<!-- authority-consumer: network.failure network_failure.shaped_before_first_byte network_failure.transport_before_first_byte network_failure.shaped_after_first_byte network_failure.transport_after_first_byte --><!-- authority-consumer: live_backoff.health backoff --><!-- authority-consumer: live_backoff.reason models.source.backoff.connection_failed -->+<!-- authority-consumer: response_correlation.header request-id x-request-id -->| Guard | Boundary || --- | --- |diff --git a/docs/plans/model-hub-contracts/mirror-registry.json b/docs/plans/model-hub-contracts/mirror-registry.jsonindex 269806901..9f58e4f9a 100644--- a/docs/plans/model-hub-contracts/mirror-registry.json+++ b/docs/plans/model-hub-contracts/mirror-registry.json@@ -1029,6 +1029,37 @@"marker": "authority-consumer: catalog.guard.error "}]+ },+ {+ "id": "D30",+ "domain": "response correlation header names",+ "authority": {+ "kind": "schema_vocabulary",+ "schema": "turn-provenance.schema.json",+ "paths": ["/definitions/responseCorrelationId/properties/header"]+ },+ "consumers": [+ {+ "kind": "python_string_assignment",+ "file": "core/handlers/model_hub/adapter.py",+ "name": "RESPONSE_CORRELATION_HEADER_NAMES"+ },+ {+ "kind": "python_string_assignment",+ "file": "docs/plans/model-hub-contracts/adapter-interface.py",+ "name": "RESPONSE_CORRELATION_HEADER_NAMES"+ },+ {+ "kind": "typescript_string_union",+ "file": "ui/src/components/settings/models/types.ts",+ "name": "ResponseCorrelationHeader"+ },+ {+ "kind": "marker_tail",+ "file": "docs/plans/model-hub-contracts/api.md",+ "marker": "authority-consumer: response_correlation.header "+ }+ ]}],"ownership_checks": [diff --git a/docs/plans/model-hub-contracts/turn-provenance.schema.json b/docs/plans/model-hub-contracts/turn-provenance.schema.jsonindex 74b0124f1..33ffd6648 100644--- a/docs/plans/model-hub-contracts/turn-provenance.schema.json+++ b/docs/plans/model-hub-contracts/turn-provenance.schema.json@@ -17,6 +17,24 @@"uniqueItems": true,"items": { "type": "string", "minLength": 1 },"description": "The exact hop tier declaration consulted when stripped_reasoning_efforts was produced."+ },+ "responseCorrelationId": {+ "type": "object",+ "required": ["header", "value"],+ "additionalProperties": false,+ "properties": {+ "header": {+ "enum": ["request-id", "x-request-id"],+ "description": "Canonical lowercase response header name. request-id is the provider request id; x-request-id is the relay/request correlation id."+ },+ "value": {+ "type": "string",+ "minLength": 1,+ "maxLength": 256,+ "pattern": "^[\\u0021-\\u007e]+$",+ "description": "Bounded printable-ASCII correlation value after credential-like and control-character rejection."+ }+ }}},"required": [@@ -101,6 +119,13 @@"maxLength": 400,"description": "Optional error text for this attempt, credential-redacted, stripped of control and bidi characters, whitespace-collapsed, and bounded to 400 characters by the engine client: the upstream's message, or for a network attempt the engine's connection diagnostic. Display only; never read by classification. Absent when no message was observed and in historical records."},+ "response_correlation_ids": {+ "type": "array",+ "maxItems": 2,+ "uniqueItems": true,+ "items": { "$ref": "#/definitions/responseCorrelationId" },+ "description": "Optional bounded response correlation facts for this exact attempt. Only request-id and x-request-id survive; arbitrary response headers, controls and credential-like values are discarded."+ },"reason": {"enum": ["quota_exhausted",@@ -138,7 +163,14 @@"channel": { "enum": ["native_cli", "hub"] },"stripped_reasoning_efforts": { "$ref": "#/definitions/strippedReasoningEfforts" },"declared_reasoning_efforts": { "$ref": "#/definitions/declaredReasoningEfforts" },- "origin": { "$ref": "hop-origin.schema.json" }+ "origin": { "$ref": "hop-origin.schema.json" },+ "response_correlation_ids": {+ "type": "array",+ "maxItems": 2,+ "uniqueItems": true,+ "items": { "$ref": "#/definitions/responseCorrelationId" },+ "description": "Optional bounded response correlation facts for this exact served request."+ }},"dependencies": {"stripped_reasoning_efforts": ["declared_reasoning_efforts"],@@ -247,6 +279,13 @@"minLength": 1,"maxLength": 400,"description": "Optional upstream error message for this terminal attempt, under the same redaction and bound as failed_attempts[].upstream_detail. Display only; absent when no message was observed and in historical records."+ },+ "response_correlation_ids": {+ "type": "array",+ "maxItems": 2,+ "uniqueItems": true,+ "items": { "$ref": "#/definitions/responseCorrelationId" },+ "description": "Optional bounded response correlation facts for this exact terminal attempt."}},"dependencies": {@@ -266,7 +305,8 @@"anyOf": [{ "required": ["http_status"] },{ "required": ["upstream_error_code"] },- { "required": ["upstream_detail"] }+ { "required": ["upstream_detail"] },+ { "required": ["response_correlation_ids"] }]}},diff --git a/tests/test_model_hub_config.py b/tests/test_model_hub_config.pyindex 2d6b1460e..b4431d203 100644--- a/tests/test_model_hub_config.py+++ b/tests/test_model_hub_config.py@@ -1005,6 +1005,64 @@ def test_turn_provenance_contract_preserves_an_empty_stripped_effort():_assert_valid("turn-provenance.schema.json", payload)+def test_turn_provenance_correlation_ids_are_strict_and_attempt_scoped():+ served = copy.deepcopy(_schema("turn-provenance.schema.json")["examples"][0])+ served["served"]["response_correlation_ids"] = [+ {"header": "request-id", "value": "provider-request-1"},+ {"header": "x-request-id", "value": "relay-request-1"},+ ]+ _assert_valid("turn-provenance.schema.json", served)++ terminal = copy.deepcopy(served)+ terminal.update(+ outcome="failed_terminal",+ failed_attempts=[],+ served=None,+ terminal_error={+ "source_id": "src_anthkey01",+ "configured_model_id": "claude-opus-4-6",+ "channel": "hub",+ "reason": "protocol_error",+ "stream_started": False,+ "response_correlation_ids": [+ {"header": "x-request-id", "value": "relay-terminal-1"},+ ],+ },+ )+ _assert_valid("turn-provenance.schema.json", terminal)++ for invalid_id in (+ {"header": "set-cookie", "value": "session=secret"},+ {"header": "request-id", "value": "bad\nvalue"},+ {"header": "request-id", "value": "r" * 257},+ ):+ invalid = copy.deepcopy(served)+ invalid["served"]["response_correlation_ids"] = [invalid_id]+ with pytest.raises(ValidationError):+ Draft7Validator(_schema("turn-provenance.schema.json")).validate(invalid)++ backend_error = copy.deepcopy(terminal)+ backend_error["terminal_error"] = {+ "source_id": "src_anthkey01",+ "configured_model_id": "claude-opus-4-6",+ "channel": "hub",+ "reason": "backend_error",+ "stream_started": None,+ "response_correlation_ids": [+ {"header": "request-id", "value": "not-a-backend-fact"},+ ],+ }+ with pytest.raises(ValidationError):+ Draft7Validator(_schema("turn-provenance.schema.json")).validate(backend_error)++ canceled = copy.deepcopy(_schema("turn-provenance.schema.json")["examples"][-1])+ canceled["canceled_attempt"]["response_correlation_ids"] = [+ {"header": "request-id", "value": "not-a-served-response"},+ ]+ with pytest.raises(ValidationError):+ Draft7Validator(_schema("turn-provenance.schema.json")).validate(canceled)++def test_permission_denial_contract_has_no_fallback_member():retired_reason = "permission_denied"event_schema = _schema("resolution-event.schema.json")diff --git a/tests/test_model_hub_engine_response_headers.py b/tests/test_model_hub_engine_response_headers.pyindex 6360e1622..a805aa962 100644--- a/tests/test_model_hub_engine_response_headers.py+++ b/tests/test_model_hub_engine_response_headers.py@@ -19,15 +19,21 @@ from aiohttp import webfrom core.handlers.model_hub.turn_gateway import ModelHubTurnGatewayfrom tests.test_model_hub_declared_tools import COMPLETED, INPUT, _wirefrom tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _service, _source-from vibe.model_hub_runtime.client import ENGINE_RESPONSE_HEADER_ALLOW_LIST, _allowed_engine_header+from vibe.model_hub_runtime.client import (+ ENGINE_RESPONSE_HEADER_ALLOW_LIST,+ _allowed_engine_header,+)+from core.handlers.model_hub.adapter import sanitize_response_correlation_ids# What an upstream may send and the engine forwards: none of it belongs to Avibe's callers.INJECTED = {"Set-Cookie": "upstream-session=secret; Path=/; HttpOnly",- "Request-Id": "req_upstream_1",+ "Anthropic-Request-Id": "anthropic-request-1","Anthropic-Organization-Id": "org-upstream","Retry-After-Ms": "1500","Anthropic-Ratelimit-Unified-5h-Reset": "1797000000",+ "X-Request-Id": "relay-request-1",+ "Request-Id": "provider-request-1","X-Upstream-Injected": "1",}JSON_BODY = {@@ -66,11 +72,36 @@ async def _engine(status: int, stream: bool):await runner.cleanup()-def test_only_the_retry_advice_avibe_uses_is_read_from_an_engine_response():- assert ENGINE_RESPONSE_HEADER_ALLOW_LIST == {"retry-after"}+def test_only_allowlisted_headers_are_read_from_an_engine_response():+ assert ENGINE_RESPONSE_HEADER_ALLOW_LIST == {"retry-after", "request-id", "x-request-id"}headers = {**INJECTED, "Retry-After": "120"}assert _allowed_engine_header(headers, "Retry-After") == "120"- assert {name: _allowed_engine_header(headers, name) for name in INJECTED} == dict.fromkeys(INJECTED)+ assert _allowed_engine_header(headers, "Request-Id") == "provider-request-1"+ assert _allowed_engine_header(headers, "X-Request-Id") == "relay-request-1"+ assert sanitize_response_correlation_ids(headers) == (+ ("request-id", "provider-request-1"),+ ("x-request-id", "relay-request-1"),+ )+ assert _allowed_engine_header(headers, "Set-Cookie") is None+++ ("headers", "expected"),+ [+ ({"X-Request-Id": "relay-1\nforged"}, ()),+ ({"X-Request-Id": "r" * 257}, ()),+ ({"X-Request-Id": "Bearer secret-token-value"}, ()),+ ({"X-Request-Id": "AIzaSyA1234567890abcdefghijklmnopqrstuv"}, ()),+ ({"X-Request-Id": "ghp_1234567890abcdefghijklmnopqrstuv"}, ()),+ ({"X-Request-Id": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.signature"}, ()),+ ({"X-Request-Id": "sk-live_abcdefghijklmnop"}, ()),+ ({"X-Request-Id": "token=secret-value"}, ()),+ ({"Set-Cookie": "session=secret", "Organization-Id": "org-1"}, ()),+ ({"Request-Id": "provider-1", "Set-Cookie": "session=secret"}, (("request-id", "provider-1"),)),+ ],+)+def test_response_correlation_ids_are_bounded_and_credential_free(headers, expected):+ assert sanitize_response_correlation_ids(headers) == expected@pytest.mark.asynciodiff --git a/tests/test_model_hub_provenance.py b/tests/test_model_hub_provenance.pyindex 58a43e1bf..00b9eb5cf 100644--- a/tests/test_model_hub_provenance.py+++ b/tests/test_model_hub_provenance.py@@ -354,6 +354,252 @@ def test_new_terminal_records_retain_only_safe_recognized_error_metadata(assert "credential" not in serialized+def test_response_correlation_ids_stay_on_the_exact_failed_attempt_and_terminal_error(tmp_path):+ registry = TurnCorrelationRegistry(BoundedProvenanceStore(tmp_path / "records.json"))+ token = registry.credentials("claude", "fixture", "turn-correlation")+ turn_id = registry.begin_gateway_request(+ backend="claude", token=token, requested_model_id=MODEL,+ )+ registry.begin_attempt(+ turn_id, source_id="src_primary01", resolved_model_id=MODEL,+ channel="hub", via_mapping=False,+ )+ registry.finish_attempt(+ turn_id,+ outcome=RawCallOutcome(+ kind=RawOutcomeKind.HTTP_ERROR,+ http_status=429,+ error_code="rate_limit_error",+ redacted_message=None,+ stream_started=False,+ model_id=MODEL,+ source_id="src_primary01",+ upstream_detail="first upstream detail",+ response_correlation_ids=(+ ("x-request-id", "relay-first"),+ ("request-id", "provider-first"),+ ),+ ),+ decision=ResolutionDecision("fallback", reason="rate_limited", cooldown_seconds=30),+ )+ registry.begin_attempt(+ turn_id, source_id="src_terminal01", resolved_model_id=MODEL,+ channel="hub", via_mapping=False,+ )+ registry.finish_attempt(+ turn_id,+ outcome=RawCallOutcome(+ kind=RawOutcomeKind.HTTP_ERROR,+ http_status=400,+ error_code="invalid_request_error",+ redacted_message=None,+ stream_started=False,+ model_id=MODEL,+ source_id="src_terminal01",+ upstream_detail="terminal upstream detail",+ response_correlation_ids=(("request-id", "provider-terminal"),),+ ),+ decision=ResolutionDecision("surface", error_code="upstream_request_invalid", downstream_status=400),+ )+ registry.settle(turn_id, settled_by=SETTLED_BY_TERMINAL_RESULT)++ record = registry.store.get(turn_id)+ assert record is not None+ assert record["failed_attempts"][0]["response_correlation_ids"] == [+ {"header": "request-id", "value": "provider-first"},+ {"header": "x-request-id", "value": "relay-first"},+ ]+ assert record["terminal_error"]["response_correlation_ids"] == [+ {"header": "request-id", "value": "provider-terminal"},+ ]+ assert record["failed_attempts"][0]["upstream_detail"] == "first upstream detail"+ assert record["terminal_error"]["upstream_detail"] == "terminal upstream detail"+++def test_served_response_correlation_ids_belong_to_the_served_attempt(tmp_path):+ registry = TurnCorrelationRegistry(BoundedProvenanceStore(tmp_path / "records.json"))+ token = registry.credentials("claude", "fixture", "turn-served-correlation")+ turn_id = registry.begin_gateway_request(+ backend="claude", token=token, requested_model_id=MODEL,+ )+ registry.begin_attempt(+ turn_id, source_id="src_first0001", resolved_model_id=MODEL,+ channel="hub", via_mapping=False,+ )+ registry.finish_attempt(+ turn_id,+ outcome=RawCallOutcome(+ kind=RawOutcomeKind.HTTP_ERROR,+ http_status=503,+ error_code="server_error",+ redacted_message=None,+ stream_started=False,+ model_id=MODEL,+ source_id="src_first0001",+ response_correlation_ids=(("x-request-id", "failed-request"),),+ ),+ decision=ResolutionDecision("fallback", reason="server_error", cooldown_seconds=30),+ )+ registry.begin_attempt(+ turn_id, source_id="src_served01", resolved_model_id=MODEL,+ channel="hub", via_mapping=False,+ )+ registry.finish_attempt(+ turn_id,+ outcome=RawCallOutcome(+ kind=RawOutcomeKind.SUCCESS,+ http_status=200,+ error_code=None,+ redacted_message=None,+ stream_started=True,+ model_id=MODEL,+ source_id="src_served01",+ response_correlation_ids=(("x-request-id", "served-request"),),+ ),+ decision=ResolutionDecision("return"),+ )+ registry.settle(turn_id, settled_by=SETTLED_BY_TERMINAL_RESULT)++ record = registry.store.get(turn_id)+ assert record is not None+ assert record["failed_attempts"] == [+ {+ "source_id": "src_first0001",+ "configured_model_id": MODEL,+ "channel": "hub",+ "reason": "server_error",+ "http_status": 503,+ "response_correlation_ids": [+ {"header": "x-request-id", "value": "failed-request"},+ ],+ },+ ]+ assert record["served"] == {+ "source_id": "src_served01",+ "configured_model_id": MODEL,+ "channel": "hub",+ "response_correlation_ids": [+ {"header": "x-request-id", "value": "served-request"},+ ],+ }+ assert record["terminal_error"] is None+++def test_backend_failure_drops_response_correlation_ids_from_prior_served_response(tmp_path):+ registry = TurnCorrelationRegistry(BoundedProvenanceStore(tmp_path / "records.json"))+ token = registry.credentials("codex", "fixture", "turn-backend-failure-correlation")+ turn_id = registry.begin_gateway_request(+ backend="codex", token=token, requested_model_id=MODEL,+ )+ registry.begin_attempt(+ turn_id, source_id="src_served01", resolved_model_id=MODEL,+ channel="hub", via_mapping=False,+ )+ registry.finish_attempt(+ turn_id,+ outcome=RawCallOutcome(+ kind=RawOutcomeKind.SUCCESS,+ http_status=200,+ error_code=None,+ redacted_message=None,+ stream_started=True,+ model_id=MODEL,+ source_id="src_served01",+ response_correlation_ids=(("x-request-id", "served-request"),),+ ),+ decision=ResolutionDecision("return"),+ )+ registry.fail_hub_attempt(turn_id, error_kind="stalled", diagnostic="backend rejected the response")++ trace = registry._traces[turn_id]+ assert trace.terminal_error is not None+ assert trace.terminal_error["reason"] == "backend_error"+ assert "response_correlation_ids" not in trace.terminal_error+++def test_response_correlation_ids_are_sanitized_before_provenance_persistence(tmp_path):+ registry = TurnCorrelationRegistry(BoundedProvenanceStore(tmp_path / "records.json"))+ token = registry.credentials("claude", "fixture", "turn-correlation-safe")+ turn_id = registry.begin_gateway_request(+ backend="claude", token=token, requested_model_id=MODEL,+ )+ registry.begin_attempt(+ turn_id, source_id="src_primary01", resolved_model_id=MODEL,+ channel="hub", via_mapping=False,+ )+ registry.finish_attempt(+ turn_id,+ outcome=RawCallOutcome(+ kind=RawOutcomeKind.HTTP_ERROR,+ http_status=400,+ error_code="invalid_request_error",+ redacted_message=None,+ stream_started=False,+ model_id=MODEL,+ source_id="src_primary01",+ response_correlation_ids=(+ ("x-request-id", "bad\nvalue"),+ ("request-id", "Bearer secret-token-value"),+ ("Set-Cookie", "session=secret"),+ ("Request-Id", "provider-safe"),+ ),+ ),+ decision=ResolutionDecision("surface", error_code="upstream_request_invalid", downstream_status=400),+ )+ registry.settle(turn_id, settled_by=SETTLED_BY_TERMINAL_RESULT)++ record = registry.store.get(turn_id)+ assert record is not None+ assert record["terminal_error"]["response_correlation_ids"] == [+ {"header": "request-id", "value": "provider-safe"},+ ]+ serialized = registry.store.path.read_text()+ assert "secret-token" not in serialized+ assert "Set-Cookie" not in serialized+++def test_engine_down_terminal_error_keeps_safe_response_correlation_ids(tmp_path):+ registry = TurnCorrelationRegistry(BoundedProvenanceStore(tmp_path / "records.json"))+ token = registry.credentials("claude", "fixture", "turn-engine-down-correlation")+ turn_id = registry.begin_gateway_request(+ backend="claude", token=token, requested_model_id=MODEL,+ )+ registry.begin_attempt(+ turn_id, source_id="src_engine01", resolved_model_id=MODEL,+ channel="hub", via_mapping=False,+ )+ registry.finish_attempt(+ turn_id,+ outcome=RawCallOutcome(+ kind=RawOutcomeKind.NETWORK_ERROR,+ http_status=502,+ error_code="engine_down",+ redacted_message=None,+ stream_started=False,+ model_id=MODEL,+ source_id="src_engine01",+ response_correlation_ids=(+ ("x-request-id", "relay-engine-down"),+ ),+ ),+ decision=ResolutionDecision("surface", error_code="engine_down", downstream_status=502),+ )+ registry._terminalize_gateway_exit(turn_id, stream_started=False, force=True)+ registry.settle(turn_id, settled_by=SETTLED_BY_TERMINAL_RESULT)++ record = registry.store.get(turn_id)+ assert record is not None+ assert record["terminal_error"] == {+ "source_id": None,+ "configured_model_id": None,+ "channel": None,+ "reason": "engine_down",+ "stream_started": False,+ "response_correlation_ids": [+ {"header": "x-request-id", "value": "relay-engine-down"},+ ],+ }+def test_model_history_read_preserves_legacy_records_and_current_route_state(tmp_path):service, store, adapter = _service(tmp_path, _sparse_config())legacy = _record("legacy")diff --git a/tests/test_model_hub_retry_advice.py b/tests/test_model_hub_retry_advice.pyindex 4bbe6a1a6..5972dc3c8 100644--- a/tests/test_model_hub_retry_advice.py+++ b/tests/test_model_hub_retry_advice.py@@ -114,6 +114,52 @@ def test_existing_outcome_constructors_keep_retry_advice_optional(optional_countassert outcome.usage == (usage if optional_count else None)[email protected]("protocol", WIRE)[email protected]("stream", [False, True], ids=["buffered", "streaming"])+async def test_failed_response_carries_only_allowlisted_correlation_ids(protocol: str, stream: bool) -> None:+ async def respond(_request: web.Request) -> web.Response:+ return web.Response(+ status=400,+ body=ERROR_BODY,+ headers={+ "X-Request-Id": "relay-request-1",+ "Request-Id": "provider-request-1",+ "Set-Cookie": "session=secret",+ "Organization-Id": "org-private",+ },+ content_type="application/json",+ )++ async with _engine(respond) as (client, _requests):+ outcome, body = await _invoke(client, protocol, stream)++ assert outcome.kind is RawOutcomeKind.HTTP_ERROR+ assert outcome.response_correlation_ids == (+ ("request-id", "provider-request-1"),+ ("x-request-id", "relay-request-1"),+ )+ assert body == b""++[email protected]("protocol", WIRE)[email protected]("stream", [False, True], ids=["buffered", "streaming"])+async def test_success_response_carries_its_own_correlation_ids(protocol: str, stream: bool) -> None:+ _path, buffered, streamed, _stream_error = WIRE[protocol]++ async def respond(_request: web.Request) -> web.Response:+ return web.Response(+ status=200,+ body=streamed if stream else buffered,+ headers={"X-Request-Id": "served-request-1"},+ content_type="text/event-stream" if stream else "application/json",+ )++ async with _engine(respond) as (client, _requests):+ outcome, _body = await _invoke(client, protocol, stream)++ assert outcome.kind is RawOutcomeKind.SUCCESS+ assert outcome.response_correlation_ids == (("x-request-id", "served-request-1"),)+@pytest.mark.parametrize("protocol", WIRE)@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streaming"])@pytest.mark.parametrize(diff --git a/ui/e2e/support/api.ts b/ui/e2e/support/api.tsindex b8bcc9bf3..a3d5e82bb 100644--- a/ui/e2e/support/api.ts+++ b/ui/e2e/support/api.ts@@ -96,6 +96,8 @@ export type Probe = {export type TurnReceipt = { ok: true; session_id: string; turn_id: string; queued: boolean };+export type ResponseCorrelationId = { header: 'request-id' | 'x-request-id'; value: string };+export type RecordedTurn = {turn_id: string;ts: string;@@ -107,8 +109,9 @@ export type RecordedTurn = {reason: string;http_status?: number | null;upstream_error_code?: string | null;+ response_correlation_ids?: ResponseCorrelationId[];};- served: null | { source_id: string; configured_model_id: string };+ served: null | { source_id: string; configured_model_id: string; response_correlation_ids?: ResponseCorrelationId[] };};export type Runtime = {@@ -355,8 +358,8 @@ export class HubApi {const payload = await this.read<{ ok: boolean; contract_version: number; provenance: RecordedTurn | null }>(`/api/models/agents/${backend}/provenance?model=${encodeURIComponent(model)}`,);- if (payload.ok !== true || payload.contract_version !== 11 || !('provenance' in payload)) {- throw new Error('Latest recorded turn did not use the v10 provenance envelope');+ if (payload.ok !== true || payload.contract_version !== 20 || !('provenance' in payload)) {+ throw new Error('Latest recorded turn did not use the v20 provenance envelope');}return payload.provenance;}diff --git a/ui/src/components/settings/models/types.ts b/ui/src/components/settings/models/types.tsindex 2327546fb..084cec4c4 100644--- a/ui/src/components/settings/models/types.ts+++ b/ui/src/components/settings/models/types.ts@@ -642,6 +642,13 @@ export type AgentChain = {supply_state: 'ok' | 'waiting' | 'interrupted';};+export type ResponseCorrelationHeader = 'request-id' | 'x-request-id';+export type ResponseCorrelationId = {+ /** Canonical response header name; values are bounded and sanitized server-side. */+ header: ResponseCorrelationHeader;+ value: string;+};+export type RecordedAttempt = {source_id: string;configured_model_id: string;@@ -649,7 +656,9 @@ export type RecordedAttempt = {stripped_reasoning_efforts?: string[];declared_reasoning_efforts?: string[];origin?: { provider: string; api: ModelProtocol; model: string };+ response_correlation_ids?: ResponseCorrelationId[];};+export type CanceledAttempt = Omit<RecordedAttempt, 'response_correlation_ids'>;export type TurnProvenance = {contract_version: (typeof PERSISTED_TURN_CONTRACT_VERSIONS)[number];turn_id: string;@@ -659,7 +668,7 @@ export type TurnProvenance = {outcome: 'served' | 'exhausted' | 'failed_terminal' | 'no_candidate' | 'canceled';failed_attempts: Array<RecordedAttempt & { reason: ResolutionReason; http_status?: number | null; upstream_detail?: string }>;served: RecordedAttempt | null;- canceled_attempt: RecordedAttempt | null;+ canceled_attempt: CanceledAttempt | null;terminal_error: {source_id: string | null;configured_model_id: string | null;@@ -674,6 +683,7 @@ export type TurnProvenance = {http_status?: number | null;upstream_error_code?: string | null;upstream_detail?: string;+ response_correlation_ids?: ResponseCorrelationId[];error_kind?: never;backend_error_detail?: never;} | {diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.pyindex db2f4a86a..e79f522af 100644--- a/vibe/model_hub_runtime/client.py+++ b/vibe/model_hub_runtime/client.py@@ -29,7 +29,9 @@ from core.handlers.model_hub.adapter import (InvokeCancelledError,RawCallOutcome,RawOutcomeKind,+ RESPONSE_CORRELATION_HEADER_NAMES,UpstreamRejection,+ sanitize_response_correlation_ids,)from core.handlers.model_hub.async_owner import run_owned_in_threadfrom core.handlers.model_hub.classification import UPSTREAM_MACHINE_ERROR_CODES@@ -101,10 +103,14 @@ _ProjectedJSON = TypeVar("_ProjectedJSON")# The engine forwards an upstream's response headers (``passthrough-headers``),# minus hop-by-hop ones, and on errors it filters nothing. Avibe reads only these-# and builds every gateway response from its own headers, so no other upstream-# header (``set-cookie``, ``request-id``, an organization id) reaches Vibey, the-# Web UI or a gateway client. A header joins this list with the code that uses it.-ENGINE_RESPONSE_HEADER_ALLOW_LIST: frozenset[str] = frozenset({"retry-after"})+# allowlisted names and builds every gateway response from its own headers, so no+# other upstream header (``set-cookie``, organization ids, rate-limit details)+# reaches Vibey, the Web UI or a gateway client. Correlation values are projected+# again by ``sanitize_response_correlation_ids`` before they cross the outcome+# boundary.+ENGINE_RESPONSE_HEADER_ALLOW_LIST: frozenset[str] = frozenset(+ {"retry-after", *RESPONSE_CORRELATION_HEADER_NAMES}+)def _allowed_engine_header(headers: Mapping[str, str], name: str) -> str | None:@@ -560,6 +566,7 @@ class EngineClient:response: aiohttp.ClientResponse | None = Noneretry_after: str | None = Noneresponse_received_at: datetime | None = None+ response_correlation_ids: tuple[tuple[str, str], ...] = ()first_received = Falsemodel_output_started = Falseownership_transferred = False@@ -584,6 +591,8 @@ class EngineClient:outcome = replace(outcome, usage=wire_state.usage)if wire_state is not None and wire_state.model_output_started:outcome = replace(outcome, recovery_verified=True)+ if response_correlation_ids:+ outcome = replace(outcome, response_correlation_ids=response_correlation_ids)if response_received_at is not None:outcome = replace(outcome,@@ -610,6 +619,7 @@ class EngineClient:headers=headers,allow_redirects=False,)+ response_correlation_ids = sanitize_response_correlation_ids(response.headers)if response.status >= 300:response_received_at = datetime.now(timezone.utc)retry_after = _allowed_engine_header(response.headers, "Retry-After")@@ -683,6 +693,7 @@ class EngineClient:model_id=model_id,http_status=response.status,stream_started=False,+ response_correlation_ids=response_correlation_ids,)assert outcome is not Noneresponse.close()@@ -729,6 +740,7 @@ class EngineClient:model_id=model_id,http_status=response.status,stream_started=observation.outcome == "served",+ response_correlation_ids=response_correlation_ids,)assert outcome is not Noneresponse.close()@@ -752,6 +764,7 @@ class EngineClient:wire_state=wire_state,source=source,model_id=model_id,+ response_correlation_ids=response_correlation_ids,)model_output_started = wire_state.model_output_startedif prelude_outcome is not None:@@ -776,6 +789,7 @@ class EngineClient:outcome_future=outcome_future,wire_state=wire_state,on_transport_done=on_transport_done,+ response_correlation_ids=response_correlation_ids,)async def close_response_stream() -> None:@@ -792,6 +806,8 @@ class EngineClient:model_id,response.status,)+ if outcome is not None:+ outcome = replace(outcome, response_correlation_ids=response_correlation_ids)if outcome is not None and not outcome_future.done():outcome_future.set_result(outcome)@@ -1654,6 +1670,7 @@ async def _read_stream_prelude(wire_state: ProtocolSSEState,source: SourceRecord,model_id: str,+ response_correlation_ids: tuple[tuple[str, str], ...] = (),) -> RawCallOutcome | None:"""Buffer transport metadata until the sole first-model-output fact.@@ -1671,7 +1688,7 @@ async def _read_stream_prelude(response.status,)if outcome is not None:- return outcome+ return replace(outcome, response_correlation_ids=response_correlation_ids)chunk = await response.content.read(_STREAM_CHUNK_BYTES)if not chunk:completion = _observed_stream_terminal_outcome(@@ -1681,8 +1698,10 @@ async def _read_stream_prelude(response.status,)if completion is not None:- return completion- return _prelude_ended_outcome(source, model_id, response.status)+ return replace(completion, response_correlation_ids=response_correlation_ids)+ return _prelude_ended_outcome(+ source, model_id, response.status, response_correlation_ids=response_correlation_ids,+ )await _received(chunk, prelude=prelude, wire_state=wire_state)return None@@ -1703,6 +1722,7 @@ def _prelude_ended_outcome(source: SourceRecord,model_id: str,http_status: int,+ response_correlation_ids: tuple[tuple[str, str], ...] = (),) -> RawCallOutcome:return _outcome(kind=RawOutcomeKind.NETWORK_ERROR,@@ -1710,6 +1730,7 @@ def _prelude_ended_outcome(model_id=model_id,http_status=http_status,message="upstream stream ended before model output",+ response_correlation_ids=response_correlation_ids,)@@ -1723,6 +1744,7 @@ async def _response_stream(protocol: str,outcome_future: asyncio.Future[RawCallOutcome],wire_state: ProtocolSSEState,+ response_correlation_ids: tuple[tuple[str, str], ...] = (),on_transport_done: Callable[[], None] | None = None,) -> AsyncIterator[bytes]:outcome: RawCallOutcome | None = None@@ -1796,6 +1818,8 @@ async def _response_stream(model_id,response.status,)+ if outcome is not None:+ outcome = replace(outcome, response_correlation_ids=response_correlation_ids)prelude.close()response.close()try:@@ -1832,6 +1856,7 @@ def _reduce_protocol_observation(model_id: str,http_status: int | None,stream_started: bool,+ response_correlation_ids: tuple[tuple[str, str], ...] = (),) -> RawCallOutcome | None:"""Sole conversion from protocol observations to runtime call outcomes.@@ -1850,6 +1875,7 @@ def _reduce_protocol_observation(model_id=model_id,http_status=http_status,stream_started=stream_started,+ response_correlation_ids=response_correlation_ids,usage=observation.usage,recovery_verified=observation.recovery_verified,hosted_tools=observation.hosted_tools,@@ -1896,6 +1922,7 @@ def _reduce_protocol_observation(model_id=model_id,message="engine could not reach the upstream",stream_started=stream_started,+ response_correlation_ids=response_correlation_ids,usage=observation.usage,recovery_verified=observation.recovery_verified,upstream_detail=upstream_detail,@@ -1910,6 +1937,7 @@ def _reduce_protocol_observation(error_candidates=candidates,message=observation.message or "upstream returned a protocol error event",stream_started=stream_started,+ response_correlation_ids=response_correlation_ids,usage=observation.usage,recovery_verified=observation.recovery_verified,upstream_detail=upstream_detail,@@ -1930,6 +1958,7 @@ def _reduce_protocol_observation(http_status=http_status,message=observation.message or "upstream emitted invalid protocol data",stream_started=stream_started,+ response_correlation_ids=response_correlation_ids,usage=observation.usage,recovery_verified=observation.recovery_verified,)@@ -2022,6 +2051,7 @@ def _outcome(stream_started: bool = False,usage: ProtocolUsageReport | None = None,recovery_verified: bool = False,+ response_correlation_ids: tuple[tuple[str, str], ...] = (),upstream_detail: str | None = None,os_errno: int | None = None,rejection: UpstreamRejection | None = None,@@ -2039,6 +2069,7 @@ def _outcome(error_candidates=error_candidates,usage=usage,recovery_verified=recovery_verified,+ response_correlation_ids=response_correlation_ids,upstream_detail=upstream_detail,os_errno=os_errno,rejection=rejection,