Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

84.7 KB
Raw Blame
mod runtime_support;

#[path = "p0_gateway_runtime/bearer_auth.rs"]
mod bearer_auth;
#[path = "p0_gateway_runtime/decision_branches.rs"]
mod decision_branches;
#[path = "p0_gateway_runtime/hermes_identity.rs"]
mod hermes_identity;

use std::collections::BTreeMap;
use std::time::{Duration, Instant};

use hiroute_gateway::attempt_outcome::{
    AttemptFailureClass, ConnectorErrorProfile, FailureStateScope, ProviderFailureKind,
    RawAttemptFailure, classify_failure,
};

use runtime_support::*;

const RESPONSES_OK: &[u8] = br#"{"id":"accepted","model":"runtime-native","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}"#;
const RESPONSES_TOOL_CALL: &[u8] = br#"{"id":"tool-call","model":"runtime-native","status":"completed","output":[{"type":"function_call","id":"fc-native","call_id":"native-call","namespace":"tools","name":"lookup","arguments":"{}","status":"completed"}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}"#;
const RESPONSES_STREAM_OK: &[u8] = b"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"stream-ok\",\"model\":\"runtime-native\"}}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"runtime-message\",\"output_index\":0,\"content_index\":0,\"delta\":\"ok\"}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"stream-ok\",\"model\":\"runtime-native\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"runtime-message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"ok\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n";
const FALLBACK_OK: &[u8] = br#"{"id":"accepted-fallback","model":"runtime-native","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fallback"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}"#;
const CLASSIFIER_COMPLEX: &[u8] = br#"{"group":"primary"}"#;
const CLASSIFIER_SIMPLE: &[u8] = br#"{"group":"regular"}"#;
const CLASSIFIER_SIMPLE_WITH_ASSESSMENT: &[u8] =
    br#"{"group":"regular","assessment":{"score":0.0,"partial":false}}"#;
const CLASSIFIER_INVALID: &[u8] = br#"{"id":"classification","model":"runtime-native","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"{\"category\":\"simple\"}"}]}]}"#;
const AUTH_ERROR: &[u8] =
    br#"{"error":{"type":"invalid_api_key","message":"private auth marker"}}"#;
const QUOTA_ERROR: &[u8] =
    br#"{"error":{"type":"insufficient_quota","message":"private quota marker"}}"#;
const OVERLOAD_ERROR: &[u8] =
    br#"{"error":{"type":"rate_limit_exceeded","message":"private overload marker"}}"#;
const PROTOCOL_ERROR: &[u8] =
    br#"{"error":{"type":"protocol_error","message":"private protocol marker"}}"#;
const PERMANENT_ERROR: &[u8] =
    br#"{"error":{"type":"invalid_request_error","message":"private permanent marker"}}"#;

#[path = "p0_gateway_runtime/reasoning.rs"]
mod reasoning;

#[path = "p0_gateway_runtime/accepted_history.rs"]
mod accepted_history;

#[path = "p0_gateway_runtime/responses_options.rs"]
mod responses_options;

#[path = "p0_gateway_runtime/nonstream_terminal.rs"]
mod nonstream_terminal;

#[test]
fn fixed_requests_retry_only_same_source_keys_and_never_enter_an_allowed_plan() {
    for (status, error) in [(401, AUTH_ERROR), (403, AUTH_ERROR), (429, QUOTA_ERROR)] {
        let primary = NativeProvider::start(vec![
            ProviderReply::Complete {
                status,
                error_kind: None,
                body: error,
            },
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
        ]);
        let other_account = NativeProvider::start(vec![ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: FALLBACK_OK,
        }]);
        let other_model = NativeProvider::start(vec![ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: FALLBACK_OK,
        }]);
        let fixture = RuntimeFixture::launch_fixed(
            &[&primary, &other_account, &other_model],
            2,
            &[2, 1, 1],
            None,
        );
        let response = fixture.request();
        assert_eq!(response.status, 200, "status={status}");
        assert_eq!(primary.calls(), 2, "status={status}");
        assert_eq!((other_account.calls(), other_model.calls()), (0, 0));
        let requests = primary.requests();
        for (index, wire) in requests.iter().enumerate() {
            assert_eq!(
                wire_header(wire, "authorization"),
                Some(format!("Bearer provider-secret-1-{}", index + 1))
            );
            assert!(String::from_utf8_lossy(wire).contains("runtime-native-model-1"));
            assert!(wire_header(wire, "x-hiroute-token").is_none());
        }
    }
}

#[path = "p0_gateway_runtime/fixed_isolation.rs"]
mod fixed_isolation;

#[test]
fn real_hirouted_reaches_native_provider_and_falls_back_before_commit() {
    hiroute_e2e::p0_execution_receipt!(
        "runtime.fallback",
        [
            "runtime.binding_fault",
            "runtime.frozen_budget",
            "runtime.precommit_fallback"
        ]
    );
    let first = NativeProvider::start(vec![ProviderReply::Complete {
        status: 503,
        error_kind: None,
        body: b"private first-provider error",
    }]);
    let second = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: br#"{"id":"accepted-second","model":"runtime-native","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}"#,
    }]);
    let fixture = RuntimeFixture::launch(&[&first, &second], 2);

    let response = fixture.request();

    assert_eq!(
        response.status,
        200,
        "{}",
        String::from_utf8_lossy(&response.body)
    );
    assert_ne!(response.status, 501);
    let document: serde_json::Value = serde_json::from_slice(&response.body).unwrap();
    assert_eq!(document["id"], "accepted-second");
    assert_eq!(document["output"][0]["content"][0]["text"], "ok");
    assert_eq!(first.calls(), 1);
    assert_eq!(second.calls(), 1);
    let first_wire = String::from_utf8_lossy(&first.requests()[0]).to_ascii_lowercase();
    let second_wire = String::from_utf8_lossy(&second.requests()[0]).to_ascii_lowercase();
    assert!(first_wire.contains("authorization: bearer provider-secret-1"));
    assert!(second_wire.contains("authorization: bearer provider-secret-2"));
    assert!(first_wire.contains("post /v1/responses http/1.1"));
    assert!(
        !response
            .body
            .windows(b"private first-provider error".len())
            .any(|window| window == b"private first-provider error")
    );
}

#[test]
fn real_listener_consumes_planner_order_and_binds_reason_ledger_identity() {
    hiroute_e2e::p0_execution_receipt!(
        "planner.runtime_consumption",
        [
            "planner.frozen_order_consumed",
            "planner.reason_ledger_identity"
        ]
    );
    let publication_first = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let planner_first = NativeProvider::start(vec![ProviderReply::Complete {
        status: 503,
        error_kind: None,
        body: br#"{"error":{"type":"server_error","message":"retry"}}"#,
    }]);
    let fixture = RuntimeFixture::launch_with_publication_candidates(
        &[&publication_first, &planner_first],
        2,
        Some(&[
            PublicationCandidate {
                provider_index: 1,
                upstream_protocol: "responses",
                statically_enabled: true,
            },
            PublicationCandidate {
                provider_index: 0,
                upstream_protocol: "responses",
                statically_enabled: true,
            },
        ]),
    );

    assert_eq!(fixture.request().status, 200);
    assert_eq!(planner_first.calls(), 1);
    assert_eq!(publication_first.calls(), 1);
    let planner_wire = planner_first.requests();
    let publication_wire = publication_first.requests();
    let reason_one = wire_header(&planner_wire[0], "x-hiroute-reason-ledger-id").unwrap();
    let reason_two = wire_header(&publication_wire[0], "x-hiroute-reason-ledger-id").unwrap();
    assert_eq!(reason_one, reason_two);
    assert!(reason_one.starts_with("sha256:"));
    assert_ne!(
        wire_header(&planner_wire[0], "x-hiroute-profile-digest"),
        wire_header(&publication_wire[0], "x-hiroute-profile-digest")
    );
}

#[test]
fn real_listener_never_materializes_planner_exclusions() {
    let excluded = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let selected = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let fixture = RuntimeFixture::launch_with_publication_candidates(
        &[&excluded, &selected],
        2,
        Some(&[
            PublicationCandidate {
                provider_index: 0,
                upstream_protocol: "responses",
                statically_enabled: false,
            },
            PublicationCandidate {
                provider_index: 1,
                upstream_protocol: "responses",
                statically_enabled: true,
            },
        ]),
    );

    assert_eq!(fixture.request().status, 200);
    assert_eq!(excluded.calls(), 0);
    assert_eq!(selected.calls(), 1);
}

#[test]
fn malformed_success_and_typed_nonstream_error_fall_back_before_commit() {
    for (case, first_reply) in [
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: br#"{"id":"malformed"}"#,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: br#"{"id":"empty-success","model":"runtime-native","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":0,"total_tokens":1}}"#,
        },
        ProviderReply::Complete {
            status: 429,
            error_kind: None,
            body: br#"{"error":{"type":"insufficient_quota","message":"retry"}}"#,
        },
    ]
    .into_iter()
    .enumerate()
    {
        let first = NativeProvider::start(vec![first_reply]);
        let second = NativeProvider::start(vec![ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        }]);
        let fixture = RuntimeFixture::launch(&[&first, &second], 2);
        let response = fixture.request();
        assert_eq!(
            response.status,
            200,
            "case={case} first_calls={} second_calls={} body={}",
            first.calls(),
            second.calls(),
            String::from_utf8_lossy(&response.body)
        );
        assert_eq!(first.calls(), 1, "case={case}");
        assert_eq!(second.calls(), 1, "case={case}");
    }
}

#[test]
fn forged_generic_error_header_cannot_change_connector_classification() {
    for (status, forged_kind) in [(429, "quota"), (422, "protocol")] {
        let forged = NativeProvider::start(vec![
            ProviderReply::Complete {
                status,
                error_kind: Some(forged_kind),
                body: br#"{"error":{"type":"invented_provider_signal","message":"stop"}}"#,
            },
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
        ]);
        let fallback = NativeProvider::start(vec![ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        }]);
        let fixture = RuntimeFixture::launch(&[&forged, &fallback], 2);

        let response = fixture.request();
        assert_eq!(
            response.status,
            200,
            "forged_calls={} fallback_calls={} body={}",
            forged.calls(),
            fallback.calls(),
            String::from_utf8_lossy(&response.body)
        );
        assert_eq!(forged.calls(), 1);
        assert_eq!(fallback.calls(), 1);
        assert!(!String::from_utf8_lossy(&response.body).contains("invented_provider_signal"));
        let next = fixture
            .request_body(br#"{"model":"runtime-model","input":"new request","stream":false}"#);
        assert_eq!(next.status, 200);
        assert_eq!(
            forged.calls(),
            2,
            "forged kind cannot quarantine an unclassified failure"
        );
        assert_eq!(fallback.calls(), 1);
    }
}

#[test]
fn control_only_successful_streams_preserve_native_output() {
    for first_body in [
        b"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"empty-stream\",\"model\":\"runtime-native\"}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"empty-stream\",\"model\":\"runtime-native\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":0,\"total_tokens\":1}}}\n\n".as_slice(),
        b": keepalive\n\nevent: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"control-only\",\"model\":\"runtime-native\"}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"control-only\",\"model\":\"runtime-native\",\"status\":\"completed\",\"output\":[]}}\n\n".as_slice(),
    ] {
        let first = NativeProvider::start(vec![ProviderReply::StreamComplete {
            status: 200,
            body: first_body,
        }]);
        let second = NativeProvider::start(vec![ProviderReply::StreamComplete {
            status: 200,
            body: RESPONSES_STREAM_OK,
        }]);
        let fixture = RuntimeFixture::launch_with_observation(
            &[&first, &second], 2, ObservationFaults::healthy());

        let response =
            fixture.request_body(br#"{"model":"runtime-model","input":"hello","stream":true}"#);
        assert_eq!(response.status, 200);
        assert_eq!(first.calls(), 1);
        assert_eq!(second.calls(), 0);
        assert!(!String::from_utf8_lossy(&response.body).contains("response.output_text.delta"));
        assert!(String::from_utf8_lossy(&response.body).contains("response.completed"));
    }
}

#[test]
fn streaming_failures_before_body_relay_even_after_success_headers() {
    for (case, reply) in [
        ProviderReply::StreamComplete {
            status: 200,
            body: FAILED_STREAM,
        },
        ProviderReply::StreamThenClose {
            body: b"",
            declared_length: 100,
        },
        ProviderReply::StreamComplete {
            status: 200,
            body: b"event: response.output_text.delta\ndata: {invalid}\n\n",
        },
    ]
    .into_iter()
    .enumerate()
    {
        let first = NativeProvider::start(vec![reply]);
        let fallback = NativeProvider::start(vec![ProviderReply::StreamComplete {
            status: 200,
            body: RESPONSES_STREAM_OK,
        }]);
        let fixture = RuntimeFixture::launch_with_observation(
            &[&first, &fallback],
            2,
            ObservationFaults::healthy(),
        );
        let response =
            fixture.request_body(br#"{"model":"runtime-model","input":"hello","stream":true}"#);
        assert_eq!(response.status, 200);
        assert_eq!(first.calls(), 1);
        assert_eq!(fallback.calls(), 1);
        let body = String::from_utf8_lossy(&response.body);
        assert!(body.contains("response.completed"), "{body}");
        assert!(
            !body.contains("failed-stream"),
            "failed prefix must not leak: {body}"
        );
        let facts = wait_execution_facts(&fixture, 1);
        assert!(
            facts
                .iter()
                .any(|row| row.pointer("/fact/kind").and_then(|v| v.as_str())
                    == Some("attempt_finished")
                    && row.pointer("/fact/disposition").and_then(|v| v.as_str())
                        == Some("continue")
                    && row.pointer("/fact/retryable").and_then(|v| v.as_bool()) == Some(true)),
            "retry diagnostics must retain the decision that caused fallback: {facts:?}"
        );
        if case == 0 {
            assert!(
                facts
                    .iter()
                    .any(|row| row.pointer("/fact/kind").and_then(|v| v.as_str())
                        == Some("usage_and_cache")
                        && row.pointer("/fact/input_tokens").and_then(|v| v.as_u64()) == Some(7)
                        && row.pointer("/fact/output_tokens").and_then(|v| v.as_u64()) == Some(2)),
                "failed attempt usage must survive fallback: {facts:?}"
            );
        }
        assert!(
            facts
                .iter()
                .any(|row| row.pointer("/fact/kind").and_then(|v| v.as_str())
                    == Some("request_finished")
                    && row.pointer("/fact/outcome").and_then(|v| v.as_str()) == Some("accepted")),
            "fallback must complete the request: {facts:?}"
        );
    }
}

const FAILED_STREAM: &[u8] = b"event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"failed-stream\",\"model\":\"runtime-native\",\"status\":\"failed\",\"output\":[],\"error\":{\"type\":\"server_error\",\"message\":\"retry\"},\"usage\":{\"input_tokens\":7,\"output_tokens\":2}}}\n\n";

#[test]
fn streaming_failure_after_delivered_body_stays_failed_and_retains_usage() {
    use std::io::Read;
    let early = b"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"failed-stream\",\"model\":\"runtime-native\"}}\n\n";
    let first = NativeProvider::start(vec![ProviderReply::StreamDrip {
        status: 200,
        chunks: vec![early.to_vec(), FAILED_STREAM.to_vec()],
        interval: Duration::from_secs(1),
    }]);
    let fallback = NativeProvider::start(vec![ProviderReply::StreamComplete {
        status: 200,
        body: RESPONSES_STREAM_OK,
    }]);
    let fixture = RuntimeFixture::launch_classified_with_observation(
        &[&first, &fallback],
        2,
        ObservationFaults::healthy(),
    );
    let mut stream = open_request(
        fixture.address,
        "POST",
        "/v1/responses",
        &[("X-HiRoute-Token", "runtime-token")],
        br#"{"model":"runtime-model","input":"hello","stream":true}"#,
    );
    stream
        .set_read_timeout(Some(Duration::from_millis(800)))
        .unwrap();
    let mut received = Vec::new();
    let mut chunk = [0u8; 4096];
    // Native model identifiers are rewritten to the served alias. Match the
    // delivered event, not the provider's pre-projection JSON bytes.
    let first_body_marker = b"event: response.created\n";
    while !received
        .windows(first_body_marker.len())
        .any(|part| part == first_body_marker)
    {
        let count = stream
            .read(&mut chunk)
            .expect("first body must be delivered before failure");
        assert!(count > 0);
        received.extend_from_slice(&chunk[..count]);
    }
    stream
        .set_read_timeout(Some(Duration::from_secs(5)))
        .unwrap();
    stream.read_to_end(&mut received).unwrap();
    assert!(String::from_utf8_lossy(&received).contains("response.failed"));
    assert_eq!(first.calls(), 1);
    assert_eq!(
        fallback.calls(),
        0,
        "body delivery closes transparent relay"
    );
    let facts = wait_execution_facts(&fixture, 1);
    assert!(
        facts
            .iter()
            .any(|row| row.pointer("/fact/kind").and_then(|v| v.as_str())
                == Some("agent_turn_finished")
                && row.pointer("/fact/status").and_then(|v| v.as_str()) == Some("failed")),
        "a delivered model failure must not complete the Agent turn: {facts:?}"
    );
    for (kind, outcome) in [
        ("request_finished", "failed"),
        ("attempt_finished", "rejected"),
    ] {
        assert!(
            facts
                .iter()
                .any(
                    |row| row.pointer("/fact/kind").and_then(|v| v.as_str()) == Some(kind)
                        && row.pointer("/fact/outcome").and_then(|v| v.as_str()) == Some(outcome)
                ),
            "transport completion must retain model failure: {facts:?}"
        );
    }
    assert!(
        facts
            .iter()
            .any(|row| row.pointer("/fact/kind").and_then(|v| v.as_str())
                == Some("usage_and_cache")
                && row.pointer("/fact/input_tokens").and_then(|v| v.as_u64()) == Some(7)
                && row.pointer("/fact/output_tokens").and_then(|v| v.as_u64()) == Some(2)),
        "failed stream must retain reported usage: {facts:?}"
    );
}

#[test]
fn successful_stream_forwards_unknown_events_before_any_semantic_output() {
    use std::io::Read;
    let unknown = b"event: response.vendor_extension\ndata: {\"opaque\":\"early-event\"}\n\n";
    let provider = NativeProvider::start(vec![ProviderReply::StreamDrip {
        status: 200,
        chunks: vec![unknown.to_vec(), RESPONSES_STREAM_OK.to_vec()],
        interval: Duration::from_secs(5),
    }]);
    let fixture = RuntimeFixture::launch(&[&provider], 1);
    let mut stream = open_request(
        fixture.address,
        "POST",
        "/v1/responses",
        &[("X-HiRoute-Token", "runtime-token")],
        br#"{"model":"runtime-model","input":"hello","stream":true}"#,
    );
    stream
        .set_read_timeout(Some(Duration::from_secs(2)))
        .unwrap();
    let mut received = Vec::new();
    let mut chunk = [0u8; 4096];
    while !received.windows(unknown.len()).any(|part| part == unknown) {
        let count = stream
            .read(&mut chunk)
            .expect("unknown event must arrive before the provider resumes");
        assert!(count > 0, "stream ended before its unknown event");
        received.extend_from_slice(&chunk[..count]);
    }
    assert!(received.starts_with(b"HTTP/1.1 200"));
    assert_eq!(provider.calls(), 1);
}

#[test]
fn real_listener_preserves_native_terminal_tail_split_across_provider_writes() {
    let tail = b"event: response.vendor_extension\ndata: opaque\n\n";
    let split = tail.len() / 2;
    let mut first = RESPONSES_STREAM_OK.to_vec();
    first.extend_from_slice(&tail[..split]);
    let provider = NativeProvider::start(vec![ProviderReply::StreamDrip {
        status: 200,
        chunks: vec![first, tail[split..].to_vec()],
        interval: Duration::from_millis(120),
    }]);
    let fixture = RuntimeFixture::launch(&[&provider], 1);
    let response =
        fixture.request_body(br#"{"model":"runtime-model","input":"hello","stream":true}"#);

    assert_eq!(response.status, 200);
    assert_eq!(provider.calls(), 1);
    assert!(response.body.windows(tail.len()).any(|part| part == tail));
    assert!(
        String::from_utf8_lossy(&response.body).contains("event: response.completed"),
        "{}",
        String::from_utf8_lossy(&response.body)
    );
}

#[test]
fn cross_protocol_candidate_uses_native_decoder_and_ingress_renderer() {
    let provider = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: br#"{"id":"chat-native","object":"chat.completion","model":"native-chat","choices":[{"index":0,"message":{"role":"assistant","content":"cross-protocol"},"finish_reason":"stop","logprobs":null}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#,
    }]);
    let fixture = RuntimeFixture::launch_with_publication_candidates(
        &[&provider],
        1,
        Some(&[PublicationCandidate {
            provider_index: 0,
            upstream_protocol: "chat_completions",
            statically_enabled: true,
        }]),
    );

    let response = fixture.request();
    assert_eq!(response.status, 200);
    let document: serde_json::Value = serde_json::from_slice(&response.body).unwrap();
    assert_eq!(document["object"], "response");
    assert_eq!(
        document["output"][0]["content"][0]["text"],
        "cross-protocol"
    );
    assert!(
        String::from_utf8_lossy(&provider.requests()[0])
            .to_ascii_lowercase()
            .contains("post /v1/chat/completions http/1.1")
    );
}

#[test]
fn dns_failure_does_not_create_or_consume_the_only_attempt() {
    let dns_candidate = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let reachable = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let fixture = RuntimeFixture::launch_with_endpoints(
        &[&dns_candidate, &reachable],
        1,
        &[
            "http://does-not-resolve.invalid:9".into(),
            format!("http://localhost:{}", reachable.address().port()),
        ],
    );

    assert_eq!(fixture.request().status, 200);
    assert_eq!(dns_candidate.calls(), 0);
    assert_eq!(reachable.calls(), 1);
}

#[test]
fn real_listener_401_and_403_disable_only_the_exact_credential_key() {
    for status in [401, 403] {
        let primary = NativeProvider::start(vec![
            ProviderReply::Complete {
                status,
                error_kind: None,
                body: AUTH_ERROR,
            },
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
        ]);
        let forbidden_fallback = NativeProvider::start(vec![ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: FALLBACK_OK,
        }]);
        let fixture =
            RuntimeFixture::launch_with_key_counts(&[&primary, &forbidden_fallback], 3, &[2, 1]);

        let first = fixture.request();
        assert_eq!(first.status, 200, "status={status}");
        assert_eq!(primary.calls(), 2, "status={status}");
        assert_eq!(forbidden_fallback.calls(), 0, "status={status}");
        assert!(!String::from_utf8_lossy(&first.body).contains("private auth marker"));

        let second = fixture.request();
        assert_eq!(second.status, 200, "status={status}");
        assert_eq!(primary.calls(), 3, "status={status}");
        assert_eq!(forbidden_fallback.calls(), 0, "status={status}");
        let requests = primary.requests();
        assert_eq!(
            wire_header(&requests[0], "authorization").as_deref(),
            Some("Bearer provider-secret-1-1")
        );
        assert_eq!(
            wire_header(&requests[1], "authorization").as_deref(),
            Some("Bearer provider-secret-1-2")
        );
        assert_eq!(
            wire_header(&requests[2], "authorization").as_deref(),
            Some("Bearer provider-secret-1-2"),
            "the disabled first key must remain skipped on the next request"
        );
    }
}

#[test]
fn real_listener_distinguishes_quota_key_scope_from_overload_binding_scope() {
    hiroute_e2e::p0_execution_receipt!(
        "runtime.key_handoff",
        [
            "runtime.multiple_keys",
            "runtime.quota_key_handoff",
            "runtime.binding_overload"
        ]
    );
    struct Case {
        name: &'static str,
        primary_replies: Vec<ProviderReply>,
        max_attempts: u32,
        primary_after_first: usize,
    }

    for case in [
        Case {
            name: "quota",
            primary_replies: vec![
                ProviderReply::Complete {
                    status: 429,
                    error_kind: None,
                    body: QUOTA_ERROR,
                },
                ProviderReply::Complete {
                    status: 429,
                    error_kind: None,
                    body: QUOTA_ERROR,
                },
            ],
            max_attempts: 3,
            primary_after_first: 2,
        },
        Case {
            name: "binding-overload",
            primary_replies: vec![ProviderReply::Complete {
                status: 429,
                error_kind: None,
                body: OVERLOAD_ERROR,
            }],
            max_attempts: 2,
            primary_after_first: 1,
        },
    ] {
        let primary = NativeProvider::start(case.primary_replies);
        let fallback = NativeProvider::start(vec![
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: FALLBACK_OK,
            },
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: FALLBACK_OK,
            },
        ]);
        let fixture = RuntimeFixture::launch_with_key_counts(
            &[&primary, &fallback],
            case.max_attempts,
            &[2, 1],
        );

        let first = fixture.request();
        assert_eq!(first.status, 200, "case={}", case.name);
        assert_eq!(
            primary.calls(),
            case.primary_after_first,
            "case={}",
            case.name
        );
        assert_eq!(fallback.calls(), 1, "case={}", case.name);
        assert!(String::from_utf8_lossy(&first.body).contains("accepted-fallback"));

        let second = fixture.request();
        assert_eq!(second.status, 200, "case={}", case.name);
        assert_eq!(
            primary.calls(),
            case.primary_after_first,
            "case={} must remain excluded by its exact state scope",
            case.name
        );
        assert_eq!(fallback.calls(), 2, "case={}", case.name);

        let requests = primary.requests();
        assert_eq!(
            wire_header(&requests[0], "authorization").as_deref(),
            Some("Bearer provider-secret-1-1")
        );
        if case.name == "quota" {
            assert_eq!(
                wire_header(&requests[1], "authorization").as_deref(),
                Some("Bearer provider-secret-1-2"),
                "quota must cool one key and consume the next exact key before candidate fallback"
            );
        }
    }
}

#[test]
fn real_listener_all_prebody_400_and_422_relay_without_quarantining() {
    struct Case {
        name: &'static str,
        status: u16,
        body: &'static [u8],
        second_primary_reply: ProviderReply,
        expected_downstream: u16,
        expected_fallback_calls: usize,
    }

    for case in [
        Case {
            name: "reasoning-budget-400",
            status: 400,
            body: br#"{"error":{"code":"InvalidParameter","message":"<400> InternalError.Algo.InvalidParameter: The thinking_budget parameter must be a positive integer and not greater than 131072"}}"#,
            second_primary_reply: ProviderReply::Complete { status: 200, error_kind: None, body: RESPONSES_OK },
            expected_downstream: 200,
            expected_fallback_calls: 1,
        },
        Case {
            name: "other-invalid-parameter-400",
            status: 400,
            body: br#"{"error":{"code":"InvalidParameter","message":"private invalid tool schema"}}"#,
            second_primary_reply: ProviderReply::Complete { status: 200, error_kind: None, body: RESPONSES_OK },
            expected_downstream: 200,
            expected_fallback_calls: 1,
        },
        Case {
            name: "protocol-400",
            status: 400,
            body: PROTOCOL_ERROR,
            second_primary_reply: ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
            expected_downstream: 200,
            expected_fallback_calls: 1,
        },
        Case {
            name: "permanent-422",
            status: 422,
            body: PERMANENT_ERROR,
            second_primary_reply: ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
            expected_downstream: 200,
            expected_fallback_calls: 1,
        },
    ] {
        let primary = NativeProvider::start(vec![
            ProviderReply::Complete {
                status: case.status,
                error_kind: None,
                body: case.body,
            },
            case.second_primary_reply,
        ]);
        let fallback = NativeProvider::start(vec![ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: FALLBACK_OK,
        }]);
        let fixture = RuntimeFixture::launch(&[&primary, &fallback], 2);

        let first = fixture.request();
        assert_eq!(first.status, case.expected_downstream, "case={}", case.name);
        assert_eq!(primary.calls(), 1, "case={}", case.name);
        assert_eq!(
            fallback.calls(),
            case.expected_fallback_calls,
            "case={}",
            case.name
        );
        assert!(!String::from_utf8_lossy(&first.body).contains("private"));

        let second = fixture
            .request_body(br#"{"model":"runtime-model","input":"hello-rebuilt","stream":false}"#);
        assert_eq!(
            second.status, case.expected_downstream,
            "case={}",
            case.name
        );
        assert_eq!(
            primary.calls(),
            2,
            "case={} must not persist protocol/permanent failures into RuntimeState",
            case.name
        );
        assert_eq!(
            fallback.calls(),
            case.expected_fallback_calls,
            "case={}",
            case.name
        );
    }
}

#[test]
fn real_listener_first_byte_timeout_cools_binding_and_falls_back() {
    let stalled = NativeProvider::start(vec![ProviderReply::Stall {
        duration: Duration::from_millis(500),
    }]);
    let fallback = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: FALLBACK_OK,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: FALLBACK_OK,
        },
    ]);
    let fixture = RuntimeFixture::launch_with_attempt_timeout(
        &[&stalled, &fallback],
        2,
        Duration::from_millis(200),
    );

    let started = Instant::now();
    let first = fixture.request();
    assert_eq!(first.status, 200);
    assert!(started.elapsed() >= Duration::from_millis(150));
    assert_eq!(stalled.calls(), 1);
    assert_eq!(fallback.calls(), 1);

    let second = fixture.request();
    assert_eq!(second.status, 200);
    assert_eq!(stalled.calls(), 1, "timed out binding must be cooling down");
    assert_eq!(fallback.calls(), 2);
}

#[test]
fn real_listener_candidate_exhaustion_returns_a_sanitized_gateway_failure() {
    let first = NativeProvider::start(vec![ProviderReply::Complete {
        status: 503,
        error_kind: None,
        body: b"private exhausted first",
    }]);
    let second = NativeProvider::start(vec![ProviderReply::Complete {
        status: 503,
        error_kind: None,
        body: b"private exhausted second",
    }]);
    let fixture = RuntimeFixture::launch(&[&first, &second], 2);

    let response = fixture.request();

    assert_eq!(response.status, 503);
    assert_eq!(first.calls(), 1);
    assert_eq!(second.calls(), 1);
    assert_eq!(
        serde_json::from_slice::<serde_json::Value>(&response.body).unwrap(),
        serde_json::json!({
            "error": {
                "code": "UPSTREAM_ATTEMPT_FAILED",
                "type": "upstream_error",
            }
        })
    );
}

#[test]
fn real_listener_reports_cooling_recovers_once_and_resets_the_backoff_step() {
    let provider = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 503,
            error_kind: None,
            body: b"private transient failure",
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
        ProviderReply::Complete {
            status: 503,
            error_kind: None,
            body: b"private transient failure after recovery",
        },
    ]);
    let fixture = RuntimeFixture::launch(&[&provider], 1);

    assert_eq!(fixture.request().status, 503, "seed the binding cooldown");
    let response = fixture.request();

    assert_eq!(response.status, 503);
    assert_eq!(
        provider.calls(),
        1,
        "cooling candidates must not reach upstream"
    );
    let retry_after = response.headers["retry-after"].parse::<u64>().unwrap();
    assert!((1..=2).contains(&retry_after));
    assert_eq!(
        serde_json::from_slice::<serde_json::Value>(&response.body).unwrap(),
        serde_json::json!({
            "schema_version": "hiroute.gateway.error/v1",
            "code": "CANDIDATES_COOLING_DOWN",
            "phase": "runtime_state",
            "retry_after_seconds": retry_after,
        })
    );

    std::thread::sleep(Duration::from_millis(2_100));
    assert_eq!(
        fixture.request().status,
        200,
        "the sole recovery probe wins"
    );
    assert_eq!(provider.calls(), 2);

    assert_eq!(
        fixture.request().status,
        503,
        "a new transient failure follows the successful probe"
    );
    let reset_response = fixture.request();
    assert_eq!(
        provider.calls(),
        3,
        "the reset cooldown blocks upstream I/O"
    );
    let reset_retry_after = reset_response.headers["retry-after"]
        .parse::<u64>()
        .unwrap();
    assert!(
        (1..=2).contains(&reset_retry_after),
        "successful recovery must reset the sequence to the two-second first step"
    );
}

#[test]
fn real_listener_rest_classifier_uses_the_existing_gateway_and_selects_complex() {
    let simple = NativeProvider::start(Vec::new());
    let complex = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let classifier = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: CLASSIFIER_COMPLEX,
    }]);
    let fixture = RuntimeFixture::launch_rest_classified_with_observation(
        &[&simple, &complex, &classifier],
        2,
        ObservationFaults::healthy(),
    );

    let response = fixture
        .request_body(br#"{"model":"runtime-model","input":"latest-only-marker","stream":false}"#);

    assert_eq!(
        response.status,
        200,
        "body={}, calls={:?}",
        String::from_utf8_lossy(&response.body),
        (simple.calls(), complex.calls(), classifier.calls())
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (0, 1, 1)
    );
    let requests = classifier.requests();
    assert_eq!(
        wire_header(&requests[0], "authorization"),
        Some("Bearer provider-secret-3".into())
    );
    assert!(wire_header(&requests[0], "x-hiroute-token").is_none());
    assert!(String::from_utf8_lossy(&requests[0]).starts_with("POST /v1/decisions HTTP/1.1"));
    let split = requests[0]
        .windows(4)
        .position(|window| window == b"\r\n\r\n")
        .unwrap();
    let body: serde_json::Value = serde_json::from_slice(&requests[0][split + 4..]).unwrap();
    assert_eq!(
        body.pointer("/latest_user/0/text")
            .and_then(|value| value.as_str()),
        Some("latest-only-marker")
    );
    assert_eq!(body["visible_conversation"], serde_json::json!([]));
    assert_eq!(
        body["route"],
        serde_json::json!({"kind":"smart_saving","groups":["regular","primary"]})
    );
    assert!(body.get("decision").is_none());
    assert!(body.get("definition").is_none());
    assert!(body["preceding_stage"].is_null());
    assert_eq!(body.as_object().unwrap().len(), 5);
    assert!(body.get("model").is_none());

    let facts = wait_execution_facts(&fixture, 1);
    let main_request_id = facts
        .iter()
        .find(|fact| {
            fact.get("served_model_id")
                .and_then(serde_json::Value::as_str)
                == Some("runtime-model")
        })
        .and_then(|fact| fact.pointer("/correlation/request_id"))
        .and_then(serde_json::Value::as_str)
        .unwrap()
        .to_owned();
    assert!(facts.iter().any(|fact| {
        fact.pointer("/correlation/request_id")
            .and_then(serde_json::Value::as_str)
            == Some(main_request_id.as_str())
            && fact
                .pointer("/fact/complexity/decision_source")
                .and_then(serde_json::Value::as_str)
                == Some("external_classifier")
    }));
    let content = wait_content_for_request(&fixture, &main_request_id);
    assert!(content.iter().any(|record| {
        record
            .pointer("/correlation/request_id")
            .and_then(serde_json::Value::as_str)
            == Some(main_request_id.as_str())
    }));
}

#[test]
fn real_listener_rest_classifier_runs_once_for_each_supported_ingress() {
    let simple = NativeProvider::start(Vec::new());
    let complex = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
    ]);
    let classifier = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_COMPLEX,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_COMPLEX,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_COMPLEX,
        },
    ]);
    let fixture = RuntimeFixture::launch_rest_classified(&[&simple, &complex, &classifier], 2);
    let cases = [
        (
            "/v1/responses",
            ("X-HiRoute-Token", TOKEN),
            serde_json::json!({
                "model": "runtime-model",
                "input": "responses classifier input",
                "stream": false,
            }),
            "responses classifier input",
        ),
        (
            "/v1/chat/completions",
            ("Authorization", "Bearer runtime-chat-token"),
            serde_json::json!({
                "model": "runtime-model",
                "messages": [{"role":"user","content":"chat classifier input"}],
                "stream": false,
            }),
            "chat classifier input",
        ),
        (
            "/v1/messages",
            ("Authorization", "Bearer runtime-messages-token"),
            serde_json::json!({
                "model": "runtime-model",
                "max_tokens": 8,
                "messages": [{"role":"user","content":"messages classifier input"}],
                "stream": false,
            }),
            "messages classifier input",
        ),
    ];

    for (path, header, body, _) in &cases {
        let response = request(
            fixture.address,
            "POST",
            path,
            &[*header],
            &serde_json::to_vec(body).unwrap(),
        );
        assert_eq!(
            response.status,
            200,
            "{path}: {}",
            String::from_utf8_lossy(&response.body)
        );
    }

    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (0, 3, 3)
    );
    for (wire, (_, _, _, expected_latest)) in classifier.requests().iter().zip(cases) {
        let split = wire
            .windows(4)
            .position(|window| window == b"\r\n\r\n")
            .unwrap();
        let body: serde_json::Value = serde_json::from_slice(&wire[split + 4..]).unwrap();
        assert_eq!(
            body.pointer("/latest_user/0/text")
                .and_then(serde_json::Value::as_str),
            Some(expected_latest)
        );
    }
}

#[test]
fn real_listener_rest_classifier_streams_complete_replay_backed_context_without_truncation() {
    let simple = NativeProvider::start(Vec::new());
    let complex = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let classifier = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: CLASSIFIER_COMPLEX,
    }]);
    let fixture = RuntimeFixture::launch_rest_classified_with_replay(
        &[&simple, &complex, &classifier],
        2,
        256,
        128,
    );
    let instructions = "先前说明".repeat(2_048);
    let latest_user = format!(
        "{} literal-marker=__hiroute_content_ref_v2_0_0_1_1__",
        "分类上下文".repeat(1_024)
    );
    assert!(latest_user.chars().count() >= 4_096);
    let request_body = format!(
        r#"{{"model":"runtime-model","instructions":{},"input":{},"stream":false}}"#,
        serde_json::to_string(&instructions).unwrap(),
        serde_json::to_string(&latest_user).unwrap(),
    )
    .into_bytes();

    let response = fixture.request_body(&request_body);

    assert_eq!(
        response.status,
        200,
        "{}",
        String::from_utf8_lossy(&response.body)
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (0, 1, 1)
    );
    let requests = classifier.requests();
    let split = requests[0]
        .windows(4)
        .position(|window| window == b"\r\n\r\n")
        .unwrap();
    let body: serde_json::Value = serde_json::from_slice(&requests[0][split + 4..]).unwrap();
    assert_eq!(
        body.pointer("/latest_user/0/text"),
        Some(&serde_json::Value::String(latest_user.clone()))
    );
    assert_eq!(body["visible_conversation"], serde_json::json!([]));
    assert!(
        !serde_json::to_string(&body)
            .unwrap()
            .contains(&instructions)
    );
}

#[test]
fn real_listener_rest_classifier_can_select_the_simple_group() {
    let simple = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: CLASSIFIER_SIMPLE,
    }]);
    let fixture = RuntimeFixture::launch_rest_classified(&[&simple, &complex, &classifier], 2);

    let response = fixture.request();

    assert_eq!(response.status, 200);
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 0, 1)
    );
}

#[test]
fn real_listener_rest_classifier_reclassifies_each_decision_boundary() {
    let simple = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
    ]);
    let complex = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
    ]);
    let classifier = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_COMPLEX,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_SIMPLE,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_COMPLEX,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_SIMPLE,
        },
    ]);
    let fixture = RuntimeFixture::launch_rest_classified(&[&simple, &complex, &classifier], 2);
    let send = |input: serde_json::Value| {
        request(
            fixture.address,
            "POST",
            "/v1/responses",
            &[
                ("X-HiRoute-Token", "runtime-token"),
                ("session-id", "rest-classifier-inheritance"),
            ],
            &serde_json::to_vec(&serde_json::json!({
                "model": "runtime-model",
                "input": input,
                "stream": false,
            }))
            .unwrap(),
        )
    };
    let message = |role: &str, kind: &str, text: &str| {
        serde_json::json!({
            "type": "message",
            "role": role,
            "content": [{"type": kind, "text": text}],
        })
    };

    assert_eq!(
        send(serde_json::json!([message(
            "user",
            "input_text",
            "first task"
        )]))
        .status,
        200
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (0, 1, 1)
    );

    assert_eq!(
        send(serde_json::json!([
            message("user", "input_text", "first task"),
            message("assistant", "output_text", "ok"),
            message("user", "input_text", "continue the same task")
        ]))
        .status,
        200
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 1, 2),
        "an appended user message must not inherit the prior round's branch"
    );

    assert_eq!(
        send(serde_json::json!([message(
            "user",
            "input_text",
            "continue the same task"
        )]))
        .status,
        200
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 2, 3),
        "a rebuilt history must classify again even when latest_user is unchanged"
    );

    assert_eq!(
        send(serde_json::json!([message(
            "user",
            "input_text",
            "Summary: continue the same work"
        )]))
        .status,
        200
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (2, 2, 4),
        "a summary-shaped latest_user must use the same ContextHold decision boundary"
    );
}

#[test]
fn real_listener_context_hold_boundary_seals_unknown_without_reusing_old_competence() {
    let simple = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let complex = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_TOOL_CALL,
    }]);
    let classifier = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_COMPLEX,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_SIMPLE_WITH_ASSESSMENT,
        },
    ]);
    let fixture = RuntimeFixture::launch_rest_classified_with_observation(
        &[&simple, &complex, &classifier],
        2,
        ObservationFaults::healthy(),
    );
    let headers = [
        ("X-HiRoute-Token", "runtime-token"),
        ("session-id", "context-hold-decision-boundary"),
    ];
    let user = serde_json::json!({
        "type": "message",
        "role": "user",
        "content": [{"type": "input_text", "text": "look up the record"}],
    });
    let send = |rebuilt_history: bool| {
        let input = if rebuilt_history {
            vec![
                serde_json::json!({
                    "type":"message","role":"user",
                    "content":[{"type":"input_text","text":"A rebuilt earlier context"}]
                }),
                serde_json::json!({
                    "type":"message","role":"assistant",
                    "content":[{"type":"output_text","text":"ack"}]
                }),
                user.clone(),
            ]
        } else {
            vec![user.clone()]
        };
        let body = serde_json::json!({
            "model": "runtime-model",
            "input": input,
            "stream": false,
        });
        request(
            fixture.address,
            "POST",
            "/v1/responses",
            &headers,
            &serde_json::to_vec(&body).unwrap(),
        )
    };

    let first = send(false);
    assert_eq!(
        first.status,
        200,
        "{}",
        String::from_utf8_lossy(&first.body)
    );
    let second = send(true);
    assert_eq!(
        second.status,
        200,
        "{}",
        String::from_utf8_lossy(&second.body)
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 1, 2),
        "a rebuilt message history must use the newly selected business branch"
    );

    let classifier_requests = classifier.requests();
    let second_wire = &classifier_requests[1];
    let body_start = second_wire
        .windows(4)
        .position(|window| window == b"\r\n\r\n")
        .expect("classifier request head terminator")
        + 4;
    let body: serde_json::Value = serde_json::from_slice(&second_wire[body_start..]).unwrap();
    assert_eq!(
        body.pointer("/latest_user/0/text")
            .and_then(serde_json::Value::as_str),
        Some("look up the record"),
        "unchanged latest_user must not suppress a ContextHold boundary"
    );
    assert!(
        body["preceding_stage"].is_null(),
        "rebuilt context cannot grade an old stage"
    );
    assert_eq!(body["visible_conversation"].as_array().unwrap().len(), 1);
    assert_eq!(body["visible_conversation"][0]["status"], "unknown");
    assert_eq!(
        body["visible_conversation"][0]["steps"][0][0]["tool"],
        "tools.lookup"
    );
    assert_eq!(
        body["visible_conversation"][0]["steps"][0][0]["status"],
        "unknown"
    );

    let facts = wait_execution_facts(&fixture, 2);
    let rounds = facts
        .iter()
        .filter(|fact| {
            fact.pointer("/fact/kind")
                .and_then(serde_json::Value::as_str)
                == Some("agent_turn_finished")
        })
        .collect::<Vec<_>>();
    assert_eq!(rounds.len(), 2, "each actual execution round seals once");
    let prior = rounds
        .iter()
        .find(|fact| {
            fact.pointer("/fact/status")
                .and_then(serde_json::Value::as_str)
                == Some("unknown")
        })
        .expect("the ContextHold boundary preserves the prior unknown status");
    let current = rounds
        .iter()
        .find(|fact| {
            fact.pointer("/fact/status")
                .and_then(serde_json::Value::as_str)
                == Some("completed")
        })
        .expect("the newly selected branch completes its own round");
    assert_ne!(
        prior.pointer("/fact/segment_id"),
        current.pointer("/fact/segment_id")
    );
    assert!(
        facts.iter().all(|fact| fact.pointer("/fact/kind")
            != Some(&serde_json::json!("branch_assessment_recorded"))),
        "an unsolicited response score without an eligible target must be ignored"
    );
}

#[test]
fn real_listener_rest_classifier_sends_and_persists_the_previous_segment_assessment() {
    let simple = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
    ]);
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_SIMPLE,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: CLASSIFIER_SIMPLE_WITH_ASSESSMENT,
        },
    ]);
    let fixture = RuntimeFixture::launch_rest_classified_with_observation(
        &[&simple, &complex, &classifier],
        2,
        ObservationFaults::healthy(),
    );
    let headers = [
        ("X-HiRoute-Token", "runtime-token"),
        ("session-id", "rest-classifier-assessment"),
    ];
    let message = |role: &str, kind: &str, text: &str| {
        serde_json::json!({
            "type": "message",
            "role": role,
            "content": [{"type": kind, "text": text}],
        })
    };
    let first_user = message("user", "input_text", "Correct one typo.");
    let send = |input: serde_json::Value| {
        request(
            fixture.address,
            "POST",
            "/v1/responses",
            &headers,
            &serde_json::to_vec(&serde_json::json!({
                "model": "runtime-model",
                "input": input,
                "stream": false,
            }))
            .unwrap(),
        )
    };

    assert_eq!(send(serde_json::json!([first_user.clone()])).status, 200);
    assert_eq!(
        send(serde_json::json!([
            first_user,
            message("assistant", "output_text", "Corrected it."),
            message("user", "input_text", "That worked. Correct another typo.")
        ]))
        .status,
        200
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (2, 0, 2)
    );

    let classifier_requests = classifier.requests();
    let second_wire = &classifier_requests[1];
    let body_start = second_wire
        .windows(4)
        .position(|window| window == b"\r\n\r\n")
        .expect("classifier request head terminator")
        + 4;
    let body: serde_json::Value = serde_json::from_slice(&second_wire[body_start..]).unwrap();
    assert_eq!(body.as_object().unwrap().len(), 5);
    assert_eq!(body["preceding_stage"]["from"], 0);
    assert_eq!(body["visible_conversation"].as_array().unwrap().len(), 1);
    assert_eq!(
        body["visible_conversation"][0]["steps"][0][0]["text"], "ok",
        "history must use accepted output, not the client's rewritten assistant text"
    );

    let facts = wait_execution_facts(&fixture, 2);
    let assessment = facts
        .iter()
        .find(|fact| {
            fact.pointer("/fact/kind")
                .and_then(serde_json::Value::as_str)
                == Some("branch_assessment_recorded")
        })
        .expect("a valid second-turn assessment must be persisted");
    assert_eq!(
        assessment
            .pointer("/fact/score")
            .and_then(|value| value.as_f64()),
        Some(0.0)
    );
    let second_decision = facts
        .iter()
        .rev()
        .find(|fact| fact.pointer("/fact/complexity").is_some())
        .expect("second classification fact");
    assert_eq!(
        second_decision
            .pointer("/fact/complexity/decision_source")
            .and_then(serde_json::Value::as_str),
        Some("external_classifier")
    );
    assert_eq!(
        second_decision
            .pointer("/fact/complexity/fallback_used")
            .and_then(serde_json::Value::as_bool),
        Some(false)
    );
}

#[test]
fn real_listener_rest_classifier_freezes_the_branch_for_a_tool_continuation() {
    let simple = NativeProvider::start(Vec::new());
    let complex = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_TOOL_CALL,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
    ]);
    let classifier = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: CLASSIFIER_COMPLEX,
    }]);
    let fixture = RuntimeFixture::launch_rest_classified(&[&simple, &complex, &classifier], 2);
    let headers = [
        ("X-HiRoute-Token", "runtime-token"),
        ("session-id", "rest-classifier-tool-continuation"),
    ];
    let user = serde_json::json!({
        "type": "message",
        "role": "user",
        "content": [{"type": "input_text", "text": "look up the record"}],
    });
    let first = request(
        fixture.address,
        "POST",
        "/v1/responses",
        &headers,
        &serde_json::to_vec(&serde_json::json!({
            "model": "runtime-model",
            "input": [user.clone()],
            "stream": false,
        }))
        .unwrap(),
    );
    assert_eq!(
        first.status,
        200,
        "{}",
        String::from_utf8_lossy(&first.body)
    );
    let first_body: serde_json::Value = serde_json::from_slice(&first.body).unwrap();
    let logical_call_id = first_body["output"][0]["call_id"]
        .as_str()
        .expect("Gateway response exposes the logical tool call ID");

    let second = request(
        fixture.address,
        "POST",
        "/v1/responses",
        &headers,
        &serde_json::to_vec(&serde_json::json!({
            "model": "runtime-model",
            "input": [
                user,
                {"type":"function_call","call_id":logical_call_id,"namespace":"tools","name":"lookup","arguments":"{}"},
                {"type":"function_call_output","call_id":logical_call_id,"output":"found"}
            ],
            "stream": false,
        }))
        .unwrap(),
    );
    assert_eq!(
        second.status,
        200,
        "{}",
        String::from_utf8_lossy(&second.body)
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (0, 2, 1),
        "a tool continuation must keep the turn's frozen branch without another REST decision"
    );
}

#[test]
#[ignore = "explicit paid smoke: requires a locally running trusted classifier service"]
fn live_hiroute_to_jev_smoke_selects_and_records_an_assessment() {
    let endpoint = std::env::var("HIROUTE_LIVE_CLASSIFIER_ENDPOINT")
        .expect("set HIROUTE_LIVE_CLASSIFIER_ENDPOINT to the local Jev service /v1/decisions URL");
    let replies = || {
        vec![
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
            ProviderReply::Complete {
                status: 200,
                error_kind: None,
                body: RESPONSES_OK,
            },
        ]
    };
    let simple = NativeProvider::start(replies());
    let complex = NativeProvider::start(replies());
    let unused_classifier = NativeProvider::start(Vec::new());
    let fixture = RuntimeFixture::launch_rest_classified_at(
        &[&simple, &complex, &unused_classifier],
        2,
        &endpoint,
    );
    let headers = [
        ("X-HiRoute-Token", "runtime-token"),
        ("session-id", "live-jev-smoke"),
    ];
    let message = |role: &str, kind: &str, text: &str| {
        serde_json::json!({
            "type": "message",
            "role": role,
            "content": [{"type": kind, "text": text}],
        })
    };
    let first_user = message(
        "user",
        "input_text",
        "Correct the typo 'recieve' in README.",
    );
    let first = request(
        fixture.address,
        "POST",
        "/v1/responses",
        &headers,
        &serde_json::to_vec(&serde_json::json!({
            "model": "runtime-model",
            "input": [first_user.clone()],
            "stream": false,
        }))
        .unwrap(),
    );
    assert_eq!(
        first.status,
        200,
        "{}",
        String::from_utf8_lossy(&first.body)
    );

    let second = request(
        fixture.address,
        "POST",
        "/v1/responses",
        &headers,
        &serde_json::to_vec(&serde_json::json!({
            "model": "runtime-model",
            "input": [
                first_user,
                message("assistant", "output_text", "Fixed the typo."),
                message("user", "input_text", "That worked. Correct the typo 'seperate' too.")
            ],
            "stream": false,
        }))
        .unwrap(),
    );
    assert_eq!(
        second.status,
        200,
        "{}",
        String::from_utf8_lossy(&second.body)
    );
    assert_eq!(unused_classifier.calls(), 0);
    assert_eq!(simple.calls() + complex.calls(), 2);
    let facts = wait_execution_facts(&fixture, 2);
    let decisions = facts
        .iter()
        .filter_map(|fact| fact.pointer("/fact/complexity"))
        .collect::<Vec<_>>();
    assert_eq!(
        decisions.len(),
        2,
        "both turns must record a classification"
    );
    assert!(
        decisions.iter().all(|decision| {
            decision
                .get("decision_source")
                .and_then(serde_json::Value::as_str)
                == Some("external_classifier")
                && decision
                    .get("fallback_used")
                    .and_then(serde_json::Value::as_bool)
                    == Some(false)
        }),
        "both Jev decisions must be accepted without local fallback: {decisions:?}"
    );
    let assessment = facts
        .iter()
        .find(|fact| {
            fact.pointer("/fact/kind")
                .and_then(serde_json::Value::as_str)
                == Some("branch_assessment_recorded")
        })
        .expect("the second decision must persist Jev's optional competence assessment");
    let score = assessment
        .pointer("/fact/score")
        .and_then(serde_json::Value::as_f64)
        .expect("assessment score");
    assert!((0.0..=1.0).contains(&score));
}

#[test]
fn real_listener_rejects_the_removed_llm_envelope_and_falls_back_to_local_rules() {
    let simple = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: CLASSIFIER_INVALID,
    }]);
    let fixture = RuntimeFixture::launch_rest_classified(&[&simple, &complex, &classifier], 2);

    let response = fixture.request();

    assert_eq!(response.status, 200);
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 0, 1),
        "invalid classifier output must use the local simple decision"
    );
}

#[test]
fn real_listener_unavailable_rest_classifier_falls_back_to_local_rules() {
    let simple = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![ProviderReply::Complete {
        status: 503,
        error_kind: None,
        body: b"classifier unavailable",
    }]);
    let fixture = RuntimeFixture::launch_rest_classified(&[&simple, &complex, &classifier], 2);

    let response = fixture.request();

    assert_eq!(response.status, 200);
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 0, 1),
        "an unavailable classifier must use exactly one local-rule fallback"
    );
}

#[test]
fn real_listener_rejected_classifier_input_falls_back_once_and_records_reason() {
    let simple = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![ProviderReply::Complete {
        status: 413,
        error_kind: None,
        body: b"classifier input rejected",
    }]);
    let fixture = RuntimeFixture::launch_rest_classified_with_observation(
        &[&simple, &complex, &classifier],
        2,
        ObservationFaults::healthy(),
    );

    let response = fixture.request();

    assert_eq!(response.status, 200);
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 0, 1)
    );
    let facts = wait_execution_facts(&fixture, 1);
    assert!(facts.iter().any(|fact| {
        fact.pointer("/fact/complexity/fallback_used")
            .and_then(serde_json::Value::as_bool)
            == Some(true)
            && fact
                .pointer("/fact/complexity/fallback_reason")
                .and_then(serde_json::Value::as_str)
                == Some("rejected_input")
    }));
}

#[test]
fn real_listener_classifier_timeout_falls_back_once_within_the_source_deadline() {
    let simple = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: RESPONSES_OK,
    }]);
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![ProviderReply::Stall {
        duration: Duration::from_secs(4),
    }]);
    let fixture = RuntimeFixture::launch_rest_classified_with_classifier_timeout(
        &[&simple, &complex, &classifier],
        2,
        Duration::from_millis(300),
    );

    let started = Instant::now();
    let response = fixture.request();

    assert_eq!(response.status, 200);
    assert!(started.elapsed() >= Duration::from_millis(250));
    assert!(started.elapsed() < Duration::from_secs(2));
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (1, 0, 1)
    );
}

#[test]
fn real_listener_source_deadline_during_rest_classification_never_falls_back() {
    let simple = NativeProvider::start(Vec::new());
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![ProviderReply::Stall {
        duration: Duration::from_secs(2),
    }]);
    let fixture = RuntimeFixture::launch_rest_classified_with_overall_timeout(
        &[&simple, &complex, &classifier],
        2,
        Duration::from_millis(500),
    );

    let started = Instant::now();
    let response = fixture.request();

    assert_eq!(response.status, 408);
    assert!(started.elapsed() < Duration::from_secs(2));
    assert_eq!(
        serde_json::from_slice::<serde_json::Value>(&response.body).unwrap(),
        serde_json::json!({
            "schema_version": "hiroute.gateway.error/v1",
            "code": "REQUEST_DEADLINE_EXCEEDED",
            "phase": "request_authority",
        })
    );
    assert_eq!(
        (simple.calls(), complex.calls(), classifier.calls()),
        (0, 0, 1)
    );
}

#[test]
fn real_listener_disconnect_during_rest_classification_never_runs_the_main_model() {
    let simple = NativeProvider::start(Vec::new());
    let complex = NativeProvider::start(Vec::new());
    let classifier = NativeProvider::start(vec![ProviderReply::Stall {
        duration: Duration::from_secs(4),
    }]);
    let fixture = RuntimeFixture::launch_rest_classified(&[&simple, &complex, &classifier], 2);
    let downstream = open_request(
        fixture.address,
        "POST",
        "/v1/responses",
        &[("X-HiRoute-Token", "runtime-token")],
        br#"{"model":"runtime-model","input":"cancel me","stream":false}"#,
    );
    let deadline = Instant::now() + Duration::from_secs(2);
    while classifier.calls() == 0 && Instant::now() < deadline {
        std::thread::sleep(Duration::from_millis(5));
    }
    assert_eq!(
        classifier.calls(),
        1,
        "classification call must be in flight"
    );
    drop(downstream);

    std::thread::sleep(Duration::from_millis(3_250));
    assert_eq!(
        (simple.calls(), complex.calls()),
        (0, 0),
        "disconnect must not run a rule fallback or main-model attempt"
    );
    assert_eq!(
        request(fixture.address, "GET", "/_hiroute/ready", &[], b"").status,
        200
    );
}

#[test]
fn real_listener_runtime_state_cas_write_failure_is_fail_closed_and_one_shot() {
    hiroute_e2e::p0_execution_receipt!(
        "runtime.cas_write_fault",
        [
            "runtime.cas_write_fault",
            "runtime.fail_closed_503",
            "runtime.zero_fallback_after_fault",
            "runtime.zero_bad_state_persist",
        ]
    );
    let primary = NativeProvider::start(vec![
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
        ProviderReply::Complete {
            status: 429,
            error_kind: None,
            body: QUOTA_ERROR,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
    ]);
    let fallback = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: FALLBACK_OK,
    }]);
    let fixture = RuntimeFixture::launch_with_runtime_state_control(&[&primary, &fallback], 2);

    let invalid = fixture.arm_runtime_state_fault("invalid-cas-count", "compare_and_swap_exact", 0);
    assert_fault_control_nack(&invalid);
    assert_eq!(
        fixture.request().status,
        200,
        "invalid command must not arm a fault"
    );
    assert_eq!((primary.calls(), fallback.calls()), (1, 0));

    let armed = fixture.arm_runtime_state_fault("arm-cas-once", "compare_and_swap_exact", 1);
    assert_fault_control_ack(&armed, "compare_and_swap_exact", 1);
    assert_runtime_state_503(&fixture.request());
    assert_eq!(primary.calls(), 2);
    assert_eq!(fallback.calls(), 0, "failed CAS must close before relay");

    assert_eq!(fixture.request().status, 200);
    assert_eq!(
        primary.calls(),
        3,
        "the failed CAS must not persist cooldown"
    );
    assert_eq!(fallback.calls(), 0);
    let facts = wait_execution_facts(&fixture, 3);
    assert_single_planner_run_per_request(&facts, 3);
    assert_runtime_state_outcomes(&facts, "compare_and_swap_exact", &["authority_error"]);
    let fault_request =
        request_id_for_runtime_state_outcome(&facts, "compare_and_swap_exact", "authority_error");
    assert_exact_attempt_lifecycle(&facts, &fault_request, 1, 1);
}

#[test]
fn real_listener_probe_write_failure_creates_no_attempt_and_recovers_once() {
    hiroute_e2e::p0_execution_receipt!(
        "runtime.probe_write_fault",
        [
            "runtime.probe_write_fault",
            "runtime.probe_zero_attempt",
            "runtime.probe_one_shot_recovery",
        ]
    );
    let primary = NativeProvider::start(vec![
        ProviderReply::CompleteWithRetryAfter {
            status: 429,
            retry_after_secs: 0,
            body: QUOTA_ERROR,
        },
        ProviderReply::Complete {
            status: 200,
            error_kind: None,
            body: RESPONSES_OK,
        },
    ]);
    let fallback = NativeProvider::start(vec![ProviderReply::Complete {
        status: 200,
        error_kind: None,
        body: FALLBACK_OK,
    }]);
    let fixture = RuntimeFixture::launch_with_runtime_state_control(&[&primary, &fallback], 1);

    assert_eq!(fixture.request().status, 502, "seed expired key cooldown");
    assert_eq!((primary.calls(), fallback.calls()), (1, 0));
    let armed = fixture.arm_runtime_state_fault("arm-probe-once", "acquire_probe_lease_exact", 1);
    assert_fault_control_ack(&armed, "acquire_probe_lease_exact", 1);

    assert_runtime_state_503(&fixture.request());
    assert_eq!(
        (primary.calls(), fallback.calls()),
        (1, 0),
        "a failed probe write must not create an Attempt or relay"
    );
    assert_eq!(fixture.request().status, 200);
    assert_eq!((primary.calls(), fallback.calls()), (2, 0));
    let facts = wait_execution_facts(&fixture, 3);
    assert_single_planner_run_per_request(&facts, 3);
    assert_runtime_state_outcomes(
        &facts,
        "acquire_probe_lease_exact",
        &["acquired", "authority_error"],
    );
    let fault_request = request_id_for_runtime_state_outcome(
        &facts,
        "acquire_probe_lease_exact",
        "authority_error",
    );
    assert_exact_attempt_lifecycle(&facts, &fault_request, 0, 0);
    let recovery_request =
        request_id_for_runtime_state_outcome(&facts, "acquire_probe_lease_exact", "acquired");
    assert_exact_attempt_lifecycle(&facts, &recovery_request, 1, 1);
}

#[test]
fn typed_failure_matrix_keeps_quota_binding_and_permanent_failures_distinct() {
    let profile = ConnectorErrorProfile::exact();
    let quota = classify_failure(
        &RawAttemptFailure::Http {
            status: 429,
            kind: Some(ProviderFailureKind::Quota),
            retry_after: None,
        },
        profile,
    );
    let overload = classify_failure(
        &RawAttemptFailure::Http {
            status: 429,
            kind: Some(ProviderFailureKind::BindingOverload),
            retry_after: None,
        },
        profile,
    );
    let permanent = classify_failure(
        &RawAttemptFailure::Http {
            status: 422,
            kind: Some(ProviderFailureKind::PermanentClient),
            retry_after: None,
        },
        profile,
    );
    assert_eq!(quota.class, AttemptFailureClass::Quota);
    assert_eq!(quota.state_scope(), Some(FailureStateScope::Credential));
    assert_eq!(overload.class, AttemptFailureClass::BindingOverload);
    assert_eq!(overload.state_scope(), Some(FailureStateScope::Binding));
    assert_eq!(permanent.class, AttemptFailureClass::PermanentClient);
    assert!(permanent.is_precommit_relayable());
    assert_eq!(permanent.state_scope(), None);
}

fn assert_fault_control_ack(response: &WireResponse, operation: &str, remaining: u64) {
    assert_eq!(response.status, 200);
    let body: serde_json::Value = serde_json::from_slice(&response.body).unwrap();
    assert_eq!(body["command"], "runtime_state_write_fault");
    assert_eq!(body["outcome"], "ack");
    assert_eq!(body["code"], "RUNTIME_STATE_WRITE_FAULT_ARMED");
    assert_eq!(body["runtime_state_operation"], operation);
    assert_eq!(body["remaining_failures"], remaining);
}

fn assert_fault_control_nack(response: &WireResponse) {
    assert_eq!(response.status, 422);
    let body: serde_json::Value = serde_json::from_slice(&response.body).unwrap();
    assert_eq!(body["command"], "runtime_state_write_fault");
    assert_eq!(body["outcome"], "nack");
    assert_eq!(body["code"], "RUNTIME_STATE_WRITE_FAULT_INVALID");
}

fn assert_runtime_state_503(response: &WireResponse) {
    assert_eq!(
        response.status,
        503,
        "{}",
        String::from_utf8_lossy(&response.body)
    );
    assert_eq!(
        serde_json::from_slice::<serde_json::Value>(&response.body).unwrap(),
        serde_json::json!({
            "schema_version": "hiroute.gateway.error/v1",
            "code": "RUNTIME_STATE_AUTHORITY_UNAVAILABLE",
            "phase": "runtime_state",
        })
    );
}

fn wait_execution_facts(
    fixture: &RuntimeFixture,
    finished_requests: usize,
) -> Vec<serde_json::Value> {
    let path = fixture
        .observation_root
        .as_deref()
        .expect("observation capture")
        .join("execution-fact.jsonl");
    let deadline = Instant::now() + Duration::from_secs(5);
    loop {
        let facts = std::fs::read_to_string(&path)
            .ok()
            .into_iter()
            .flat_map(|contents| contents.lines().map(str::to_owned).collect::<Vec<_>>())
            .filter(|line| !line.is_empty())
            .map(|line| serde_json::from_str::<serde_json::Value>(&line).unwrap())
            .collect::<Vec<_>>();
        let finished = facts
            .iter()
            .filter(|fact| {
                fact.pointer("/fact/kind").and_then(|value| value.as_str())
                    == Some("request_finished")
            })
            .count();
        if finished >= finished_requests {
            return facts;
        }
        assert!(
            Instant::now() < deadline,
            "timed out waiting for execution facts: {facts:?}"
        );
        std::thread::sleep(Duration::from_millis(10));
    }
}

fn wait_content_for_request(fixture: &RuntimeFixture, request_id: &str) -> Vec<serde_json::Value> {
    let path = fixture
        .observation_root
        .as_deref()
        .expect("observation capture")
        .join("conversation-content.jsonl");
    let deadline = Instant::now() + Duration::from_secs(5);
    loop {
        let records = std::fs::read_to_string(&path)
            .ok()
            .into_iter()
            .flat_map(|contents| contents.lines().map(str::to_owned).collect::<Vec<_>>())
            .filter(|line| !line.is_empty())
            .map(|line| serde_json::from_str::<serde_json::Value>(&line).unwrap())
            .collect::<Vec<_>>();
        if records.iter().any(|record| {
            record
                .pointer("/correlation/request_id")
                .and_then(serde_json::Value::as_str)
                == Some(request_id)
                && record.get("phase").and_then(serde_json::Value::as_str) == Some("finish")
        }) {
            return records;
        }
        assert!(
            Instant::now() < deadline,
            "timed out waiting for content observations: {records:?}"
        );
        std::thread::sleep(Duration::from_millis(10));
    }
}

fn assert_single_planner_run_per_request(facts: &[serde_json::Value], expected_requests: usize) {
    let mut requests = BTreeMap::<String, usize>::new();
    for fact in facts {
        let Some(request_id) = fact
            .pointer("/correlation/request_id")
            .and_then(|value| value.as_str())
        else {
            continue;
        };
        if fact.pointer("/fact/kind").and_then(|value| value.as_str()) == Some("route_decision") {
            *requests.entry(request_id.to_owned()).or_default() += 1;
        }
    }
    assert_eq!(requests.len(), expected_requests);
    assert!(requests.values().all(|planner_runs| *planner_runs == 1));
}

fn assert_runtime_state_outcomes(facts: &[serde_json::Value], operation: &str, expected: &[&str]) {
    let mut outcomes = facts
        .iter()
        .filter(|fact| {
            fact.pointer("/fact/operation")
                .and_then(|value| value.as_str())
                == Some(operation)
        })
        .filter_map(|fact| {
            fact.pointer("/fact/outcome")
                .and_then(|value| value.as_str())
        })
        .collect::<Vec<_>>();
    outcomes.sort_unstable();
    assert_eq!(outcomes, expected);
}

fn request_id_for_runtime_state_outcome(
    facts: &[serde_json::Value],
    operation: &str,
    outcome: &str,
) -> String {
    let request_ids = facts
        .iter()
        .filter(|fact| {
            fact.pointer("/fact/kind")
                .and_then(serde_json::Value::as_str)
                == Some("runtime_state")
                && fact
                    .pointer("/fact/operation")
                    .and_then(serde_json::Value::as_str)
                    == Some(operation)
                && fact
                    .pointer("/fact/outcome")
                    .and_then(serde_json::Value::as_str)
                    == Some(outcome)
        })
        .filter_map(|fact| {
            fact.pointer("/correlation/request_id")
                .and_then(serde_json::Value::as_str)
        })
        .collect::<std::collections::BTreeSet<_>>();
    assert_eq!(
        request_ids.len(),
        1,
        "runtime state {operation}/{outcome} must identify one request: {request_ids:?}"
    );
    request_ids.into_iter().next().unwrap().to_owned()
}

fn assert_exact_attempt_lifecycle(
    facts: &[serde_json::Value],
    request_id: &str,
    expected_started: u64,
    expected_finished: u64,
) {
    let request_facts = facts
        .iter()
        .filter(|fact| {
            fact.pointer("/correlation/request_id")
                .and_then(serde_json::Value::as_str)
                == Some(request_id)
        })
        .collect::<Vec<_>>();
    let attempt_ordinals = |kind| {
        request_facts
            .iter()
            .filter(|fact| {
                fact.pointer("/fact/kind")
                    .and_then(serde_json::Value::as_str)
                    == Some(kind)
            })
            .map(|fact| {
                fact.pointer("/fact/ordinal")
                    .and_then(serde_json::Value::as_u64)
                    .expect("attempt receipt ordinal")
            })
            .collect::<Vec<_>>()
    };
    assert_eq!(
        attempt_ordinals("attempt_started"),
        (1..=expected_started).collect::<Vec<_>>(),
        "request {request_id} has an unexpected started Attempt sequence"
    );
    assert_eq!(
        attempt_ordinals("attempt_finished"),
        (1..=expected_finished).collect::<Vec<_>>(),
        "request {request_id} has an unexpected finished Attempt sequence"
    );
    let terminals = request_facts
        .iter()
        .filter(|fact| {
            fact.pointer("/fact/kind")
                .and_then(serde_json::Value::as_str)
                == Some("request_finished")
        })
        .collect::<Vec<_>>();
    assert_eq!(
        terminals.len(),
        1,
        "request {request_id} has one terminal execution receipt"
    );
    let terminal = terminals[0];
    assert_eq!(
        terminal
            .pointer("/fact/attempts_started")
            .and_then(serde_json::Value::as_u64),
        Some(expected_started),
        "request {request_id} terminal started-attempt count"
    );
    assert_eq!(
        terminal
            .pointer("/fact/attempts_finished")
            .and_then(serde_json::Value::as_u64),
        Some(expected_finished),
        "request {request_id} terminal finished-attempt count"
    );
}