Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

56.6 KB
Raw Blame
#[cfg(all(unix, debug_assertions, feature = "e2e-test-control"))]
#[path = "tests/usage_order.rs"]
mod usage_order;

#[path = "tests/preplanner.rs"]
mod preplanner;

use serde_json::{Value, json};

use hiroute_diagnostics::runtime::DiagnosticsPort;

use crate::ports::{ProbeLeaseOutcome, RuntimeStateEntry, RuntimeStateKey};
use hiroute_gateway_core::runtime::attempt::{
    AttemptGeneration, AttemptId, Disposition, PublishedDisposition, RequestId,
};

use super::contracts::{contract_sources, port_set_digest, schema_digest};
use super::otel::{OtelAttempt, OtelGenAiMapper};
use super::*;

fn feedback_identity(channel: &str) -> ObservationFeedbackIdentityV1 {
    ObservationFeedbackIdentityV1 {
        channel: channel.into(),
        producer_id: "producer:test".into(),
        producer_epoch: "epoch:test".into(),
        stream_id: "stream:test".into(),
    }
}

fn content_acknowledgement() -> ObservationContentAcknowledgementV1 {
    ObservationContentAcknowledgementV1 {
        request_id: "request:test".into(),
        direction: ObservationContentDirectionV1::ResponseDelivered,
        fork_id: "fork:test".into(),
        next_chunk_ordinal: 4,
        transcript_root: Some("transcript:result".into()),
        delta_parent_transcript_root: Some("transcript:parent".into()),
        acknowledged_blobs: vec![ObservationBlobAcknowledgementV1 {
            content_id: "content:test".into(),
            digest: "blob:digest".into(),
        }],
    }
}

#[test]
fn native_agent_session_derivation_reuses_the_producer_identity_formula() {
    let key = [41_u8; 32];
    let workspace = WorkspaceId::default();
    let plan_id = hiroute_domain::AgentPlanId::parse("plan/live-check").unwrap();
    let trust = FrozenExecutionTrustV1 {
        authority_id: "authority/live-check".into(),
        authority_epoch: 3,
        served_model_id: "hiroute.live.model".into(),
        selector_source: hiroute_domain::SelectorSourceV1::TrustedModelAlias,
        agent_plan_id: Some(plan_id.clone()),
        route: ModelRequestRouteV2::Plan {
            revision: 7,
            semantic_digest: hiroute_domain::CanonicalDigest::of_bytes(b"live-route"),
        },
        plan_display_name: None,
        gateway_publication_revision: "11".into(),
        gateway_publication_digest: hiroute_domain::CanonicalDigest::of_bytes(b"live-publication"),
        grant_id: "grant/live-check".into(),
        grant_generation: 5,
        ingress_protocol: hiroute_domain::IngressProtocolV1::Responses,
    };
    let identity = NativeAgentObservationIdentityV1::CodexThreadSession {
        thread_id: "thread-live".into(),
        session_id: "session-live".into(),
    };
    let expected = SessionId::parse(reliable_observation_session_id(
        &key,
        workspace.as_str(),
        &trust.authority_id,
        &trust.grant_id,
        plan_id.as_str(),
        &trust.served_model_id,
        "responses",
        "codex_thread_session",
        &["thread-live", "session-live"],
    ))
    .unwrap();
    assert_eq!(
        derive_native_agent_observation_session_id(&workspace, &key, &trust, &identity).unwrap(),
        expected
    );
    assert!(
        derive_native_agent_observation_session_id(&workspace, &[0; 32], &trust, &identity)
            .is_err()
    );
    let qoder = NativeAgentObservationIdentityV1::QoderSession {
        session_id: "qoder-session".into(),
    };
    let header_session = NativeAgentObservationIdentityV1::CodexSession {
        session_id: "qoder-session".into(),
    };
    let session =
        derive_native_agent_observation_session_id(&workspace, &key, &trust, &qoder).unwrap();
    assert_eq!(
        session,
        derive_native_agent_observation_session_id(&workspace, &key, &trust, &header_session)
            .unwrap()
    );
    let mut other_grant = trust.clone();
    other_grant.grant_id = "grant/other".into();
    assert_ne!(
        session,
        derive_native_agent_observation_session_id(&workspace, &key, &other_grant, &qoder).unwrap()
    );
    let mut messages_trust = trust.clone();
    messages_trust.ingress_protocol = hiroute_domain::IngressProtocolV1::Messages;
    let messages_session =
        derive_native_agent_observation_session_id(&workspace, &key, &messages_trust, &qoder)
            .unwrap();
    assert_ne!(
        session, messages_session,
        "protocol remains part of the trusted session identity"
    );
    assert_eq!(
        messages_session,
        derive_native_agent_observation_session_id(
            &workspace,
            &key,
            &messages_trust,
            &header_session
        )
        .unwrap()
    );
    for (identity, kind) in [
        (
            NativeAgentObservationIdentityV1::PiSession {
                session_id: "pi-native".into(),
            },
            "pi_session",
        ),
        (
            NativeAgentObservationIdentityV1::DeepseekHarnessSession {
                session_id: "dsh-native".into(),
            },
            "deepseek_harness_session",
        ),
        (
            NativeAgentObservationIdentityV1::OpenCodeSession {
                session_id: "opencode-native".into(),
            },
            "opencode_session",
        ),
    ] {
        let raw = match &identity {
            NativeAgentObservationIdentityV1::PiSession { session_id }
            | NativeAgentObservationIdentityV1::DeepseekHarnessSession { session_id }
            | NativeAgentObservationIdentityV1::OpenCodeSession { session_id } => session_id,
            _ => unreachable!(),
        };
        let expected = SessionId::parse(reliable_observation_session_id(
            &key,
            workspace.as_str(),
            &trust.authority_id,
            &trust.grant_id,
            trust.agent_plan_id.as_ref().unwrap().as_str(),
            &trust.served_model_id,
            "responses",
            kind,
            &[raw],
        ))
        .unwrap();
        assert_eq!(
            derive_native_agent_observation_session_id(&workspace, &key, &trust, &identity)
                .unwrap(),
            expected
        );
        assert_ne!(
            derive_native_agent_observation_session_id(
                &workspace,
                &key,
                &messages_trust,
                &identity
            )
            .unwrap(),
            expected
        );
    }
    let mut wrong_protocol = trust;
    wrong_protocol.ingress_protocol = hiroute_domain::IngressProtocolV1::ChatCompletions;
    assert!(
        derive_native_agent_observation_session_id(&workspace, &key, &wrong_protocol, &qoder)
            .is_err()
    );
    let opencode = NativeAgentObservationIdentityV1::OpenCodeSession {
        session_id: "opencode-native".into(),
    };
    let expected_chat = SessionId::parse(reliable_observation_session_id(
        &key,
        workspace.as_str(),
        &wrong_protocol.authority_id,
        &wrong_protocol.grant_id,
        wrong_protocol.agent_plan_id.as_ref().unwrap().as_str(),
        &wrong_protocol.served_model_id,
        "chat_completions",
        "opencode_session",
        &["opencode-native"],
    ))
    .unwrap();
    assert_eq!(
        derive_native_agent_observation_session_id(&workspace, &key, &wrong_protocol, &opencode)
            .unwrap(),
        expected_chat
    );
    wrong_protocol.ingress_protocol = hiroute_domain::IngressProtocolV1::Unknown;
    assert_eq!(
        derive_native_agent_observation_session_id(&workspace, &key, &wrong_protocol, &opencode),
        Err(NativeAgentObservationSessionError::InvalidTrust)
    );
}

fn request(policy: OtelContentPolicy) -> RequestObservation {
    request_with_diagnostics(policy, DiagnosticsPort::default())
}

fn request_with_diagnostics(
    policy: OtelContentPolicy,
    diagnostics: DiagnosticsPort,
) -> RequestObservation {
    request_with_capture(policy, diagnostics, true)
}

fn request_with_capture(
    policy: OtelContentPolicy,
    diagnostics: DiagnosticsPort,
    enabled: bool,
) -> RequestObservation {
    request_with_sinks(
        policy,
        diagnostics,
        enabled,
        GatewayObservationSinks::discard(),
    )
}

fn request_with_sinks(
    policy: OtelContentPolicy,
    diagnostics: DiagnosticsPort,
    enabled: bool,
    sinks: GatewayObservationSinks,
) -> RequestObservation {
    let gateway = GatewayObservation::with_sinks_and_policy(true, 64 * 1024, sinks, policy);
    RequestObservation::new(
        super::request::RequestObservationCapture {
            enabled,
            content: true,
        },
        gateway.key,
        gateway.channels.clone(),
        policy,
        super::pricing::capture(None, "workspace:test", &[], 0),
        RequestObservationMetadata {
            workspace_id: "workspace:test".into(),
            conversation_id: "conversation:test".into(),
            session_scope: "request_scoped".into(),
            correlation_provenance: "unproven".into(),
            turn_id: "turn:test".into(),
            request_id: "request:test".into(),
            authority_id: "authority:test".into(),
            authority_epoch: 1,
            publication_revision: 7,
            publication_digest: "sha256:publication".into(),
            agent_plan_id: Some("plan/test".into()),
            route: hiroute_domain::ModelRequestRouteV2::Plan {
                revision: 5,
                semantic_digest: hiroute_domain::CanonicalDigest::of_bytes(b"plan"),
            },
            plan_display_name: Some("Test plan".into()),
            served_model_id: "served-test".into(),
            grant_id: "grant:test".into(),
            grant_generation: 3,
            ingress_protocol: "responses".into(),
        },
        diagnostics,
    )
}

#[test]
fn info_failure_keeps_actual_reasoning_and_safe_provider_attribution_without_content_capture() {
    use hiroute_diagnostics::{
        DiagnosticLevel,
        event::{ProcessRole, WireProviderError},
        record::Component,
        runtime::{DiagnosticRuntime, RuntimeConfig},
    };
    for enabled in [true, false] {
        let root = std::env::temp_dir().join(format!(
            "hiroute-info-wire-{}-{enabled}-{}",
            std::process::id(),
            super::unix_nanos()
        ));
        let runtime = DiagnosticRuntime::start(RuntimeConfig {
            root: root.clone(),
            role: ProcessRole::Daemon,
            component: Component::Gateway,
            parent_session_id: None,
            level_override: Some(DiagnosticLevel::Info),
        });
        let request = request_with_capture(OtelContentPolicy::Disabled, runtime.port(), enabled);
        assert_eq!(request.captures_content(), enabled);
        assert_eq!(request.is_enabled(), enabled);
        request.no_credential_materialized("binding:test", "credential/none/test");
        request.prepared_request_diagnostic(&http::HeaderMap::new(), br#"{"model":"qwen3.6-flash","reasoning":{"effort":"high"},"thinking":{"type":"enabled","budget_tokens":81920},"input":"private-prompt-marker"}"#);
        let mut headers = http::HeaderMap::new();
        headers.insert("x-request-id", "private-provider-id".parse().unwrap());
        request.response_head_diagnostic(&headers, 400);
        request.provider_failure_diagnostic(Some(400), WireProviderError::ThinkingBudgetRejected);
        request.disposition_published(&PublishedDisposition {
            request_id: RequestId(1),
            attempt_id: AttemptId(1),
            generation: AttemptGeneration(1),
            disposition: Disposition::Continue,
        });
        request.finish("failed");
        runtime.shutdown();
        let log = std::fs::read_to_string(root.join("daemon/current.jsonl")).unwrap();
        assert!(
            log.contains("thinking_budget_rejected") && log.contains("qwen3.6-flash"),
            "{log}"
        );
        assert!(
            log.contains("\"responses_effort\":\"high\"")
                && log.contains("\"messages_budget_tokens\":81920"),
            "{log}"
        );
        assert!(
            log.contains("upstream_request_token") && log.contains("\"outcome\":\"failed\""),
            "{log}"
        );
        assert!(
            !log.contains("private-prompt-marker") && !log.contains("private-provider-id"),
            "{log}"
        );
        assert!(
            !log.contains("\"phase\":\"request\"") && !log.contains("\"phase\":\"response\""),
            "{log}"
        );
        std::fs::remove_dir_all(root).unwrap();
    }
}

/// Drives the real observation branches that production runs: candidate staging, the
/// published accept, the first accepted frame and the terminal request, and asserts that
/// the diagnostic sink receives typed correlated events instead of raw identities.
#[test]
fn request_lifecycle_records_typed_diagnostics_without_raw_identities() {
    use hiroute_diagnostics::event::ProcessRole;
    use hiroute_diagnostics::level::DiagnosticLevel;
    use hiroute_diagnostics::record::Component;
    use hiroute_diagnostics::runtime::{DiagnosticRuntime, RuntimeConfig};
    use std::sync::atomic::{AtomicUsize, Ordering};

    static SEQUENCE: AtomicUsize = AtomicUsize::new(0);
    let nanos = std::time::SystemTime::now()
        .duration_since(std::time::SystemTime::UNIX_EPOCH)
        .expect("clock after epoch")
        .as_nanos();
    let root = std::env::temp_dir().join(format!(
        "hiroute-gateway-diagnostics-{}-{}-{nanos}",
        std::process::id(),
        SEQUENCE.fetch_add(1, Ordering::Relaxed),
    ));
    let runtime = DiagnosticRuntime::start(RuntimeConfig {
        root: root.clone(),
        role: ProcessRole::Daemon,
        component: Component::Gateway,
        parent_session_id: None,
        // Pipeline stages and staged attempts are Debug; the test asserts them.
        level_override: Some(DiagnosticLevel::Debug),
    });
    let request = request_with_diagnostics(OtelContentPolicy::Disabled, runtime.port());
    request.record_pipeline_stages(
        std::time::Duration::from_millis(1),
        std::time::Duration::from_millis(2),
        std::time::Duration::from_millis(3),
    );
    request.record_plan_stage(std::time::Duration::from_millis(4));
    request.no_credential_materialized("binding:test", "credential/none/source-local-test");
    let mut wire_headers = http::HeaderMap::new();
    wire_headers.insert(
        "x-request-id",
        "provider-secret-request-id".parse().unwrap(),
    );
    request.prepared_request_diagnostic(
        &wire_headers,
        br#"{"model":"native-model","reasoning":{"effort":"low"},"input":"secret-prompt-marker"}"#,
    );
    request.response_head_diagnostic(&wire_headers, 200);
    use hiroute_diagnostics::event::ReasoningCleanupReason;
    request.reasoning_cleanup(ReasoningCleanupReason::ContextBreakRetry, 2, Some(3));
    request.reasoning_cleanup(ReasoningCleanupReason::SuccessfulPrefixReuse, 0, Some(3));
    assert!(
        request
            .lock_state()
            .pending_attempt
            .as_ref()
            .unwrap()
            .attempt_id
            .is_empty()
    );
    request.disposition_published(&PublishedDisposition {
        request_id: RequestId(9),
        attempt_id: AttemptId(4),
        generation: AttemptGeneration(2),
        disposition: Disposition::Accept,
    });
    assert!(request.accept_current("frame:test", 128).is_some());
    request.reasoning_cleanup(ReasoningCleanupReason::ResponseProtocolProjection, 1, None);
    request.reasoning_cleanup(ReasoningCleanupReason::ResponseProtocolProjection, 3, None);
    request.response_failure(
        hiroute_diagnostics::event::ResponseFailureStage::NativeProjection,
        hiroute_diagnostics::event::ResponseFailureReason::ResourceLimit,
    );
    assert_eq!(
        request
            .lock_state()
            .accepted_attempt
            .as_ref()
            .unwrap()
            .reasoning_fields_removed,
        6
    );
    request.finish("accepted");
    runtime.shutdown();

    let log = std::fs::read_to_string(root.join("daemon").join("current.jsonl")).unwrap();
    for kind in [
        "request_begin",
        "attempt_begin",
        "semantic_commit",
        "request_end",
        "model_stage",
        "response_failure",
        "upstream_wire",
    ] {
        assert!(
            log.contains(&format!("\"{kind}\":")),
            "{kind} missing: {log}"
        );
    }
    assert!(log.contains("\"attempt_index\":1"), "{log}");
    assert!(log.contains("\"native_model\":\"native-model\""), "{log}");
    assert!(log.contains("\"responses_effort\":\"low\""), "{log}");
    assert!(
        !log.contains("provider-secret-request-id") && !log.contains("secret-prompt-marker"),
        "{log}"
    );
    assert!(log.contains("\"outcome\":\"completed\""), "{log}");
    assert!(log.contains("\"stage\":\"parse\""), "{log}");
    assert!(log.contains("\"state\":\"semantic_committed\""), "{log}");
    let records: Vec<Value> = log
        .lines()
        .map(|line| serde_json::from_str(line).unwrap())
        .collect();
    // Two warnings only: the actual request loss and the first response loss.
    // Further response losses are accumulated, not logged per stream fragment.
    assert_eq!(log.matches("\"reasoning_cleanup\":").count(), 2, "{log}");
    assert!(log.contains("\"fields_removed_so_far\":3"), "{log}");
    let serialized = records.iter().map(Value::to_string).collect::<Vec<_>>();
    let begin = serialized
        .iter()
        .find(|row| row.contains("\"attempt_begin\":"))
        .unwrap();
    // Pre-send warnings use the request/ordinal/binding, never a fake attempt
    // token. After publication the response warning has the exact core token.
    fn payload<'a>(value: &'a Value, key: &str) -> Option<&'a Value> {
        value.get(key).or_else(|| {
            value
                .as_object()?
                .values()
                .find_map(|value| payload(value, key))
        })
    }
    let begin: Value = serde_json::from_str(begin).unwrap();
    let token = &payload(&begin, "attempt_begin").unwrap()["attempt_token"];
    assert!(!token.is_null());
    for record in &records {
        if let Some(cleanup) = payload(record, "reasoning_cleanup") {
            assert_eq!(cleanup["attempt_index"], 1);
            assert!(!cleanup["binding_token"].is_null());
            if cleanup["reason"] == "response_protocol_projection" {
                assert_eq!(&cleanup["attempt_token"], token);
            } else {
                assert!(cleanup["attempt_token"].is_null());
            }
        }
    }
    assert!(log.find("\"reasoning_cleanup\":").unwrap() < log.find("\"attempt_begin\":").unwrap());
    assert!(
        !log.contains("request:test"),
        "raw request id leaked: {log}"
    );
    assert!(
        !log.contains("binding:test"),
        "raw binding id leaked: {log}"
    );
    let _ = std::fs::remove_dir_all(&root);
}

#[test]
fn accepted_client_cancellation_finishes_the_request_as_cancelled() {
    use hiroute_diagnostics::event::ProcessRole;
    use hiroute_diagnostics::record::Component;
    use hiroute_diagnostics::runtime::{DiagnosticRuntime, RuntimeConfig};
    use hiroute_gateway_core::core::execution_plan::{
        CredentialRef, PlanRevision, ResolvedTargetBindingId,
    };
    use hiroute_gateway_core::runtime::attempt::{AttemptTransportFacts, CommitFence};
    use hiroute_gateway_core::runtime::driver::{
        AttemptBudgetGrant, AttemptCleanupOutcome, AttemptCommitFacts, AttemptDownstreamOutcome,
        AttemptStreamOutcome, AttemptTerminationReason, CompletedAttemptObservation,
        RealtimeRoutingFacts, RouteDecisionId,
    };
    use std::time::{Duration, Instant};

    let root = std::env::temp_dir().join(format!(
        "hiroute-cancel-diagnostics-{}-{}",
        std::process::id(),
        std::time::SystemTime::now()
            .duration_since(std::time::SystemTime::UNIX_EPOCH)
            .unwrap()
            .as_nanos()
    ));
    let runtime = DiagnosticRuntime::start(RuntimeConfig {
        root: root.clone(),
        role: ProcessRole::Daemon,
        component: Component::Gateway,
        parent_session_id: None,
        level_override: None,
    });
    let request = request_with_diagnostics(OtelContentPolicy::Disabled, runtime.port());
    let credential = "credential/none/source-local-test";
    request.no_credential_materialized("binding:test", credential);
    request.disposition_published(&PublishedDisposition {
        request_id: RequestId(9),
        attempt_id: AttemptId(4),
        generation: AttemptGeneration(2),
        disposition: Disposition::Accept,
    });
    assert!(request.accept_current("frame:test", 128).is_some());
    let now = Instant::now();
    request.completed_attempt(
        &CompletedAttemptObservation {
            route_decision_id: RouteDecisionId(1),
            attempt_id: AttemptId(4),
            generation: AttemptGeneration(2),
            binding: ResolvedTargetBindingId::new(PlanRevision(1), 1),
            credential_ref: CredentialRef::new(credential).unwrap(),
            budget: AttemptBudgetGrant {
                issued_at: now,
                allocated: Duration::from_secs(1),
                deadline: now + Duration::from_secs(1),
            },
            routing_facts: RealtimeRoutingFacts::default(),
            provider: None,
            failure: None,
            transport: AttemptTransportFacts {
                started_at: now,
                upstream_protocol: Some(hiroute_gateway_core::transport::HttpProtocol::Http1),
                connect_elapsed: None,
                request_write_elapsed: None,
                upstream_ttfb: None,
                last_upstream_progress_at: None,
                local_read_suppressed: Duration::ZERO,
                upstream_body_bytes: 128,
                timeout: None,
            },
            disposition: Disposition::Accept,
            commits: AttemptCommitFacts {
                upstream_request: CommitFence::WriteConfirmed,
                downstream_headers: CommitFence::WriteConfirmed,
                downstream_semantic: CommitFence::WriteConfirmed,
            },
            stream: AttemptStreamOutcome::StreamStartedNoRetry,
            downstream: AttemptDownstreamOutcome::Cancelled,
            cleanup: AttemptCleanupOutcome::Completed,
            ended_at: now,
            termination_reason: AttemptTerminationReason::Cancelled,
        },
        None,
    );
    request.finish("cancelled");
    runtime.shutdown();

    let log = std::fs::read_to_string(root.join("daemon").join("current.jsonl")).unwrap();
    let events = log
        .lines()
        .map(|line| serde_json::from_str::<Value>(line).unwrap())
        .collect::<Vec<_>>();
    assert!(log.contains("\"http_protocol\":\"http1\""), "{log}");
    assert!(log.contains("\"downstream_body\":\"committed\""), "{log}");
    for kind in ["attempt_end", "request_end"] {
        assert!(
            events
                .iter()
                .any(|event| event["event"][kind]["outcome"] == "cancelled"),
            "{kind} missing or misclassified: {log}"
        );
    }
    assert!(
        events
            .iter()
            .any(|event| event["event"]["request_cancel"]["accepted"] == true),
        "accepted cancellation missing: {log}"
    );
    let _ = std::fs::remove_dir_all(&root);
}

/// Rejects every delivery, so the channel's own sink-failure projection is exercised
/// without a real sink outage.
struct RejectingSink;

impl ObservationRecordSink for RejectingSink {
    fn deliver(&self, record: &ObservationRecord) -> Result<ObservationAck, ObservationNack> {
        let identity = &record.stamp().identity;
        Err(Box::new(ObservationNackV1 {
            schema_version: OBSERVATION_NACK_SCHEMA.into(),
            identity: ObservationFeedbackIdentityV1 {
                channel: identity.channel.to_string(),
                producer_id: identity.producer_id.to_string(),
                producer_epoch: identity.producer_epoch.to_string(),
                stream_id: identity.stream_id.to_string(),
            },
            rejected_sequence: record.stamp().sequence,
            expected_sequence: record.stamp().sequence,
            retryable: true,
            detail: ObservationNackDetailV1::ReceiverUnavailable {
                retry_after_millis: None,
            },
        }))
    }
}

/// A normal streamed answer can produce more than 256 small content deltas or
/// execution facts while the storage worker is busy. The byte budget, not a
/// second fixed event cap, must determine whether these small records fit.
#[test]
fn observation_channels_keep_small_bursts_contiguous_during_sink_backpressure() {
    use std::sync::Condvar;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use std::time::{Duration, Instant};

    struct GatedSink {
        gate: Arc<(Mutex<bool>, Condvar)>,
        delivered: AtomicUsize,
        gaps: AtomicUsize,
    }

    impl ObservationRecordSink for GatedSink {
        fn deliver(&self, record: &ObservationRecord) -> Result<ObservationAck, ObservationNack> {
            let (lock, wake) = &*self.gate;
            let released = lock.lock().unwrap_or_else(|error| error.into_inner());
            let _released = wake
                .wait_while(released, |released| !*released)
                .unwrap_or_else(|error| error.into_inner());
            if record.is_gap_heartbeat() {
                self.gaps.fetch_add(1, Ordering::Relaxed);
            } else {
                self.delivered.fetch_add(1, Ordering::Relaxed);
            }
            Ok(accounted_acknowledgement(record))
        }
    }

    const RECORDS: usize = 600;
    for channel in [ObservationChannel::Content, ObservationChannel::Facts] {
        let sink = Arc::new(GatedSink {
            gate: Arc::new((Mutex::new(false), Condvar::new())),
            delivered: AtomicUsize::new(0),
            gaps: AtomicUsize::new(0),
        });
        let discard = GatewayObservationSinks::discard();
        let mut sinks = GatewayObservationSinks {
            lifecycle: discard.lifecycle,
            execution_fact: discard.execution_fact,
            conversation_content: discard.conversation_content,
            run_relation: discard.run_relation,
            native_continuation: discard.native_continuation,
            otel: discard.otel,
        };
        let selected = match channel {
            ObservationChannel::Content => &mut sinks.conversation_content,
            ObservationChannel::Facts => &mut sinks.execution_fact,
            _ => unreachable!(),
        };
        *selected = sink.clone();
        let gateway = GatewayObservation::with_sinks_and_policy(
            true,
            4 * 1024 * 1024,
            sinks,
            OtelContentPolicy::Disabled,
        );

        for ordinal in 0..RECORDS {
            let publish = |_, sequence, _| json!({"schema_version":"hiroute.observation.test/v1","sequence":sequence,"ordinal":ordinal});
            match channel {
                ObservationChannel::Content => gateway.channels.content.publish(publish),
                ObservationChannel::Facts => gateway.channels.execution.publish(publish),
                _ => unreachable!(),
            }
        }
        let (lock, wake) = &*sink.gate;
        *lock.lock().unwrap_or_else(|error| error.into_inner()) = true;
        wake.notify_all();

        let deadline = Instant::now() + Duration::from_secs(5);
        while sink.delivered.load(Ordering::Relaxed) < RECORDS && Instant::now() < deadline {
            std::thread::sleep(Duration::from_millis(10));
        }
        assert_eq!(
            sink.delivered.load(Ordering::Relaxed),
            RECORDS,
            "{channel:?}"
        );
        assert_eq!(sink.gaps.load(Ordering::Relaxed), 0, "{channel:?}");
    }
}

#[test]
fn channel_loss_and_sink_nack_are_projected_without_record_payloads() {
    use hiroute_diagnostics::event::ProcessRole;
    use hiroute_diagnostics::record::Component;
    use hiroute_diagnostics::runtime::{DiagnosticRuntime, RuntimeConfig};
    use std::sync::atomic::{AtomicUsize, Ordering};

    static SEQUENCE: AtomicUsize = AtomicUsize::new(0);
    let nanos = std::time::SystemTime::now()
        .duration_since(std::time::SystemTime::UNIX_EPOCH)
        .expect("clock after epoch")
        .as_nanos();
    let root = std::env::temp_dir().join(format!(
        "hiroute-gateway-channel-diagnostics-{}-{}-{nanos}",
        std::process::id(),
        SEQUENCE.fetch_add(1, Ordering::Relaxed),
    ));
    let runtime = DiagnosticRuntime::start(RuntimeConfig {
        root: root.clone(),
        role: ProcessRole::Daemon,
        component: Component::Gateway,
        parent_session_id: None,
        // Channel loss and NACK are warnings; the default info threshold admits them.
        level_override: None,
    });
    let discard: Arc<dyn ObservationRecordSink> = GatewayObservationSinks::discard().lifecycle;
    let gateway = GatewayObservation::with_sinks_and_policy(
        true,
        1024,
        GatewayObservationSinks {
            lifecycle: discard,
            execution_fact: Arc::new(RejectingSink),
            conversation_content: GatewayObservationSinks::discard().conversation_content,
            run_relation: GatewayObservationSinks::discard().run_relation,
            native_continuation: GatewayObservationSinks::discard().native_continuation,
            otel: GatewayObservationSinks::discard().otel,
        },
        OtelContentPolicy::Disabled,
    )
    .with_diagnostics(runtime.port());

    // Over the byte capacity: the channel declares a loss range instead of queueing it.
    gateway.channels.lifecycle.publish(|_, _, _| {
        let mut payload = b"channel-payload-sentinel".to_vec();
        payload.resize(64 * 1024, b'x');
        payload
    });
    // A sink that rejects the record: the failure is reported once, with its reason.
    gateway
        .channels
        .execution
        .publish(|_, _, _| br#"{"schema_version":"hiroute.observation.test/v1"}"#.to_vec());

    let path = root.join("daemon").join("current.jsonl");
    let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
    let log = loop {
        let text = std::fs::read_to_string(&path).unwrap_or_default();
        if (text.contains("\"observation_gap\":") && text.contains("\"sink_nack\":"))
            || std::time::Instant::now() >= deadline
        {
            break text;
        }
        std::thread::sleep(std::time::Duration::from_millis(20));
    };
    runtime.shutdown();

    assert!(log.contains("\"observation_gap\":"), "{log}");
    assert!(log.contains("\"reason\":\"queue_overflow\""), "{log}");
    assert!(log.contains("\"missing\":1"), "{log}");
    assert!(log.contains("\"sink_nack\":"), "{log}");
    assert!(log.contains("\"reason\":\"unavailable\""), "{log}");
    assert!(
        !log.contains("channel-payload-sentinel"),
        "channel payload leaked: {log}"
    );
    let _ = std::fs::remove_dir_all(&root);
}

/// The capture summary reports the real byte count, 64KiB chunks and short reads of a
/// request whose canonical content lives in the replay backing, and never the content.
#[test]
fn request_content_capture_reports_bytes_chunks_and_short_reads() {
    use hiroute_diagnostics::event::ProcessRole;
    use hiroute_diagnostics::level::DiagnosticLevel;
    use hiroute_diagnostics::record::Component;
    use hiroute_diagnostics::runtime::{DiagnosticRuntime, RuntimeConfig};
    use hiroute_gateway_core::runtime::body::BudgetTree;
    use std::sync::atomic::{AtomicUsize, Ordering};

    static SEQUENCE: AtomicUsize = AtomicUsize::new(0);
    let nanos = std::time::SystemTime::now()
        .duration_since(std::time::SystemTime::UNIX_EPOCH)
        .expect("clock after epoch")
        .as_nanos();
    let unique = format!("{}-{nanos}", SEQUENCE.fetch_add(1, Ordering::Relaxed));
    let diagnostics_root = std::env::temp_dir().join(format!(
        "hiroute-gateway-content-diagnostics-{}-{unique}",
        std::process::id(),
    ));
    let replay_root = std::env::temp_dir().join(format!(
        "hiroute-gateway-content-replay-{}-{unique}",
        std::process::id(),
    ));
    let sentinel = "content-summary-sentinel ".repeat(4096);
    let mut request_ir = crate::server::core_runtime::adapters::decode_ingress_request(
        crate::server::request_plan::IngressProtocol::Responses,
        &json!({
            "model": "alias",
            "input": [{
                "type": "message",
                "role": "user",
                "content": [{"type": "input_text", "text": sentinel}],
            }],
        }),
    )
    .expect("decode ingress request");
    let manager = crate::replay::ReplayManager::open(crate::replay::ReplayConfig {
        root: replay_root.clone(),
        memory_threshold_bytes: 4 * 1024 * 1024,
        // Small records make the capture read the backing in many short reads.
        record_bytes: 31,
        orphan_ttl: std::time::Duration::from_secs(60),
    })
    .expect("replay manager");
    let tree = BudgetTree::new(8 * 1024 * 1024, 8 * 1024 * 1024).expect("budget tree");
    let budget = tree.stream(8 * 1024 * 1024).expect("stream budget");
    let store = manager.begin_request(budget.clone()).expect("replay store");
    crate::content_ref::externalize_model_request(&mut request_ir, &store, 8)
        .expect("externalize canonical content");
    let runtime = DiagnosticRuntime::start(RuntimeConfig {
        root: diagnostics_root.clone(),
        role: ProcessRole::Daemon,
        component: Component::Gateway,
        parent_session_id: None,
        // The capture summary is Info; the default threshold would admit it too.
        level_override: Some(DiagnosticLevel::Debug),
    });
    let observation = request_with_diagnostics(OtelContentPolicy::Disabled, runtime.port());
    observation.capture_request(&request_ir, &store);
    runtime.shutdown();

    let log =
        std::fs::read_to_string(diagnostics_root.join("daemon").join("current.jsonl")).unwrap();
    assert!(log.contains("\"content_capture_end\":"), "{log}");
    assert!(log.contains("\"role\":\"request\""), "{log}");
    assert!(log.contains("\"outcome\":\"success\""), "{log}");
    assert!(log.contains("\"max_chunk_bytes\":65536"), "{log}");
    assert!(log.contains("\"chunks\":2"), "{log}");
    assert!(!log.contains("\"short_reads\":0"), "{log}");
    assert!(log.contains("\"spill_begin\":"), "{log}");
    assert!(log.contains("\"mode\":\"memory\""), "{log}");
    assert!(
        !log.contains("content-summary-sentinel"),
        "content leaked: {log}"
    );

    let line = log
        .lines()
        .find(|line| line.contains("\"content_capture_end\":"))
        .expect("one capture summary line");
    let summary: Value = serde_json::from_str(line).expect("summary is valid JSON");
    let bytes = summary["event"]["content_capture_end"]["bytes"]
        .as_u64()
        .expect("byte count");
    assert!(bytes >= sentinel.len() as u64, "{log}");
    let read_calls = summary["event"]["content_capture_end"]["read_calls"]
        .as_u64()
        .expect("read calls");
    assert!(read_calls > 2, "{log}");
    let _ = std::fs::remove_dir_all(&diagnostics_root);
    let _ = std::fs::remove_dir_all(&replay_root);
}

#[test]
fn predictable_workspace_hmac_key_disables_observation() {
    let gateway = GatewayObservation::with_sinks_and_policy_and_workspace_key(
        true,
        64 * 1024,
        GatewayObservationSinks::discard(),
        OtelContentPolicy::Disabled,
        [0_u8; 32],
    );
    assert!(!gateway.is_enabled());
}

#[test]
fn external_port_schema_and_aggregate_digests_are_exact() {
    for (source, schema, expected) in contract_sources() {
        let parsed: Value = serde_json::from_str(source).expect("contract source is valid JSON");
        assert_eq!(parsed.get("schema").and_then(Value::as_str), Some(schema));
        assert!(!source.contains('\n'));
        assert_eq!(schema_digest(source), expected);
    }
    let contracts = gateway_port_contracts();
    assert_eq!(contracts.schema_version, GATEWAY_PORT_SET_SCHEMA);
    assert_eq!(contracts.gate, "G_port_process_22020");
    assert_eq!(contracts.ports.len(), 6);
    assert_eq!(port_set_digest(&contracts.ports), GATEWAY_PORT_SET_DIGEST);
    assert_eq!(
        contracts
            .ports
            .iter()
            .map(|port| port.schema_version.as_str())
            .collect::<Vec<_>>(),
        [
            RUNTIME_PUBLICATION_PORT_SCHEMA,
            CREDENTIAL_PORT_SCHEMA,
            RUNTIME_STATE_PORT_SCHEMA,
            LIFECYCLE_FACT_PORT_SCHEMA,
            EXECUTION_FACT_PORT_SCHEMA,
            CONVERSATION_CONTENT_PORT_SCHEMA,
        ]
    );
    for schema in [
        LIFECYCLE_FACT_PORT_SCHEMA,
        EXECUTION_FACT_PORT_SCHEMA,
        CONVERSATION_CONTENT_PORT_SCHEMA,
    ] {
        let source = gateway_port_contract_source(schema).unwrap();
        assert!(source.contains("hiroute.observation.ack/v2"));
        assert!(source.contains("hiroute.observation.nack/v1"));
        assert!(source.contains("\"free_form_reason\":\"forbidden\""));
    }
}

#[test]
fn observation_ack_v2_has_exact_golden_shape_and_denies_unknown_fields() {
    let acknowledgement = ObservationAckV2 {
        schema_version: OBSERVATION_ACK_SCHEMA.into(),
        identity: feedback_identity(CONVERSATION_CONTENT_CHANNEL),
        highest_contiguous_sequence: 6,
        highest_accounted_sequence: 9,
        content_acknowledgement: Some(content_acknowledgement()),
    };
    assert_eq!(
        serde_json::to_value(&acknowledgement).unwrap(),
        json!({
            "schema_version": "hiroute.observation.ack/v2",
            "identity": {
                "channel": "conversation_content",
                "producer_id": "producer:test",
                "producer_epoch": "epoch:test",
                "stream_id": "stream:test"
            },
            "highest_contiguous_sequence": 6,
            "highest_accounted_sequence": 9,
            "content_acknowledgement": {
                "request_id": "request:test",
                "direction": "response_delivered",
                "fork_id": "fork:test",
                "next_chunk_ordinal": 4,
                "transcript_root": "transcript:result",
                "delta_parent_transcript_root": "transcript:parent",
                "acknowledged_blobs": [{
                    "content_id": "content:test",
                    "digest": "blob:digest"
                }]
            }
        })
    );
    acknowledgement.validate().unwrap();

    let mut unknown_top_level = serde_json::to_value(&acknowledgement).unwrap();
    unknown_top_level
        .as_object_mut()
        .unwrap()
        .insert("reason".into(), json!("not permitted"));
    assert!(serde_json::from_value::<ObservationAckV2>(unknown_top_level).is_err());

    let mut unknown_nested = serde_json::to_value(&acknowledgement).unwrap();
    unknown_nested["content_acknowledgement"]
        .as_object_mut()
        .unwrap()
        .insert("opaque".into(), json!({}));
    assert!(serde_json::from_value::<ObservationAckV2>(unknown_nested).is_err());
}

#[test]
fn observation_nack_v1_all_closed_details_round_trip() {
    let blob = ObservationBlobAcknowledgementV1 {
        content_id: "content:test".into(),
        digest: "blob:digest".into(),
    };
    let details = vec![
        ObservationNackDetailV1::ReceiverUnavailable {
            retry_after_millis: Some(25),
        },
        ObservationNackDetailV1::UnsupportedSchema {
            rejected_schema_version: "hiroute.observation.future/v9".into(),
            supported_schema_versions: vec![EXECUTION_FACT_SCHEMA.into()],
        },
        ObservationNackDetailV1::InvalidEnvelope {
            violation: ObservationEnvelopeViolationV1::MissingField,
            field: Some("event_id".into()),
        },
        ObservationNackDetailV1::MissingSequenceRanges {
            ranges: vec![ObservationSequenceRangeV1 {
                first_sequence: 3,
                last_sequence: 5,
            }],
        },
        ObservationNackDetailV1::SequenceEventConflict {
            sequence: 7,
            expected_event_id: "event:expected".into(),
            rejected_event_id: "event:rejected".into(),
        },
        ObservationNackDetailV1::MissingPrerequisite {
            prerequisite_sequence: 6,
            prerequisite_event_id: Some("event:prerequisite".into()),
        },
        ObservationNackDetailV1::UnknownTranscriptRoot {
            request_id: "request:test".into(),
            direction: ObservationContentDirectionV1::RequestInput,
            fork_id: "fork:test".into(),
            transcript_root: "transcript:unknown".into(),
        },
        ObservationNackDetailV1::MissingBlob {
            request_id: "request:test".into(),
            direction: ObservationContentDirectionV1::ResponseDelivered,
            fork_id: "fork:test".into(),
            blobs: vec![blob],
        },
        ObservationNackDetailV1::ChunkOrdinalConflict {
            request_id: "request:test".into(),
            direction: ObservationContentDirectionV1::ResponseDelivered,
            fork_id: "fork:test".into(),
            expected_chunk_ordinal: 4,
            rejected_chunk_ordinal: 6,
        },
        ObservationNackDetailV1::ContentStateConflict {
            request_id: "request:test".into(),
            direction: ObservationContentDirectionV1::ResponseDelivered,
            fork_id: "fork:test".into(),
            expected_phase: ObservationContentPhaseV1::Append,
            rejected_phase: ObservationContentPhaseV1::Finish,
        },
        ObservationNackDetailV1::DigestMismatch {
            subject: ObservationDigestSubjectV1::ContentBlob,
            subject_id: "content:test".into(),
            expected_digest: "blob:expected".into(),
            rejected_digest: "blob:rejected".into(),
        },
        ObservationNackDetailV1::ImmutableProjectionConflict {
            projection_key: "receipt:test".into(),
            existing_digest: "sha256:existing".into(),
            rejected_digest: "sha256:rejected".into(),
        },
    ];
    let expected_kinds = [
        "receiver_unavailable",
        "unsupported_schema",
        "invalid_envelope",
        "missing_sequence_ranges",
        "sequence_event_conflict",
        "missing_prerequisite",
        "unknown_transcript_root",
        "missing_blob",
        "chunk_ordinal_conflict",
        "content_state_conflict",
        "digest_mismatch",
        "immutable_projection_conflict",
    ];
    for (detail, expected_kind) in details.into_iter().zip(expected_kinds) {
        let nack = ObservationNackV1 {
            schema_version: OBSERVATION_NACK_SCHEMA.into(),
            identity: feedback_identity(CONVERSATION_CONTENT_CHANNEL),
            rejected_sequence: 7,
            expected_sequence: 3,
            retryable: true,
            detail,
        };
        nack.validate().unwrap();
        let value = serde_json::to_value(&nack).unwrap();
        assert_eq!(value["detail"]["kind"], expected_kind);
        assert!(value.get("reason").is_none());
        assert_eq!(
            serde_json::from_value::<ObservationNackV1>(value).unwrap(),
            nack
        );
    }
}

#[test]
fn observation_nack_v1_denies_unknown_reason_and_detail() {
    let valid = json!({
        "schema_version": "hiroute.observation.nack/v1",
        "identity": {
            "channel": "execution_fact",
            "producer_id": "producer:test",
            "producer_epoch": "epoch:test",
            "stream_id": "stream:test"
        },
        "rejected_sequence": 7,
        "expected_sequence": 3,
        "retryable": true,
        "detail": {"kind": "receiver_unavailable", "retry_after_millis": 25}
    });
    assert!(serde_json::from_value::<ObservationNackV1>(valid.clone()).is_ok());

    let mut free_form_reason = valid.clone();
    free_form_reason
        .as_object_mut()
        .unwrap()
        .insert("reason".into(), json!("receiver failed"));
    assert!(serde_json::from_value::<ObservationNackV1>(free_form_reason).is_err());

    let mut unknown_detail_field = valid.clone();
    unknown_detail_field["detail"]
        .as_object_mut()
        .unwrap()
        .insert("opaque".into(), json!({}));
    assert!(serde_json::from_value::<ObservationNackV1>(unknown_detail_field).is_err());

    let mut unknown_detail_kind = valid;
    unknown_detail_kind["detail"]["kind"] = json!("free_form");
    assert!(serde_json::from_value::<ObservationNackV1>(unknown_detail_kind).is_err());
}

#[test]
fn observation_feedback_validation_fails_closed() {
    let mut acknowledgement = ObservationAckV2 {
        schema_version: OBSERVATION_ACK_SCHEMA.into(),
        identity: feedback_identity("execution_fact"),
        highest_contiguous_sequence: 8,
        highest_accounted_sequence: 7,
        content_acknowledgement: None,
    };
    assert_eq!(
        acknowledgement.validate(),
        Err(ObservationFeedbackValidationError::InvalidFrontier)
    );
    acknowledgement.highest_contiguous_sequence = 7;
    acknowledgement.content_acknowledgement = Some(content_acknowledgement());
    assert_eq!(
        acknowledgement.validate(),
        Err(ObservationFeedbackValidationError::InvalidContentChannel)
    );
    acknowledgement.identity = feedback_identity(CONVERSATION_CONTENT_CHANNEL);
    acknowledgement
        .content_acknowledgement
        .as_mut()
        .unwrap()
        .request_id = " ".into();
    assert!(matches!(
        acknowledgement.validate(),
        Err(ObservationFeedbackValidationError::EmptyField(_))
    ));
    acknowledgement.schema_version = "hiroute.observation.ack/v1".into();
    assert_eq!(
        acknowledgement.validate(),
        Err(ObservationFeedbackValidationError::UnsupportedSchema)
    );

    let mut nack = ObservationNackV1 {
        schema_version: OBSERVATION_NACK_SCHEMA.into(),
        identity: feedback_identity("execution_fact"),
        rejected_sequence: 7,
        expected_sequence: 3,
        retryable: true,
        detail: ObservationNackDetailV1::UnknownTranscriptRoot {
            request_id: "request:test".into(),
            direction: ObservationContentDirectionV1::RequestInput,
            fork_id: "fork:test".into(),
            transcript_root: "transcript:unknown".into(),
        },
    };
    assert_eq!(
        nack.validate(),
        Err(ObservationFeedbackValidationError::InvalidContentChannel)
    );
    nack.identity = feedback_identity(CONVERSATION_CONTENT_CHANNEL);
    nack.expected_sequence = 0;
    assert_eq!(
        nack.validate(),
        Err(ObservationFeedbackValidationError::InvalidSequence)
    );
    nack.expected_sequence = 3;
    nack.detail = ObservationNackDetailV1::MissingSequenceRanges {
        ranges: vec![
            ObservationSequenceRangeV1 {
                first_sequence: 5,
                last_sequence: 6,
            },
            ObservationSequenceRangeV1 {
                first_sequence: 4,
                last_sequence: 4,
            },
        ],
    };
    assert_eq!(
        nack.validate(),
        Err(ObservationFeedbackValidationError::InvalidSequenceRange)
    );
}

#[test]
fn default_gen_ai_mapper_has_standard_small_fields_and_zero_content() {
    let request = request(OtelContentPolicy::Disabled);
    let signal = OtelGenAiMapper::new(OtelContentPolicy::Disabled).attempt_span(
        &request,
        &OtelAttempt {
            ordinal: 1,
            attempt_id: "attempt:test".into(),
            stable_binding_id: "binding:test".into(),
            provider_name: "provider:test".into(),
            request_model: "native:test".into(),
        },
        "accepted",
        None,
    );
    let rendered = serde_json::to_string(&signal).unwrap();
    assert!(rendered.contains("gen_ai.operation.name"));
    assert!(rendered.contains("gen_ai.client.inference.operation.details"));
    for forbidden in [
        "prompt",
        "response content",
        "canonical_bytes_base64",
        "authorization",
        "gen_ai.input.messages",
        "gen_ai.output.messages",
    ] {
        assert!(!rendered.contains(forbidden));
    }
    assert!(
        OtelGenAiMapper::new(OtelContentPolicy::Disabled)
            .content_ref(
                "request_input",
                ContentRefV1 {
                    content_id: "content:test".into(),
                    digest: "digest:test".into(),
                    byte_count: 9,
                    media_type: "text/plain".into(),
                },
            )
            .is_none()
    );
}

#[test]
fn content_ref_opt_in_is_hiroute_extension_without_inline_bytes() {
    let signal = OtelGenAiMapper::new(OtelContentPolicy::ContentRefOnly)
        .content_ref(
            "response_delivered",
            ContentRefV1 {
                content_id: "content:test".into(),
                digest: "digest:test".into(),
                byte_count: 9,
                media_type: "text/plain".into(),
            },
        )
        .unwrap();
    let value = serde_json::to_value(signal).unwrap();
    assert_eq!(
        value.get("name"),
        Some(&Value::String("hiroute.gen_ai.response.content".into()))
    );
    assert!(value.get("content_ref").is_some());
    assert!(value.get("canonical_bytes_base64").is_none());
}

#[test]
fn mapper_and_envelope_versions_are_explicit() {
    assert!(EXECUTION_FACT_SCHEMA.ends_with("/v3"));
    assert_eq!(OTEL_MAPPER_VERSION, "hiroute.otel-gen-ai-mapper/1");
    assert_eq!(OTEL_SEMANTIC_CONVENTIONS_VERSION, "1.37.0");
    assert!(OTEL_MAPPING_DIGEST.starts_with("sha256:"));
    assert_eq!(schema_digest(OTEL_MAPPING_CONTRACT), OTEL_MAPPING_DIGEST);
    for schema in [
        LIFECYCLE_FACT_SCHEMA,
        CONVERSATION_CONTENT_SCHEMA,
        OBSERVATION_ACK_SCHEMA,
    ] {
        assert!(schema.ends_with("/v2"));
    }
    assert!(EXECUTION_FACT_SCHEMA.ends_with("/v3"));
    for schema in [
        OTEL_GEN_AI_SCHEMA,
        OBSERVATION_NACK_SCHEMA,
        OBSERVATION_GAP_HEARTBEAT_SCHEMA,
    ] {
        assert!(schema.ends_with("/v1"));
    }
}

#[test]
fn native_alias_repeated_reads_preserve_the_staged_relay_identity() {
    let request = request(OtelContentPolicy::Disabled);
    let digest = hiroute_domain::CanonicalDigest::of_bytes(b"native-profile");
    let native_alias =
        crate::runtime::native_endpoint_state_key("binding:test", digest.as_str()).unwrap();
    let candidate = super::request::CandidateObservation {
        candidate_id: "candidate:test".into(),
        stable_binding_id: "binding:test".into(),
        declared_order: 0,
        profile_digest: digest.to_string(),
        provider_name: "provider:test".into(),
        request_model: "native:test".into(),
        upstream_protocol: "responses".into(),
        model_configuration_id: "model:test".into(),
        adapter_revision: "adapter:test@1".into(),
        effective_cost_micros: None,
        cost_class: "unknown".into(),
        protocol_profile: None,
        streaming: true,
    };
    {
        let mut state = request.lock_state();
        state
            .candidates
            .insert(native_alias.clone(), candidate.clone());
        state
            .candidates
            .insert("binding:test".into(), candidate.clone());
        state.candidates.insert(
            "binding:other".into(),
            super::request::CandidateObservation {
                stable_binding_id: "binding:other".into(),
                ..candidate
            },
        );
        state.previous_attempt_id = Some("attempt:previous".into());
        state.next_attempt_reason = Some("protocol".into());
    }
    let key = RuntimeStateKey::credential(native_alias.as_str(), "credential:test", "key:test", 4);
    let active = RuntimeStateEntry::default();
    request.runtime_state_read(&key, Some(&active), "active");
    request.runtime_state_read(&key, Some(&active), "active");
    // A logical-key read is the same candidate as its native state alias.
    request.runtime_state_read(
        &RuntimeStateKey::credential("binding:test", "credential:test", "key:test", 4),
        Some(&active),
        "active",
    );
    {
        let state = request.lock_state();
        let pending = state.pending_attempt.as_ref().unwrap();
        assert_eq!(pending.stable_binding_id, "binding:test");
        assert_eq!(
            pending.previous_attempt_id.as_deref(),
            Some("attempt:previous")
        );
        assert_eq!(pending.start_reason, "protocol");
        assert_eq!(pending.credential_ref, "credential:test");
        assert_eq!(pending.key_id, "key:test");
        assert_eq!(pending.credential_generation, 4);
        assert_eq!(state.next_attempt_ordinal, 1);
    }
    // Sharing a credential does not collapse a genuinely different binding.
    request.runtime_state_read(
        &RuntimeStateKey::credential("binding:other", "credential:test", "key:test", 4),
        Some(&active),
        "active",
    );
    assert_eq!(
        request
            .lock_state()
            .pending_attempt
            .as_ref()
            .unwrap()
            .stable_binding_id,
        "binding:other"
    );
}

#[test]
fn acquired_probe_stages_and_authoritative_publication_starts_the_attempt() {
    let request = request(OtelContentPolicy::Disabled);
    let key = RuntimeStateKey::credential("binding:test", "credential:test", "key:test", 4);
    request.runtime_probe(
        &key,
        7,
        std::time::Duration::from_secs(5),
        Some(ProbeLeaseOutcome::Busy),
        "busy",
    );
    assert!(request.lock_state().pending_attempt.is_none());

    request.runtime_probe(
        &key,
        7,
        std::time::Duration::from_secs(5),
        Some(ProbeLeaseOutcome::Acquired { generation: 8 }),
        "acquired",
    );
    assert!(request.lock_state().current_attempt.is_none());
    assert!(request.lock_state().pending_attempt.is_some());
    request.disposition_published(&PublishedDisposition {
        request_id: RequestId(9),
        attempt_id: AttemptId(4),
        generation: AttemptGeneration(2),
        disposition: Disposition::Accept,
    });
    let state = request.lock_state();
    let attempt = state
        .current_attempt
        .as_ref()
        .expect("core publication promotes the staged candidate to an Attempt");
    assert_eq!(attempt.ordinal, 1);
    assert_eq!(attempt.start_reason, "initial_candidate");
    assert!(attempt.attempt_id.starts_with("attempt-"));
}

#[test]
fn explicit_no_credential_materialization_stages_the_authorized_attempt() {
    let request = request(OtelContentPolicy::Disabled);
    request.no_credential_materialized("binding:test", "credential/none/source-local-test");
    {
        let state = request.lock_state();
        let pending = state
            .pending_attempt
            .as_ref()
            .expect("verified unauthenticated materialization stages an observation attempt");
        assert_eq!(pending.credential_ref, "credential/none/source-local-test");
        assert_eq!(pending.key_id, pending.credential_ref);
        assert_eq!(pending.credential_generation, 0);
    }

    request.disposition_published(&PublishedDisposition {
        request_id: RequestId(9),
        attempt_id: AttemptId(4),
        generation: AttemptGeneration(2),
        disposition: Disposition::Accept,
    });

    let state = request.lock_state();
    assert_eq!(state.next_attempt_ordinal, 2);
    assert_eq!(
        state
            .current_attempt
            .as_ref()
            .map(|attempt| attempt.ordinal),
        Some(1)
    );
}

#[test]
fn pending_candidate_without_core_attempt_authority_finishes_with_zero_attempts() {
    let request = request(OtelContentPolicy::Disabled);
    let key = RuntimeStateKey::credential("binding:test", "credential:test", "key:test", 4);
    request.runtime_probe(
        &key,
        7,
        std::time::Duration::from_secs(5),
        Some(ProbeLeaseOutcome::Acquired { generation: 8 }),
        "acquired",
    );
    assert!(request.lock_state().pending_attempt.is_some());

    request.finish("gateway_error");

    let state = request.lock_state();
    assert!(state.request_finished);
    assert!(state.pending_attempt.is_none());
    assert_eq!(state.next_attempt_ordinal, 1);
    assert_eq!(state.attempts_finished, 0);
}

#[test]
fn cpa_execution_metadata_is_trusted_only_for_verified_managed_target() {
    use hiroute_diagnostics::event::CpaExecution;
    let observation = request(OtelContentPolicy::Disabled);
    let mut headers = http::HeaderMap::new();
    headers.insert("x-hiroute-cpa-execution", "v1;2;1;1;1".parse().unwrap());
    observation.response_head_diagnostic(&headers, 200);
    assert!(
        observation
            .lock_state()
            .response_wire
            .as_ref()
            .unwrap()
            .cpa_execution
            .is_none()
    );
    observation.lock_state().managed_cpa = true;
    observation.response_head_diagnostic(&headers, 200);
    assert!(matches!(
        observation
            .lock_state()
            .response_wire
            .as_ref()
            .unwrap()
            .cpa_execution,
        Some(CpaExecution::Known {
            inference_attempts: 2,
            auth_recovery_successes: 1,
            ..
        })
    ));
    observation.response_head_diagnostic(&http::HeaderMap::new(), 503);
    assert_eq!(
        observation
            .lock_state()
            .response_wire
            .as_ref()
            .unwrap()
            .cpa_execution,
        Some(CpaExecution::Unknown)
    );
}