Skip to content
Python
15.8 KB
Raw Blame
//! Real loopback SSE through the same request boundary used by every model purpose.
//! No model keys, cloud traffic or tool execution. Usage snapshots replace, never add.
use openworker_runtime::{
    connections::{Choice, Manager},
    model::{CallContext, Model},
    provider_auth::{self, AccessToken},
    telemetry::*,
};
use serde_json::{json, Value};
use std::{
    sync::{Arc, Mutex},
    time::Duration,
};
use tokio::io::{AsyncReadExt, AsyncWriteExt};

struct CaptureSink(Arc<Mutex<Vec<Record>>>);
impl Sink for CaptureSink {
    fn export(&mut self, records: &[Record], _: Snapshot) -> bool {
        self.0.lock().unwrap().extend_from_slice(records);
        true
    }
}
async fn endpoint(responses: Vec<(u16, String)>) -> (String, tokio::task::JoinHandle<()>) {
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let url = format!("http://{}/v1", listener.local_addr().unwrap());
    let task = tokio::spawn(async move {
        for (status, body) in responses {
            let (mut s, _) = listener.accept().await.unwrap();
            let mut raw = vec![];
            let mut buf = [0; 4096];
            loop {
                let n = s.read(&mut buf).await.unwrap();
                assert!(n > 0);
                raw.extend_from_slice(&buf[..n]);
                if let Some(end) = raw.windows(4).position(|w| w == b"\r\n\r\n") {
                    let h = String::from_utf8_lossy(&raw[..end]).to_ascii_lowercase();
                    let len: usize = h
                        .lines()
                        .find_map(|s| s.strip_prefix("content-length: "))
                        .unwrap()
                        .parse()
                        .unwrap();
                    if raw.len() >= end + 4 + len {
                        break;
                    }
                }
                assert!(raw.len() < 65536);
            }
            s.write_all(format!("HTTP/1.1 {status} Fixture\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",body.len()).as_bytes()).await.unwrap();
            // First-byte timing must survive transport fragmentation.
            for part in body.as_bytes().chunks(13) {
                s.write_all(part).await.unwrap();
            }
        }
    });
    (url, task)
}
fn sse(events: Vec<Value>) -> String {
    events
        .into_iter()
        .map(|v| format!("data: {v}\n\n"))
        .collect()
}
fn chat(text: &str, usage: bool) -> String {
    let mut frames = vec![
        json!({"choices":[{"index":0,"delta":{"content":text},"finish_reason":null}]}),
        json!({"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}),
    ];
    if usage {
        frames.push(json!({"choices":[],"usage":{"prompt_tokens":100,"completion_tokens":20,"prompt_tokens_details":{"cached_tokens":40},"completion_tokens_details":{"reasoning_tokens":5}}}));
    }
    sse(frames) + "data: [DONE]\n\n"
}
fn model(
    manager: &Arc<Manager>,
    api: &str,
    url: &str,
    t: &Telemetry,
    purpose: ModelPurpose,
) -> Model {
    manager
        .set_provider(json!({"name":"Fixture","api":api,"base_url":url}))
        .unwrap();
    let selection = manager
        .resolve(&Choice::new("Fixture", "fixture-model"))
        .unwrap();
    let mut m = Model::new(selection.model.clone(), false);
    m.connection = Some((manager.clone(), selection));
    m.accounted(CallContext {
        telemetry: t.clone(),
        session: "PRIVATE_SESSION".into(),
        turn: "PRIVATE_TURN".into(),
        action: "request-1".into(),
        purpose,
        attempt: 1,
    })
}
async fn next(m: &Model) -> bool {
    let (tx, mut rx) = tokio::sync::mpsc::channel(32);
    let collect = async { while rx.recv().await.is_some() {} };
    let history = [json!({"role":"user","content":"PRIVATE_PROMPT"})];
    let (result, _) = tokio::join!(m.next("PRIVATE_INSTRUCTIONS", &history, &[], tx), collect);
    result.is_ok()
}
fn telemetry() -> (Telemetry, Controller, Arc<Mutex<Vec<Record>>>) {
    let rows = Arc::new(Mutex::new(vec![]));
    let (t, c) = Telemetry::start_with_content(
        CaptureSink(rows.clone()),
        128,
        ContentPolicy {
            capture: Capture::Metadata,
            ..Default::default()
        },
    )
    .unwrap();
    (t, c, rows)
}

#[tokio::test]
async fn full_content_exports_exclude_google_replay_signatures() {
    use openworker_runtime::{model::Progress, provider::Completion};
    let rows = Arc::new(Mutex::new(vec![]));
    let (t, c) = Telemetry::start_with_content(
        CaptureSink(rows.clone()),
        128,
        ContentPolicy {
            capture: Capture::Full,
            ..Default::default()
        },
    )
    .unwrap();
    let root = tempfile::tempdir().unwrap();
    let manager = Arc::new(Manager::open(root.path()).unwrap());
    let signed_tool = sse(vec![
        json!({"choices":[{"index":0,"delta":{"tool_calls":[{"id":"call-1","type":"function","function":{"name":"app__lookup","arguments":"{}"},"extra_content":{"google":{"thought_signature":"OPAQUE_TOOL_SIGNATURE"}}}]},"finish_reason":"stop"}]}),
    ]) + "data: [DONE]\n\n";
    let signed_text = sse(vec![
        json!({"choices":[{"index":0,"delta":{"content":"Public answer","extra_content":{"google":{"thought_signature":"OPAQUE_TEXT_SIGNATURE"}}},"finish_reason":"stop"}]}),
    ]) + "data: [DONE]\n\n";
    let (url, server) = endpoint(vec![(200, signed_tool), (200, signed_text)]).await;
    let m = model(&manager, "chat-completions", &url, &t, ModelPurpose::Main);
    async fn complete(model: &Model, history: &[Value]) -> Completion {
        let (tx, mut rx) = tokio::sync::mpsc::channel(32);
        let collect = async {
            let mut completion = None;
            while let Some(progress) = rx.recv().await {
                if let Progress::Complete(value) = progress {
                    completion = Some(value);
                }
            }
            completion.unwrap()
        };
        let (result, completion) =
            tokio::join!(model.next("Instructions", history, &[], tx), collect);
        result.unwrap();
        completion
    }
    let mut history = vec![json!({"role":"user","content":"Public question"})];
    history.extend(complete(&m, &history).await.output);
    history.push(json!({"type":"function_call_output","call_id":"call-1","output":"available"}));
    let second = complete(&m, &history).await;
    assert!(serde_json::to_string(&second.output)
        .unwrap()
        .contains("OPAQUE_TEXT_SIGNATURE"));
    server.await.unwrap();
    assert!(c.shutdown(Duration::from_secs(3)));
    let exported = format!("{:?}", rows.lock().unwrap());
    assert!(!exported.contains("OPAQUE_"));
    assert!(exported.contains("Public answer"));
    assert!(exported.contains("Public question"));
}
#[tokio::test]
async fn all_real_model_purposes_keep_usage_once_and_share_turn() {
    let (t, c, rows) = telemetry();
    let root = tempfile::tempdir().unwrap();
    let mgr = Arc::new(Manager::open(root.path()).unwrap());
    let (url, server) = endpoint(vec![
        (200, chat("Answer", true)),
        (200, chat(r#"{"decision":"ask","reason":"Check"}"#, true)),
        (200, chat("Summary", true)),
    ])
    .await;
    let m = model(&mgr, "chat-completions", &url, &t, ModelPurpose::Main);
    assert!(next(&m).await);
    let review = m.clone().accounted(CallContext {
        purpose: ModelPurpose::AutoApprover,
        action: "review".into(),
        ..m.observation.clone().unwrap()
    });
    review.review("PRIVATE_REVIEW").await;
    let summary = m.clone().accounted(CallContext {
        purpose: ModelPurpose::Compaction,
        action: "summary".into(),
        ..m.observation.clone().unwrap()
    });
    assert_eq!(
        summary
            .summarize("PRIVATE_SYSTEM", "PRIVATE_CONTEXT", 256)
            .await
            .unwrap(),
        "Summary"
    );
    server.await.unwrap();
    assert!(c.shutdown(Duration::from_secs(3)));
    let r = rows.lock().unwrap();
    let calls: Vec<_> = r
        .iter()
        .filter(|r| r.operation == Operation::Model)
        .collect();
    assert_eq!(calls.len(), 3);
    assert_eq!(
        r.iter()
            .filter(|r| r.operation == Operation::ModelStarted)
            .count(),
        3
    );
    let fixture: Value =
        serde_json::from_str(include_str!("../../tests/contracts/observability.json")).unwrap();
    assert_eq!(
        calls.iter().map(|r| r.input_tokens.unwrap()).sum::<u64>(),
        fixture["three_calls"]["input"].as_u64().unwrap()
    );
    assert_eq!(
        calls.iter().map(|r| r.output_tokens.unwrap()).sum::<u64>(),
        fixture["three_calls"]["output"].as_u64().unwrap()
    );
    for call in &calls {
        assert_eq!(
            call.usage_details,
            TokenUsage {
                input: Some(100),
                output: Some(20),
                cache_read: Some(40),
                reasoning: Some(5),
                ..Default::default()
            }
        );
        assert!(call.first_output.is_some());
        assert_eq!(call.context.trace, calls[0].context.trace);
        assert_eq!(call.turn_hash, calls[0].turn_hash);
    }
    assert_eq!(
        calls
            .iter()
            .map(|r| r.purpose.unwrap().name())
            .collect::<Vec<_>>(),
        vec!["main", "auto_approver", "compaction"]
    );
    assert!(!format!("{r:?}").contains("PRIVATE"));
    assert!(r
        .iter()
        .filter(|r| r.operation == Operation::ModelStarted)
        .all(|r| r.input_tokens.is_none() && calls.iter().any(|c| Some(c.context) == r.parent)));
}
#[tokio::test]
async fn anthropic_partial_usage_survives_truncation_without_authorizing_output() {
    let (t, c, rows) = telemetry();
    let root = tempfile::tempdir().unwrap();
    let mgr = Arc::new(Manager::open(root.path()).unwrap());
    let start = json!({"type":"message_start","message":{"role":"assistant","content":[],"usage":{"input_tokens":20,"cache_read_input_tokens":10,"cache_creation_input_tokens":5}}});
    let truncated = sse(vec![
        start.clone(),
        json!({"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call1","name":"write_file","input":{}}}),
        json!({"type":"message_delta","delta":{"stop_reason":"max_tokens"},"usage":{"output_tokens":12}}),
    ]);
    let (url, server) = endpoint(vec![(200, sse(vec![start])), (200, truncated)]).await;
    let m = model(&mgr, "anthropic", &url, &t, ModelPurpose::Main);
    assert!(!next(&m).await);
    assert!(!next(&m).await);
    server.await.unwrap();
    assert!(c.shutdown(Duration::from_secs(3)));
    let r = rows.lock().unwrap();
    let calls: Vec<_> = r
        .iter()
        .filter(|r| r.operation == Operation::Model)
        .collect();
    assert_eq!(calls.len(), 2);
    assert_eq!(
        calls[0].usage_details,
        TokenUsage {
            input: Some(35),
            output: None,
            cache_read: Some(10),
            cache_write: Some(5),
            reasoning: None
        }
    );
    assert_eq!(calls[0].first_output, None);
    assert_eq!(calls[1].output_tokens, Some(12));
    assert!(calls[1].first_output.is_some());
    assert!(calls.iter().all(|r| r.outcome == Outcome::Error));
}
#[tokio::test]
async fn responses_failed_request_keeps_reported_cost_and_tool_first_output() {
    let (t, c, rows) = telemetry();
    let root = tempfile::tempdir().unwrap();
    let mgr = Arc::new(Manager::open(root.path()).unwrap());
    let body = sse(vec![
        json!({"type":"response.created"}),
        json!({"type":"response.output_item.added","item":{"type":"function_call","name":"write_file"}}),
        json!({"type":"response.incomplete","response":{"incomplete_details":{"reason":"max_output_tokens"},"usage":{"input_tokens":100,"output_tokens":20,"input_tokens_details":{"cached_tokens":40},"output_tokens_details":{"reasoning_tokens":5}}}}),
    ]);
    let (url, server) = endpoint(vec![(200, body)]).await;
    let m = model(&mgr, "responses", &url, &t, ModelPurpose::Main);
    assert!(!next(&m).await);
    server.await.unwrap();
    assert!(c.shutdown(Duration::from_secs(3)));
    let r = rows.lock().unwrap();
    let call = r.iter().find(|r| r.operation == Operation::Model).unwrap();
    assert_eq!(call.outcome, Outcome::Error);
    assert_eq!(call.input_tokens, Some(100));
    assert_eq!(call.usage_details.reasoning, Some(5));
    assert!(call.first_output.is_some());
}
#[tokio::test]
async fn authentication_refresh_records_both_http_attempts_and_missing_usage_is_unknown() {
    let (t, c, rows) = telemetry();
    let root = tempfile::tempdir().unwrap();
    let mgr = Arc::new(Manager::open(root.path()).unwrap());
    let (url, server) = endpoint(vec![
        (401, r#"{"error":{"code":"invalid_api_key"}}"#.into()),
        (200, chat("OK", false)),
    ])
    .await;
    let m = model(&mgr, "chat-completions", &url, &t, ModelPurpose::Main);
    mgr.register_access_auth(
        "Fixture",
        Some(Arc::new(|reason| {
            assert!(matches!(reason, provider_auth::Reason::Rejected));
            Box::pin(async {
                AccessToken::new("synthetic-replacement", provider_auth::now() + 900)
            })
        })),
    )
    .unwrap();
    mgr.update_access_token(
        "Fixture",
        AccessToken::new("synthetic-original", provider_auth::now() + 900).unwrap(),
    )
    .unwrap();
    assert!(next(&m).await);
    server.await.unwrap();
    assert!(c.shutdown(Duration::from_secs(3)));
    let r = rows.lock().unwrap();
    let calls: Vec<_> = r
        .iter()
        .filter(|r| r.operation == Operation::Model)
        .collect();
    assert_eq!(calls.len(), 2);
    assert_eq!(calls[0].outcome, Outcome::Error);
    assert_eq!(calls[1].outcome, Outcome::Ok);
    assert_ne!(calls[0].context.span, calls[1].context.span);
    assert_eq!(calls[0].identity.action, calls[1].identity.action);
    assert!(calls
        .iter()
        .all(|r| r.input_tokens.is_none() && r.output_tokens.is_none()));
    assert!(!format!("{r:?}").contains("synthetic-"));
}

#[tokio::test]
async fn cancellation_keeps_partial_usage_and_interrupted_outcome() {
    let (t, c, rows) = telemetry();
    let root = tempfile::tempdir().unwrap();
    let mgr = Arc::new(Manager::open(root.path()).unwrap());
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let url = format!("http://{}/v1", listener.local_addr().unwrap());
    let server = tokio::spawn(async move {
        let (mut socket, _) = listener.accept().await.unwrap();
        let mut raw = [0; 8192];
        assert!(socket.read(&mut raw).await.unwrap() > 0);
        let start = sse(vec![
            json!({"type":"message_start","message":{"role":"assistant","content":[],"usage":{"input_tokens":35,"output_tokens":0}}}),
        ]);
        socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 100000\r\n\r\n{start}").as_bytes()).await.unwrap();
        std::future::pending::<()>().await;
    });
    let m = model(&mgr, "anthropic", &url, &t, ModelPurpose::Main);
    let (tx, mut rx) = tokio::sync::mpsc::channel(32);
    let request = tokio::spawn(async move {
        m.next(
            "private",
            &[json!({"role":"user","content":"private"})],
            &[],
            tx,
        )
        .await
    });
    let progress = tokio::time::timeout(Duration::from_secs(5), rx.recv())
        .await
        .unwrap()
        .unwrap();
    assert!(matches!(
        progress,
        openworker_runtime::model::Progress::Usage(TokenUsage {
            input: Some(35),
            output: Some(0),
            ..
        })
    ));
    request.abort();
    let _ = request.await;
    server.abort();
    let _ = server.await;
    assert!(c.shutdown(Duration::from_secs(3)));
    let r = rows.lock().unwrap();
    let call = r.iter().find(|r| r.operation == Operation::Model).unwrap();
    assert_eq!(call.outcome, Outcome::Interrupted);
    assert_eq!(call.input_tokens, Some(35));
    assert_eq!(call.output_tokens, Some(0));
    assert!(call.first_output.is_none());
}