Lightweight coding agent that runs in your terminal
Rust
Add app-server coverage for plugin measurement analytics (#38278)
Kyle Brownc38d59fca023
diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rsindex 2acc86caee..c0ef692285 100644--- a/codex-rs/app-server/tests/suite/v2/analytics.rs+++ b/codex-rs/app-server/tests/suite/v2/analytics.rs@@ -1,15 +1,37 @@use anyhow::Result;use app_test_support::ChatGptAuthFixture;use app_test_support::DEFAULT_CLIENT_NAME;+use app_test_support::TestAppServer;+use app_test_support::create_final_assistant_message_sse_response;+use app_test_support::create_mock_responses_server_sequence;+use app_test_support::create_shell_command_sse_response;+use app_test_support::to_response;use app_test_support::write_chatgpt_auth;+use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url;+use codex_app_server_protocol::JSONRPCResponse;+use codex_app_server_protocol::RequestId;+use codex_app_server_protocol::SandboxPolicy;+use codex_app_server_protocol::ThreadStartParams;+use codex_app_server_protocol::ThreadStartResponse;+use codex_app_server_protocol::TurnStartParams;+use codex_app_server_protocol::UserInput;use codex_config::types::AuthCredentialsStoreMode;use codex_config::types::OtelExporterKind;use codex_config::types::OtelHttpProtocol;use codex_core::config::ConfigBuilder;+use codex_core_plugins::loader::curated_plugin_cache_version;+use codex_core_plugins::store::PluginStore;+use codex_plugin::PluginId;+use core_test_support::responses;+use core_test_support::skip_if_no_network;+use core_test_support::skip_if_remote;+use core_test_support::skip_if_wine_exec;use pretty_assertions::assert_eq;use serde_json::Value;+use serde_json::json;use std::collections::HashMap;use std::path::Path;+use std::path::PathBuf;use std::time::Duration;use tempfile::TempDir;use tokio::time::timeout;@@ -230,3 +252,335 @@ pub(crate) fn assert_basic_thread_initialized_event();assert!(event["event_params"]["created_at"].as_u64().is_some());}++const METRICS_PLUGIN_ID: &str = "sample@openai-curated";+const TEST_CURATED_PLUGIN_SHA: &str = "0123456789abcdef0123456789abcdef01234567";++#[derive(Clone, Copy)]+enum PluginMetricsRuntime {+ Classic,+ Unified,+ UnifiedBackground,+}++fn write_curated_metrics_plugin(codex_home: &Path) -> Result<PathBuf> {+ let plugin_id = PluginId::parse(METRICS_PLUGIN_ID)?;+ let plugin_root = PluginStore::new(codex_home.to_path_buf()).plugin_root(+ &plugin_id,+ &curated_plugin_cache_version(TEST_CURATED_PLUGIN_SHA),+ );+ let script_path = plugin_root.join("scripts/run.sh");+ std::fs::create_dir_all(plugin_root.join(".codex-plugin"))?;+ std::fs::create_dir_all(script_path.parent().expect("script path has parent"))?;+ std::fs::write(+ plugin_root.join(".codex-plugin/plugin.json"),+ r#"{"name":"sample","version":"0.1.0"}"#,+ )?;+ std::fs::write(+ plugin_root.join("analytics.yaml"),+ r#"version: 1+operations:+ scan:+ path: ./scripts/run.sh+ measurements:+ findings:+ dimensions:+ severity: [high, low]+ files_scanned: {}+"#,+ )?;+ std::fs::write(+ &script_path,+ r#"test -n "$CODEX_PLUGIN_METRICS_OUTPUT"+sleep "${1:-0.3}"+printf '%s' '{"version":1,"measurements":[{"name":"findings","value":3,"dimensions":{"severity":"high"}},{"name":"files_scanned","value":17}]}' > "$CODEX_PLUGIN_METRICS_OUTPUT"+"#,+ )?;++ let curated_repo = codex_home.join(".tmp/plugins");+ std::fs::create_dir_all(curated_repo.join(".agents/plugins"))?;+ std::fs::write(+ curated_repo.join(".agents/plugins/marketplace.json"),+ r#"{+ "name": "openai-curated",+ "plugins": [+ {"name": "sample", "source": {"source": "local", "path": "./plugins/sample"}}+ ]+}"#,+ )?;+ std::fs::write(+ codex_home.join(".tmp/plugins.sha"),+ format!("{TEST_CURATED_PLUGIN_SHA}\n"),+ )?;+ Ok(script_path.into_path_buf())+}++async fn assert_plugin_measurement_analytics(runtime: PluginMetricsRuntime) -> Result<()> {+ skip_if_no_network!(Ok(()));+ skip_if_remote!(+ Ok(()),+ "trusted plugin metrics fixture uses a local Codex home cache"+ );+ skip_if_wine_exec!(Ok(()), "plugin metrics fixture is Unix-only");++ let codex_home = TempDir::new()?;+ let script_path = write_curated_metrics_plugin(codex_home.path())?;+ let background = matches!(runtime, PluginMetricsRuntime::UnifiedBackground);+ let unified_exec = !matches!(runtime, PluginMetricsRuntime::Classic);+ let mut command = vec![+ "/bin/sh".to_string(),+ script_path.to_string_lossy().into_owned(),+ ];+ if background {+ command.push("1.0".to_string());+ }+ let call_id = "curated-plugin-metrics";+ let command_response = match runtime {+ PluginMetricsRuntime::Classic => {+ create_shell_command_sse_response(command, /*workdir*/ None, Some(5_000), call_id)?+ }+ PluginMetricsRuntime::Unified | PluginMetricsRuntime::UnifiedBackground => {+ let arguments = serde_json::to_string(&json!({+ "cmd": shlex::try_join(command.iter().map(String::as_str))?,+ "yield_time_ms": if background { 10 } else { 1_000 },+ }))?;+ responses::sse(vec![+ responses::ev_response_created("resp-1"),+ responses::ev_function_call(call_id, "exec_command", &arguments),+ responses::ev_completed("resp-1"),+ ])+ }+ };+ let final_response = create_final_assistant_message_sse_response("done")?;+ let server =+ create_mock_responses_server_sequence(vec![command_response, final_response]).await;++ let analytics_server = responses::start_mock_server().await;+ write_mock_responses_config_toml_with_chatgpt_base_url(+ codex_home.path(),+ &server.uri(),+ &analytics_server.uri(),+ )?;+ let config_path = codex_home.path().join("config.toml");+ let config = std::fs::read_to_string(&config_path)?;+ std::fs::write(+ config_path,+ format!(+ r#"{config}+[features]+plugins = true+remote_plugin = false+unified_exec = {unified_exec}+shell_zsh_fork = false+unified_exec_zsh_fork = false++[plugins."{METRICS_PLUGIN_ID}"]+enabled = true+"#,+ ),+ )?;+ mount_analytics_capture(&analytics_server, codex_home.path()).await?;++ let mut mcp = TestAppServer::builder()+ .with_codex_home(codex_home.path())+ .without_managed_config()+ .build()+ .await?;+ timeout(Duration::from_secs(10), mcp.initialize()).await??;+ let thread_request = mcp+ .send_thread_start_request_with_auto_env(ThreadStartParams {+ model: Some("mock-model".to_string()),+ ..Default::default()+ })+ .await?;+ let thread_response: JSONRPCResponse = timeout(+ Duration::from_secs(10),+ mcp.read_stream_until_response_message(RequestId::Integer(thread_request)),+ )+ .await??;+ let ThreadStartResponse { thread, .. } = to_response(thread_response)?;+ let thread_id = thread.id.clone();++ let turn_request = mcp+ .send_turn_start_request(TurnStartParams {+ thread_id: thread.id,+ input: vec![UserInput::Text {+ text: "run the curated plugin metrics script".to_string(),+ text_elements: Vec::new(),+ }],+ sandbox_policy: Some(SandboxPolicy::ReadOnly {+ network_access: false,+ }),+ ..Default::default()+ })+ .await?;+ timeout(+ Duration::from_secs(10),+ mcp.read_stream_until_response_message(RequestId::Integer(turn_request)),+ )+ .await??;+ let completed_turn = timeout(+ Duration::from_secs(10),+ mcp.read_stream_until_notification_message("turn/completed"),+ )+ .await??;+ let turn_id = completed_turn+ .params+ .as_ref()+ .and_then(|params| params["turn"]["id"].as_str())+ .expect("completed turn id");++ if background {+ let model_request_bodies = server+ .received_requests()+ .await+ .unwrap_or_default()+ .into_iter()+ .filter(|request| request.url.path().ends_with("/responses"))+ .map(|request| serde_json::from_slice::<Value>(&request.body))+ .collect::<Result<Vec<_>, _>>()?;+ let request_text = serde_json::to_string(&model_request_bodies)?;+ assert!(request_text.contains("Process running with session ID "));+ assert!(!request_text.contains("Process exited with code 0"));+ }++ for measurement_name in ["findings", "files_scanned"] {+ wait_for_matching_analytics_event(&analytics_server, Duration::from_secs(10), |event| {+ event["event_type"] == "codex_plugin_measurement_event"+ && event["event_params"]["item_id"] == call_id+ && event["event_params"]["measurement_name"] == measurement_name+ })+ .await?;+ }+ let command_event =+ wait_for_matching_analytics_event(&analytics_server, Duration::from_secs(10), |event| {+ event["event_type"] == "codex_command_execution_event"+ && event["event_params"]["item_id"] == call_id+ })+ .await?;+ assert_eq!(+ json!({+ "plugin_id": command_event["event_params"]["plugin_id"],+ "script_path": command_event["event_params"]["script_path"],+ "item_id": command_event["event_params"]["item_id"],+ "exit_code": command_event["event_params"]["exit_code"],+ }),+ json!({+ "plugin_id": METRICS_PLUGIN_ID,+ "script_path": "scripts/run.sh",+ "item_id": call_id,+ "exit_code": 0,+ })+ );+ let mut measurement_events = Vec::new();+ for request in analytics_server+ .received_requests()+ .await+ .unwrap_or_default()+ {+ if request.method != "POST" || request.url.path() != "/codex/analytics-events/events" {+ continue;+ }+ let payload: Value = serde_json::from_slice(&request.body)?;+ let Some(events) = payload["events"].as_array() else {+ continue;+ };+ measurement_events.extend(+ events+ .iter()+ .filter(|event| {+ event["event_type"] == "codex_plugin_measurement_event"+ && event["event_params"]["item_id"] == call_id+ })+ .cloned(),+ );+ }+ measurement_events.sort_by(|left, right| {+ left["event_params"]["measurement_name"]+ .as_str()+ .cmp(&right["event_params"]["measurement_name"].as_str())+ });+ assert_eq!(measurement_events.len(), 2);+ let execution_id = measurement_events[0]["event_params"]["execution_id"]+ .as_str()+ .expect("measurement execution id");+ assert!(!execution_id.is_empty());+ assert_eq!(+ measurement_events[1]["event_params"]["execution_id"].as_str(),+ Some(execution_id)+ );+ assert_eq!(+ measurement_events+ .iter()+ .map(|event| json!({+ "plugin_id": event["event_params"]["plugin_id"],+ "operation": event["event_params"]["operation"],+ "measurement_name": event["event_params"]["measurement_name"],+ "number_value": event["event_params"]["number_value"],+ "dimensions": event["event_params"]["dimensions"],+ "item_id": event["event_params"]["item_id"],+ }))+ .collect::<Vec<_>>(),+ vec![+ json!({+ "plugin_id": METRICS_PLUGIN_ID,+ "operation": "scan",+ "measurement_name": "files_scanned",+ "number_value": 17.0,+ "dimensions": null,+ "item_id": call_id,+ }),+ json!({+ "plugin_id": METRICS_PLUGIN_ID,+ "operation": "scan",+ "measurement_name": "findings",+ "number_value": 3.0,+ "dimensions": {"severity": "high"},+ "item_id": call_id,+ }),+ ]+ );+ for event in &measurement_events {+ let event_params = event["event_params"]+ .as_object()+ .expect("measurement event params");+ let mut field_names = event_params.keys().map(String::as_str).collect::<Vec<_>>();+ field_names.sort_unstable();+ assert_eq!(+ field_names,+ vec![+ "dimensions",+ "execution_id",+ "item_id",+ "measurement_name",+ "number_value",+ "operation",+ "plugin_id",+ "thread_id",+ "turn_id",+ ]+ );+ assert_eq!(event_params["thread_id"], thread_id);+ assert_eq!(event_params["turn_id"], turn_id);+ }++ Ok(())+}++#[cfg_attr(windows, ignore = "plugin metrics fixture is Unix-only")]+#[tokio::test]+async fn classic_plugin_script_emits_measurement_analytics() -> Result<()> {+ assert_plugin_measurement_analytics(PluginMetricsRuntime::Classic).await+}++#[cfg_attr(windows, ignore = "plugin metrics fixture is Unix-only")]+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn unified_plugin_script_emits_measurement_analytics() -> Result<()> {+ assert_plugin_measurement_analytics(PluginMetricsRuntime::Unified).await+}++#[cfg_attr(windows, ignore = "plugin metrics fixture is Unix-only")]+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn unified_background_plugin_script_emits_measurements_after_turn_completion() -> Result<()> {+ assert_plugin_measurement_analytics(PluginMetricsRuntime::UnifiedBackground).await+}