Lightweight coding agent that runs in your terminal
Rust
Track history notes thread hint outcomes (#42247)
pmccrary-oaibdfd76964092
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lockindex f9c9089a83..5c3da721a4 100644--- a/codex-rs/Cargo.lock+++ b/codex-rs/Cargo.lock@@ -3425,6 +3425,7 @@ dependencies = [name = "codex-history-notes-extension"version = "0.0.0"dependencies = [+ "codex-analytics","codex-api","codex-client","codex-config",diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rsindex 96dce42750..6895825c87 100644--- a/codex-rs/analytics/src/client.rs+++ b/codex-rs/analytics/src/client.rs@@ -505,6 +505,12 @@ impl AnalyticsEventsClient {))));}+ pub fn track_thread_hint_status(&self, event: crate::thread_hint::ThreadHintStatusEvent) {+ self.record_fact(AnalyticsFact::Custom(+ CustomAnalyticsFact::ThreadHintStatus(Box::new(event)),+ ));+ }+pub fn track_image_preparation(&self, fact: ImagePreparationFact) {self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::ImagePreparation(Box::new(fact)),diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rsindex 0342ea79c9..a984f2791f 100644--- a/codex-rs/analytics/src/events.rs+++ b/codex-rs/analytics/src/events.rs@@ -76,6 +76,7 @@ pub(crate) enum TrackEventRequest {HookRun(CodexHookRunEventRequest),Compaction(Box<CodexCompactionEventRequest>),Goal(Box<CodexGoalEventRequest>),+ ThreadHintStatus(Box<crate::thread_hint::ThreadHintStatusEventRequest>),TurnEvent(Box<CodexTurnEventRequest>),TurnSteer(CodexTurnSteerEventRequest),ArtifactOperation(CodexArtifactOperationEventRequest),diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rsindex 4fb8bbca3d..393429aa54 100644--- a/codex-rs/analytics/src/facts.rs+++ b/codex-rs/analytics/src/facts.rs@@ -566,6 +566,7 @@ pub(crate) enum CustomAnalyticsFact {SubAgentThreadStarted(SubAgentThreadStartedInput),Compaction(Box<CodexCompactionEvent>),Goal(Box<CodexGoalEvent>),+ ThreadHintStatus(Box<crate::thread_hint::ThreadHintStatusEvent>),GuardianReview(Box<GuardianReviewEventParams>),GuardianV2(Box<GuardianV2Event>),TurnResolvedConfig(Box<TurnResolvedConfigFact>),diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rsindex 7d2f86b979..1897ec0756 100644--- a/codex-rs/analytics/src/lib.rs+++ b/codex-rs/analytics/src/lib.rs@@ -6,6 +6,7 @@ mod events;mod facts;mod guardian_v2;mod reducer;+mod thread_hint;use std::time::SystemTime;use std::time::UNIX_EPOCH;@@ -74,6 +75,8 @@ pub use facts::TurnTokenUsageFact;pub use facts::build_track_events_context;pub use guardian_v2::GuardianV2Event;pub use guardian_v2::GuardianV2EventKind;+pub use thread_hint::ThreadHintStatus;+pub use thread_hint::ThreadHintStatusEvent;#[cfg(test)]mod analytics_client_tests;diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rsindex be5b8f24cb..0378f2b19c 100644--- a/codex-rs/analytics/src/reducer.rs+++ b/codex-rs/analytics/src/reducer.rs@@ -621,6 +621,34 @@ impl AnalyticsReducer {CustomAnalyticsFact::Goal(input) => {self.ingest_goal(*input, out);}+ CustomAnalyticsFact::ThreadHintStatus(input) => {+ if let Some((connection, thread, metadata)) =+ self.thread_context_or_warn(AnalyticsDropSite {+ event_name: "codex_thread_hint_status",+ thread_id: &input.thread_id,+ turn_id: None,+ review_id: None,+ item_id: None,+ })+ {+ out.push(TrackEventRequest::ThreadHintStatus(Box::new(+ crate::thread_hint::ThreadHintStatusEventRequest {+ event_type: "codex_thread_hint_status",+ event_params: crate::thread_hint::ThreadHintStatusEventParams {+ thread_id: input.thread_id,+ session_id: metadata.session_id.clone(),+ app_server_client: thread.app_server_client(connection),+ runtime: connection.runtime.clone(),+ thread_source: metadata.thread_source.clone(),+ subagent_source: metadata.subagent_source.clone(),+ parent_thread_id: metadata.parent_thread_id.clone(),+ status: input.status,+ occurred_at_ms: input.occurred_at_ms,+ },+ },+ )));+ }+ }CustomAnalyticsFact::GuardianV2(input) => {let event_type = match &input.kind {GuardianV2EventKind::Classification { .. } => {diff --git a/codex-rs/analytics/src/thread_hint.rs b/codex-rs/analytics/src/thread_hint.rsnew file mode 100644index 0000000000..c863558b7c--- /dev/null+++ b/codex-rs/analytics/src/thread_hint.rs@@ -0,0 +1,38 @@+//! Per-attempt thread hint status analytics without hint contents.++use crate::events::CodexAppServerClientMetadata;+use crate::events::CodexRuntimeMetadata;+use codex_protocol::protocol::ThreadSource;+use serde::Serialize;++#[derive(Clone, Copy, Debug, Serialize)]+#[serde(rename_all = "snake_case")]+pub enum ThreadHintStatus {+ Succeeded,+ Failed,+}++pub struct ThreadHintStatusEvent {+ pub thread_id: String,+ pub status: ThreadHintStatus,+ pub occurred_at_ms: u64,+}++#[derive(Serialize)]+pub(crate) struct ThreadHintStatusEventRequest {+ pub(crate) event_type: &'static str,+ pub(crate) event_params: ThreadHintStatusEventParams,+}++#[derive(Serialize)]+pub(crate) struct ThreadHintStatusEventParams {+ pub(crate) thread_id: String,+ pub(crate) session_id: String,+ pub(crate) app_server_client: CodexAppServerClientMetadata,+ pub(crate) runtime: CodexRuntimeMetadata,+ pub(crate) thread_source: Option<ThreadSource>,+ pub(crate) subagent_source: Option<String>,+ pub(crate) parent_thread_id: Option<String>,+ pub(crate) status: ThreadHintStatus,+ pub(crate) occurred_at_ms: u64,+}diff --git a/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs b/codex-rs/app-server/tests/suite/v2/history_notes_extension.rsindex d49f66dd6f..8d851ddeef 100644--- a/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs+++ b/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs@@ -1,14 +1,11 @@use std::time::Duration;use anyhow::Result;-use app_test_support::ChatGptAuthFixture;use app_test_support::MockResponsesConfig;use app_test_support::TestAppServer;-use app_test_support::write_chatgpt_auth;use codex_app_server_protocol::ThreadStartParams;use codex_app_server_protocol::TurnStartParams;use codex_app_server_protocol::UserInput;-use codex_config::types::AuthCredentialsStoreMode;use core_test_support::load_default_config_for_test;use core_test_support::responses;use pretty_assertions::assert_eq;@@ -116,6 +113,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(let codex_home = TempDir::new()?;MockResponsesConfig::new(&server.uri())+ .with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri())).with_model_provider("openai-custom").with_provider_name("OpenAI").with_provider_base_url(&format!("{}/backend-api/codex", server.uri()))@@ -125,11 +123,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(server.uri(),)).write(codex_home.path())?;- write_chatgpt_auth(- codex_home.path(),- ChatGptAuthFixture::new("access-chatgpt"),- AuthCredentialsStoreMode::File,- )?;+ mount_analytics_capture(&server, codex_home.path()).await?;let mut app_server = TestAppServer::builder().with_codex_home(codex_home.path())@@ -143,7 +137,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(timeout(Duration::from_secs(10),app_server.start_turn_and_wait_for_completion(TurnStartParams {- thread_id: thread.id,+ thread_id: thread.id.clone(),input: vec![UserInput::Text {text: "inspect history and notes".to_string(),text_elements: Vec::new(),@@ -218,6 +212,22 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(&& item["name"] == "thread_hint"}));+ if use_history_notes_extension {+ let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| {+ event["event_type"] == "codex_thread_hint_status"+ && event["event_params"]["thread_id"] == thread.id+ })+ .await?;+ assert_eq!(+ event["event_params"]["status"],+ if hint_status == 200 {+ "succeeded"+ } else {+ "failed"+ },+ );+ }+Ok(())}diff --git a/codex-rs/ext/history-notes/Cargo.toml b/codex-rs/ext/history-notes/Cargo.tomlindex d043ee06e1..82a1922d15 100644--- a/codex-rs/ext/history-notes/Cargo.toml+++ b/codex-rs/ext/history-notes/Cargo.toml@@ -13,6 +13,7 @@ doctest = falseworkspace = true[dependencies]+codex-analytics = { workspace = true }codex-api = { workspace = true }codex-client = { workspace = true }codex-core = { workspace = true }diff --git a/codex-rs/ext/history-notes/src/extension.rs b/codex-rs/ext/history-notes/src/extension.rsindex 4d3e4e24f2..dfbff12cac 100644--- a/codex-rs/ext/history-notes/src/extension.rs+++ b/codex-rs/ext/history-notes/src/extension.rs@@ -1,5 +1,8 @@use std::sync::Arc;+use codex_analytics::AnalyticsEventsClient;+use codex_analytics::ThreadHintStatus;+use codex_analytics::ThreadHintStatusEvent;use codex_core::config::Config;use codex_extension_api::ConfigContributor;use codex_extension_api::ContentItemKind;@@ -104,6 +107,15 @@ impl ContextContributor for HistoryNotesExtension {let Some(identity) = thread_store.get::<HistoryNotesAgentIdentity>() else {return Vec::new();};+ let track_status = |status| {+ if let Some(analytics) = session_store.get::<AnalyticsEventsClient>() {+ analytics.track_thread_hint_status(ThreadHintStatusEvent {+ thread_id: thread_store.level_id().to_string(),+ status,+ occurred_at_ms: codex_analytics::now_unix_millis(),+ });+ }+ };let Ok(result) = config.backend.call(@@ -115,12 +127,19 @@ impl ContextContributor for HistoryNotesExtension {).awaitelse {+ track_status(ThreadHintStatus::Failed);return Vec::new();};let Some(text) = result.get("text").and_then(serde_json::Value::as_str) else {+ track_status(ThreadHintStatus::Failed);return Vec::new();};- if text.is_empty() || text.len() > MAX_THREAD_HINT_BYTES {+ if text.len() > MAX_THREAD_HINT_BYTES {+ track_status(ThreadHintStatus::Failed);+ return Vec::new();+ }+ track_status(ThreadHintStatus::Succeeded);+ if text.is_empty() {return Vec::new();}vec![PromptFragment::new(