Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Track history notes thread hint outcomes (#42247)

pmccrary-oaibdfd76964092
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index f9c9089a83..5c3da721a4 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -3425,6 +3425,7 @@ dependencies = [
name = "codex-history-notes-extension"
version = "0.0.0"
dependencies = [
+ "codex-analytics",
"codex-api",
"codex-client",
"codex-config",
diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs
index 96dce42750..6895825c87 100644
--- a/codex-rs/analytics/src/client.rs
+++ b/codex-rs/analytics/src/client.rs
@@ -505,6 +505,12 @@ impl AnalyticsEventsClient {
))));
}
+ pub fn track_thread_hint_status(&self, event: crate::thread_hint::ThreadHintStatusEvent) {
+ self.record_fact(AnalyticsFact::Custom(
+ CustomAnalyticsFact::ThreadHintStatus(Box::new(event)),
+ ));
+ }
+
pub fn track_image_preparation(&self, fact: ImagePreparationFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::ImagePreparation(Box::new(fact)),
diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs
index 0342ea79c9..a984f2791f 100644
--- a/codex-rs/analytics/src/events.rs
+++ b/codex-rs/analytics/src/events.rs
@@ -76,6 +76,7 @@ pub(crate) enum TrackEventRequest {
HookRun(CodexHookRunEventRequest),
Compaction(Box<CodexCompactionEventRequest>),
Goal(Box<CodexGoalEventRequest>),
+ ThreadHintStatus(Box<crate::thread_hint::ThreadHintStatusEventRequest>),
TurnEvent(Box<CodexTurnEventRequest>),
TurnSteer(CodexTurnSteerEventRequest),
ArtifactOperation(CodexArtifactOperationEventRequest),
diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs
index 4fb8bbca3d..393429aa54 100644
--- a/codex-rs/analytics/src/facts.rs
+++ b/codex-rs/analytics/src/facts.rs
@@ -566,6 +566,7 @@ pub(crate) enum CustomAnalyticsFact {
SubAgentThreadStarted(SubAgentThreadStartedInput),
Compaction(Box<CodexCompactionEvent>),
Goal(Box<CodexGoalEvent>),
+ ThreadHintStatus(Box<crate::thread_hint::ThreadHintStatusEvent>),
GuardianReview(Box<GuardianReviewEventParams>),
GuardianV2(Box<GuardianV2Event>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs
index 7d2f86b979..1897ec0756 100644
--- a/codex-rs/analytics/src/lib.rs
+++ b/codex-rs/analytics/src/lib.rs
@@ -6,6 +6,7 @@ mod events;
mod facts;
mod guardian_v2;
mod reducer;
+mod thread_hint;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
@@ -74,6 +75,8 @@ pub use facts::TurnTokenUsageFact;
pub use facts::build_track_events_context;
pub use guardian_v2::GuardianV2Event;
pub use guardian_v2::GuardianV2EventKind;
+pub use thread_hint::ThreadHintStatus;
+pub use thread_hint::ThreadHintStatusEvent;
#[cfg(test)]
mod analytics_client_tests;
diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs
index be5b8f24cb..0378f2b19c 100644
--- a/codex-rs/analytics/src/reducer.rs
+++ b/codex-rs/analytics/src/reducer.rs
@@ -621,6 +621,34 @@ impl AnalyticsReducer {
CustomAnalyticsFact::Goal(input) => {
self.ingest_goal(*input, out);
}
+ CustomAnalyticsFact::ThreadHintStatus(input) => {
+ if let Some((connection, thread, metadata)) =
+ self.thread_context_or_warn(AnalyticsDropSite {
+ event_name: "codex_thread_hint_status",
+ thread_id: &input.thread_id,
+ turn_id: None,
+ review_id: None,
+ item_id: None,
+ })
+ {
+ out.push(TrackEventRequest::ThreadHintStatus(Box::new(
+ crate::thread_hint::ThreadHintStatusEventRequest {
+ event_type: "codex_thread_hint_status",
+ event_params: crate::thread_hint::ThreadHintStatusEventParams {
+ thread_id: input.thread_id,
+ session_id: metadata.session_id.clone(),
+ app_server_client: thread.app_server_client(connection),
+ runtime: connection.runtime.clone(),
+ thread_source: metadata.thread_source.clone(),
+ subagent_source: metadata.subagent_source.clone(),
+ parent_thread_id: metadata.parent_thread_id.clone(),
+ status: input.status,
+ occurred_at_ms: input.occurred_at_ms,
+ },
+ },
+ )));
+ }
+ }
CustomAnalyticsFact::GuardianV2(input) => {
let event_type = match &input.kind {
GuardianV2EventKind::Classification { .. } => {
diff --git a/codex-rs/analytics/src/thread_hint.rs b/codex-rs/analytics/src/thread_hint.rs
new file mode 100644
index 0000000000..c863558b7c
--- /dev/null
+++ b/codex-rs/analytics/src/thread_hint.rs
@@ -0,0 +1,38 @@
+//! Per-attempt thread hint status analytics without hint contents.
+
+use crate::events::CodexAppServerClientMetadata;
+use crate::events::CodexRuntimeMetadata;
+use codex_protocol::protocol::ThreadSource;
+use serde::Serialize;
+
+#[derive(Clone, Copy, Debug, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ThreadHintStatus {
+ Succeeded,
+ Failed,
+}
+
+pub struct ThreadHintStatusEvent {
+ pub thread_id: String,
+ pub status: ThreadHintStatus,
+ pub occurred_at_ms: u64,
+}
+
+#[derive(Serialize)]
+pub(crate) struct ThreadHintStatusEventRequest {
+ pub(crate) event_type: &'static str,
+ pub(crate) event_params: ThreadHintStatusEventParams,
+}
+
+#[derive(Serialize)]
+pub(crate) struct ThreadHintStatusEventParams {
+ pub(crate) thread_id: String,
+ pub(crate) session_id: String,
+ pub(crate) app_server_client: CodexAppServerClientMetadata,
+ pub(crate) runtime: CodexRuntimeMetadata,
+ pub(crate) thread_source: Option<ThreadSource>,
+ pub(crate) subagent_source: Option<String>,
+ pub(crate) parent_thread_id: Option<String>,
+ pub(crate) status: ThreadHintStatus,
+ pub(crate) occurred_at_ms: u64,
+}
diff --git a/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs b/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs
index d49f66dd6f..8d851ddeef 100644
--- a/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs
+++ b/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs
@@ -1,14 +1,11 @@
use std::time::Duration;
use anyhow::Result;
-use app_test_support::ChatGptAuthFixture;
use app_test_support::MockResponsesConfig;
use app_test_support::TestAppServer;
-use app_test_support::write_chatgpt_auth;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::UserInput;
-use codex_config::types::AuthCredentialsStoreMode;
use core_test_support::load_default_config_for_test;
use core_test_support::responses;
use pretty_assertions::assert_eq;
@@ -116,6 +113,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
let codex_home = TempDir::new()?;
MockResponsesConfig::new(&server.uri())
+ .with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri()))
.with_model_provider("openai-custom")
.with_provider_name("OpenAI")
.with_provider_base_url(&format!("{}/backend-api/codex", server.uri()))
@@ -125,11 +123,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
server.uri(),
))
.write(codex_home.path())?;
- write_chatgpt_auth(
- codex_home.path(),
- ChatGptAuthFixture::new("access-chatgpt"),
- AuthCredentialsStoreMode::File,
- )?;
+ mount_analytics_capture(&server, codex_home.path()).await?;
let mut app_server = TestAppServer::builder()
.with_codex_home(codex_home.path())
@@ -143,7 +137,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
timeout(
Duration::from_secs(10),
app_server.start_turn_and_wait_for_completion(TurnStartParams {
- thread_id: thread.id,
+ thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "inspect history and notes".to_string(),
text_elements: Vec::new(),
@@ -218,6 +212,22 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
&& item["name"] == "thread_hint"
}));
+ if use_history_notes_extension {
+ let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| {
+ event["event_type"] == "codex_thread_hint_status"
+ && event["event_params"]["thread_id"] == thread.id
+ })
+ .await?;
+ assert_eq!(
+ event["event_params"]["status"],
+ if hint_status == 200 {
+ "succeeded"
+ } else {
+ "failed"
+ },
+ );
+ }
+
Ok(())
}
diff --git a/codex-rs/ext/history-notes/Cargo.toml b/codex-rs/ext/history-notes/Cargo.toml
index d043ee06e1..82a1922d15 100644
--- a/codex-rs/ext/history-notes/Cargo.toml
+++ b/codex-rs/ext/history-notes/Cargo.toml
@@ -13,6 +13,7 @@ doctest = false
workspace = true
[dependencies]
+codex-analytics = { workspace = true }
codex-api = { workspace = true }
codex-client = { workspace = true }
codex-core = { workspace = true }
diff --git a/codex-rs/ext/history-notes/src/extension.rs b/codex-rs/ext/history-notes/src/extension.rs
index 4d3e4e24f2..dfbff12cac 100644
--- a/codex-rs/ext/history-notes/src/extension.rs
+++ b/codex-rs/ext/history-notes/src/extension.rs
@@ -1,5 +1,8 @@
use std::sync::Arc;
+use codex_analytics::AnalyticsEventsClient;
+use codex_analytics::ThreadHintStatus;
+use codex_analytics::ThreadHintStatusEvent;
use codex_core::config::Config;
use codex_extension_api::ConfigContributor;
use codex_extension_api::ContentItemKind;
@@ -104,6 +107,15 @@ impl ContextContributor for HistoryNotesExtension {
let Some(identity) = thread_store.get::<HistoryNotesAgentIdentity>() else {
return Vec::new();
};
+ let track_status = |status| {
+ if let Some(analytics) = session_store.get::<AnalyticsEventsClient>() {
+ analytics.track_thread_hint_status(ThreadHintStatusEvent {
+ thread_id: thread_store.level_id().to_string(),
+ status,
+ occurred_at_ms: codex_analytics::now_unix_millis(),
+ });
+ }
+ };
let Ok(result) = config
.backend
.call(
@@ -115,12 +127,19 @@ impl ContextContributor for HistoryNotesExtension {
)
.await
else {
+ track_status(ThreadHintStatus::Failed);
return Vec::new();
};
let Some(text) = result.get("text").and_then(serde_json::Value::as_str) else {
+ track_status(ThreadHintStatus::Failed);
return Vec::new();
};
- if text.is_empty() || text.len() > MAX_THREAD_HINT_BYTES {
+ if text.len() > MAX_THREAD_HINT_BYTES {
+ track_status(ThreadHintStatus::Failed);
+ return Vec::new();
+ }
+ track_status(ThreadHintStatus::Succeeded);
+ if text.is_empty() {
return Vec::new();
}
vec![PromptFragment::new(