Lightweight coding agent that runs in your terminal
Rust
Trace turn phases and correlate accepted input with turns (#49262)
jifa7660cd15490
diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rsindex 5b335c2c05..e56106dfb6 100644--- a/codex-rs/core/src/session/turn.rs+++ b/codex-rs/core/src/session/turn.rs@@ -1479,6 +1479,12 @@ async fn run_auto_compact() -> CodexResult<()> {let turn_context = &step_context.turn;let _profile_guard = turn_context.turn_timing_state.begin_compaction();+ let _compaction_span = trace_span!(+ "codex.compaction",+ codex.turn.phase = "compaction",+ conversation.id = %sess.thread_id,+ turn.id = %turn_context.sub_id,+ );if turn_context.config.features.enabled(Feature::TokenBudget) {// Compaction is the reset request, so force a new context window// instead of consuming a pending `new_context` tool request.@@ -2547,6 +2553,13 @@ async fn try_run_sampling_request(turn_context.provider.info().name.as_str(),);let sampling_timing_guard = turn_context.turn_timing_state.begin_sampling();+ // Do not enter this span: overlapping tools must not retain it past sampling.+ let sampling_span = trace_span!(+ "codex.sampling",+ codex.turn.phase = "sampling",+ conversation.id = %sess.thread_id,+ turn.id = %turn_context.sub_id,+ );let uses_sequential_cutoff_reasoning_summaries = turn_context.config.features@@ -2787,6 +2800,14 @@ async fn try_run_sampling_request(.enabled(Feature::DeferMailboxPreemption)&& sess.input_queue.has_pending_mailbox_items().await{+ tracing::event!(+ name: "codex.mailbox_preemption",+ target: "codex_otel.trace_safe",+ tracing::Level::INFO,+ event.name = "codex.mailbox_preemption",+ conversation.id = %sess.thread_id,+ turn.id = %turn_context.sub_id,+ );break Ok(SamplingRequestResult {needs_follow_up: true,last_agent_message,@@ -3127,6 +3148,7 @@ async fn try_run_sampling_request(}}};+ drop(sampling_span);drop(sampling_timing_guard);flush_assistant_text_segments_all(@@ -3137,13 +3159,16 @@ async fn try_run_sampling_request().await;- let tool_blocking_timing_guard = if in_flight.is_empty() {- None- } else {- Some(turn_context.turn_timing_state.begin_tool_blocking())- };- drain_in_flight(&mut in_flight, sess.clone(), &step_context).await?;- drop(tool_blocking_timing_guard);+ if !in_flight.is_empty() {+ let _tool_blocking_timing_guard = turn_context.turn_timing_state.begin_tool_blocking();+ let _tool_blocking_span = trace_span!(+ "codex.tool_blocking",+ codex.turn.phase = "tool_blocking",+ conversation.id = %sess.thread_id,+ turn.id = %turn_context.sub_id,+ );+ drain_in_flight(&mut in_flight, sess.clone(), &step_context).await?;+ }if should_emit_token_count {// A tool call such as request_user_input can intentionally pause the turn. Emit tokendiff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rsindex d9a5c6a30a..4fc90cac0a 100644--- a/codex-rs/core/src/session/turn_input.rs+++ b/codex-rs/core/src/session/turn_input.rs@@ -202,13 +202,19 @@ impl PreparedTurnInputSettings {}}+#[tracing::instrument(+ name = "codex.turn_input",+ level = "trace",+ skip_all,+ fields(conversation.id = %session.thread_id, turn.id)+)]pub(super) async fn handle(session: &Arc<Session>,request: TurnInputRequest,mode: TurnInputMode,submission_id: String,) -> CodexResult<TurnInputSubmission> {- match mode {+ let result = match mode {TurnInputMode::StartOrSteer => start_or_steer(session, request, submission_id).await,TurnInputMode::StartIfIdle => {let kind = match &request.input {@@ -248,9 +254,22 @@ pub(super) async fn handle(TurnInputMode::Steer { expected_turn_id } => {steer(session, request, expected_turn_id, submission_id).await}+ };+ // Link this request's trace to the accepted turn, which may have an older trace.+ if let Ok(TurnInputSubmission::Started { turn_id } | TurnInputSubmission::Steered { turn_id }) =+ &result+ {+ tracing::Span::current().record("turn.id", turn_id);}+ result}+#[tracing::instrument(+ name = "codex.turn_input",+ level = "trace",+ skip_all,+ fields(conversation.id = %session.thread_id, turn.id)+)]pub(super) async fn handle_recovery(session: &Arc<Session>,thread_settings: ThreadSettingsOverrides,@@ -263,14 +282,18 @@ pub(super) async fn handle_recovery(turn_trigger: Some("retry".to_string()),..start_options});- start_if_idle(+ let result = start_if_idle(session,request,submission_id,TurnStartKind::Recovery,/*expected_previous_turn_id*/ None,)- .await+ .await;+ if let Ok(TurnInputSubmission::Started { turn_id }) = &result {+ tracing::Span::current().record("turn.id", turn_id);+ }+ result}async fn start_or_steer(diff --git a/codex-rs/core/src/tasks/compact.rs b/codex-rs/core/src/tasks/compact.rsindex 4235fb6de1..baa8a639e1 100644--- a/codex-rs/core/src/tasks/compact.rs+++ b/codex-rs/core/src/tasks/compact.rs@@ -34,6 +34,12 @@ impl SessionTask for CompactTask {_cancellation_token: CancellationToken,) -> SessionTaskResult {let _profile_guard = ctx.turn_timing_state.begin_compaction();+ let _compaction_span = tracing::trace_span!(+ "codex.compaction",+ codex.turn.phase = "compaction",+ conversation.id = %session.thread_id,+ turn.id = %ctx.sub_id,+ );if ctx.config.features.enabled(Feature::TokenBudget) {crate::compact_token_budget::run_manual_compact_task(session, ctx).await?;return Ok(None);diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rsindex 50f965cd5f..14805e25b5 100644--- a/codex-rs/core/tests/suite/mod.rs+++ b/codex-rs/core/tests/suite/mod.rs@@ -219,6 +219,7 @@ mod truncation;#[path = "turn_error_details_tests.rs"]mod turn_error_details;mod turn_input_submission;+mod turn_phase_trace;mod turn_state;mod unified_exec;#[path = "unified_exec_launch_failure_tests.rs"]diff --git a/codex-rs/core/tests/suite/scenarios_mailbox_preemption_tests.rs b/codex-rs/core/tests/suite/scenarios_mailbox_preemption_tests.rsindex 3fa281bff9..3d6d79ff1c 100644--- a/codex-rs/core/tests/suite/scenarios_mailbox_preemption_tests.rs+++ b/codex-rs/core/tests/suite/scenarios_mailbox_preemption_tests.rs@@ -27,6 +27,7 @@ enum Boundary {Commentary,}+#[tracing_test::traced_test]#[test_case(Boundary::Reasoning, false; "reasoning_default")]#[test_case(Boundary::Commentary, false; "commentary_default")]#[test_case(Boundary::Reasoning, true; "reasoning_deferred")]@@ -139,6 +140,19 @@ async fn mailbox_preemption_preserves_response_when_deferred(}).await;+ let logs = String::from_utf8(+ tracing_test::internal::global_buf()+ .lock()+ .expect("test log buffer is not poisoned")+ .clone(),+ )?;+ let thread_id = test.session_configured.thread_id.to_string();+ assert_eq!(+ logs.lines()+ .filter(|line| line.contains("codex.mailbox_preemption") && line.contains(&thread_id))+ .count(),+ usize::from(!defer_mailbox_preemption),+ );let requests = streaming.requests().awaitdiff --git a/codex-rs/core/tests/suite/turn_phase_trace.rs b/codex-rs/core/tests/suite/turn_phase_trace.rsnew file mode 100644index 0000000000..3614c9f34c--- /dev/null+++ b/codex-rs/core/tests/suite/turn_phase_trace.rs@@ -0,0 +1,319 @@+//! Cross-trace admission links and phase boundaries through a real blocked tool.+//! Each case uses an isolated process so SQLite workers share its tracing collector.++use codex_core::RecoverTurnRequest;+use codex_core::StartIfIdleSubmission;+use codex_core::StartThreadOptions;+use codex_core::TurnInputRequest;+use codex_core::TurnInputSubmission;+use codex_protocol::dynamic_tools::DynamicToolFunctionSpec;+use codex_protocol::dynamic_tools::DynamicToolResponse;+use codex_protocol::dynamic_tools::DynamicToolSpec;+use codex_protocol::protocol::EventMsg;+use codex_protocol::protocol::Op;+use codex_protocol::user_input::UserInput;+use core_test_support::responses::ev_assistant_message;+use core_test_support::responses::ev_completed;+use core_test_support::responses::ev_function_call;+use core_test_support::responses::ev_response_created;+use core_test_support::responses::mount_sse_once;+use core_test_support::responses::sse;+use core_test_support::responses::start_mock_server;+use core_test_support::test_codex::test_codex;+use core_test_support::wait_for_event;+use opentelemetry::KeyValue;+use opentelemetry::trace::TracerProvider as _;+use opentelemetry_sdk::propagation::TraceContextPropagator;+use opentelemetry_sdk::trace::InMemorySpanExporter;+use opentelemetry_sdk::trace::SdkTracerProvider;+use opentelemetry_sdk::trace::SpanData;+use pretty_assertions::assert_eq;+use std::time::Duration;+use test_case::test_case;+use tokio::process::Command;+use tracing_subscriber::prelude::*;++const SUBPROCESS_ENV_VAR: &str = "CODEX_TURN_PHASE_TRACE_TEST_SUBPROCESS";++fn attribute(attributes: &[KeyValue], key: &str) -> Option<String> {+ attributes+ .iter()+ .find(|attribute| attribute.key.as_str() == key)+ .map(|attribute| attribute.value.to_string())+}++async fn wait_for_span(+ exported: &InMemorySpanExporter,+ predicate: impl Fn(&SpanData) -> bool,+) -> anyhow::Result<SpanData> {+ tokio::time::timeout(Duration::from_secs(10), async {+ loop {+ if let Some(span) = exported.get_finished_spans()?.into_iter().find(&predicate) {+ break Ok(span);+ }+ tokio::task::yield_now().await;+ }+ })+ .await?+}++#[derive(Clone, Copy)]+enum Finish {+ Reply,+ Interrupt,+}++#[test_case(Finish::Reply; "reply")]+#[test_case(Finish::Interrupt; "interrupt")]+#[tokio::test(flavor = "current_thread")]+async fn steer_joins_existing_turn_without_extending_sampling(+ finish: Finish,+) -> anyhow::Result<()> {+ if std::env::var_os(SUBPROCESS_ENV_VAR).is_none() {+ let case = match finish {+ Finish::Reply => "reply",+ Finish::Interrupt => "interrupt",+ };+ let test_name = format!(+ "suite::turn_phase_trace::steer_joins_existing_turn_without_extending_sampling::{case}"+ );+ let output = Command::new(std::env::current_exe()?)+ .arg("--exact")+ .arg(&test_name)+ .env(SUBPROCESS_ENV_VAR, "1")+ .output()+ .await?;+ assert!(+ output.status.success(),+ "subprocess test `{test_name}` failed\nstdout:\n{}\nstderr:\n{}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr),+ );+ return Ok(());+ }++ opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());+ let exported = InMemorySpanExporter::default();+ let telemetry = SdkTracerProvider::builder()+ .with_simple_exporter(exported.clone())+ .build();+ tracing_subscriber::registry()+ .with(+ tracing_opentelemetry::layer()+ .with_tracer(telemetry.tracer("turn-phase-test"))+ .with_filter(tracing_subscriber::filter::filter_fn(+ codex_otel::OtelProvider::trace_export_filter,+ )),+ )+ .try_init()?;+ let server = start_mock_server().await;+ let first = mount_sse_once(+ &server,+ sse(vec![+ ev_response_created("first"),+ ev_assistant_message("preamble", "I will check."),+ ev_function_call("gate", "gate", "{}"),+ ev_completed("first"),+ ]),+ )+ .await;+ let mut test = test_codex().build_with_auto_env(&server).await?;+ let thread = test+ .thread_manager+ .start_thread(StartThreadOptions {+ dynamic_tools: vec![DynamicToolSpec::Function(DynamicToolFunctionSpec {+ name: "gate".to_string(),+ description: "Wait for the host.".to_string(),+ input_schema: serde_json::json!({"type": "object", "properties": {}}),+ defer_loading: false,+ })],+ ..StartThreadOptions::new(test.config.clone())+ })+ .await?;+ let thread_id = thread.thread_id.to_string();+ test.codex = thread.thread;+ let input = |text: &str| {+ TurnInputRequest::user_input(vec![UserInput::Text {+ text: text.to_string(),+ text_elements: Vec::new(),+ }])+ };+ let initial = tracing::info_span!(parent: None, "initial_request");+ test.codex+ .start_or_steer_turn(+ input("Start checking").with_trace(codex_otel::span_w3c_trace_context(&initial)),+ )+ .await?;+ let EventMsg::DynamicToolCallRequest(call) = wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::DynamicToolCallRequest(_))+ })+ .await+ else {+ unreachable!();+ };+ first.single_request();+ // The tool is still blocked. Sampling must already be exportable, even though+ // the dispatch future and its trace remain alive until the host responds.+ let sampling = wait_for_span(&exported, |span| span.name == "codex.sampling").await?;+ let steer = tracing::info_span!(parent: None, "steer_request");+ assert_eq!(+ test.codex+ .start_or_steer_turn(+ input("New question").with_trace(codex_otel::span_w3c_trace_context(&steer)),+ )+ .await?,+ TurnInputSubmission::Steered {+ turn_id: call.turn_id.clone()+ }+ );+ match finish {+ Finish::Reply => {+ let reply = mount_sse_once(+ &server,+ sse(vec![+ ev_response_created("second"),+ ev_assistant_message("answer", "Here is the answer."),+ ev_completed("second"),+ ]),+ )+ .await;+ test.codex+ .submit(Op::DynamicToolResponse {+ id: call.call_id.clone(),+ response: DynamicToolResponse {+ content_items: Vec::new(),+ success: true,+ },+ })+ .await?;+ wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::TurnComplete(_))+ })+ .await;+ assert!(+ reply+ .single_request()+ .message_input_texts("user")+ .contains(&"New question".to_string())+ );+ }+ Finish::Interrupt => {+ test.codex.submit(Op::Interrupt).await?;+ wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::TurnAborted(_))+ })+ .await;+ let reply = mount_sse_once(+ &server,+ sse(vec![+ ev_response_created("recovered"),+ ev_assistant_message("answer", "Recovered answer."),+ ev_completed("recovered"),+ ]),+ )+ .await;+ let recovery = tracing::info_span!(parent: None, "recovery_request");+ assert_eq!(+ test.codex+ .recover_turn_if_idle(RecoverTurnRequest {+ turn_id: call.turn_id.clone(),+ thread_settings: Default::default(),+ trace: codex_otel::span_w3c_trace_context(&recovery),+ cyber_access_program: None,+ })+ .await?,+ StartIfIdleSubmission::Started {+ turn_id: call.turn_id.clone()+ }+ );+ wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::TurnComplete(_))+ })+ .await;+ reply.single_request();+ drop(recovery);+ let recovery = wait_for_span(&exported, |span| span.name == "recovery_request").await?;+ let admitted = wait_for_span(&exported, |span| {+ span.name == "codex.turn_input"+ && span.span_context.trace_id() == recovery.span_context.trace_id()+ })+ .await?;+ assert_ne!(+ admitted.span_context.trace_id(),+ sampling.span_context.trace_id()+ );+ assert_eq!(+ (+ attribute(&admitted.attributes, "conversation.id"),+ attribute(&admitted.attributes, "turn.id"),+ ),+ (Some(thread_id.clone()), Some(call.turn_id.clone()))+ );+ }+ }+ test.codex.submit(Op::Shutdown).await?;+ wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::ShutdownComplete)+ })+ .await;+ drop(initial);+ drop(steer);+ // ShutdownComplete can precede the final dispatch span's drop. Wait for+ // its exported ready event rather than racing the last task cleanup.+ let tool_span = wait_for_span(&exported, |span| {+ span.events.events.iter().any(|event| {+ event.name == "codex.tool_result_ready"+ && attribute(&event.attributes, "call_id").as_deref() == Some(call.call_id.as_str())+ })+ })+ .await?;+ let spans = exported.get_finished_spans()?;+ let steer = spans+ .iter()+ .find(|span| span.name == "steer_request")+ .expect("steer request span exported");+ let accepted = spans+ .iter()+ .find(|span| {+ span.name == "codex.turn_input"+ && span.span_context.trace_id() == steer.span_context.trace_id()+ })+ .expect("accepted input span exported in the steer trace");+ let blocking = spans+ .iter()+ .find(|span| {+ attribute(&span.attributes, "codex.turn.phase").as_deref() == Some("tool_blocking")+ })+ .expect("tool-blocking phase span exported");+ assert_ne!(+ accepted.span_context.trace_id(),+ sampling.span_context.trace_id()+ );+ assert_eq!(+ [+ &accepted.attributes,+ &sampling.attributes,+ &blocking.attributes+ ]+ .map(|attributes| (+ attribute(attributes, "conversation.id").expect("span has a conversation ID"),+ attribute(attributes, "turn.id").expect("span has a turn ID"),+ )),+ std::array::from_fn::<_, 3, _>(|_| (thread_id.clone(), call.turn_id.clone()))+ );+ assert!(sampling.end_time <= accepted.start_time);+ assert!(sampling.end_time <= blocking.start_time);+ assert!(accepted.end_time <= blocking.end_time);+ let ready = tool_span+ .events+ .events+ .iter()+ .find(|event| {+ event.name == "codex.tool_result_ready"+ && attribute(&event.attributes, "call_id").as_deref() == Some(call.call_id.as_str())+ })+ .expect("gated tool result-ready event exported");+ assert!(accepted.end_time <= ready.timestamp);+ assert!(ready.timestamp <= blocking.end_time);+ Ok(())+}