Lightweight coding agent that runs in your terminal
Rust
Add plugin-attributed skill telemetry (#40724)
jwang-openaic51e7b373647
diff --git a/codex-rs/core/src/skills.rs b/codex-rs/core/src/skills.rsindex 4d35dd403a..152073f39b 100644--- a/codex-rs/core/src/skills.rs+++ b/codex-rs/core/src/skills.rs@@ -13,6 +13,7 @@ use codex_protocol::protocol::SkillScope;use codex_skills::SkillMetadata;use codex_skills_extension::HostSkillsLoadInput;use codex_skills_extension::detect_implicit_skill_invocation;+use codex_skills_extension::record_plugin_turn_usage;use codex_utils_absolute_path::AbsolutePathBuf;use codex_utils_path_uri::PathUri;use codex_utils_plugins::PluginSkillRoot;@@ -44,9 +45,17 @@ pub(crate) fn emit_explicit_skill_invocations(.iter().map(|skill| &skill.path_to_skills_md).collect::<HashSet<_>>();+ let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str());+ let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing();for skill in mentioned_skills {let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str());+ let plugin_id_tag =+ sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed"));let status = if injected_skill_paths.contains(&skill.path_to_skills_md) {+ record_plugin_turn_usage(+ turn_context.extension_data.as_ref(),+ skill.plugin_id.as_deref(),+ );"ok"} else {"error"@@ -58,6 +67,9 @@ pub(crate) fn emit_explicit_skill_invocations(("status", status),("skill", skill_name_tag.as_str()),("invoke_type", "explicit"),+ ("plugin_id", plugin_id_tag.as_str()),+ ("model_slug", model_slug_tag.as_str()),+ ("reasoning_effort", reasoning_effort.as_str()),],);}@@ -123,6 +135,14 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(return;}let skill_name_tag = sanitize_metric_tag_value(skill_name.as_str());+ let plugin_id_tag =+ sanitize_metric_tag_value(invocation.plugin_id.as_deref().unwrap_or("unattributed"));+ let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str());+ let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing();+ record_plugin_turn_usage(+ turn_context.extension_data.as_ref(),+ invocation.plugin_id.as_deref(),+ );for contributor in sess.services.extensions.skill_invocation_contributors() {contributor@@ -144,6 +164,9 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(("status", "ok"),("skill", skill_name_tag.as_str()),("invoke_type", "implicit"),+ ("plugin_id", plugin_id_tag.as_str()),+ ("model_slug", model_slug_tag.as_str()),+ ("reasoning_effort", reasoning_effort.as_str()),],);sess.servicesdiff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rsindex c77e26665a..99cefa6229 100644--- a/codex-rs/ext/skills/src/extension.rs+++ b/codex-rs/ext/skills/src/extension.rs@@ -67,6 +67,7 @@ use crate::state::HostSkillsStepState;use crate::state::SkillsSessionState;use crate::state::SkillsThreadState;use crate::state::SkillsTurnState;+use crate::telemetry::SkillTelemetry;use crate::tools::SkillAnalytics;use crate::tools::SkillToolAuthority;use crate::tools::skill_tools;@@ -670,6 +671,7 @@ pub fn install_with_providers_and_metrics<C>(shadow_selection: Arc::new(ShadowSelectionExperiment::new(metrics_client)),});registry.thread_lifecycle_contributor(extension.clone());+ registry.turn_lifecycle_contributor(Arc::new(SkillTelemetry));registry.config_contributor(extension.clone());registry.prompt_contributor(extension.clone());registry.turn_input_contributor(extension.clone());diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rsindex 940aa9657c..b3fdc291b9 100644--- a/codex-rs/ext/skills/src/lib.rs+++ b/codex-rs/ext/skills/src/lib.rs@@ -20,6 +20,7 @@ mod selection;mod shadow_selection_experiment;mod sources;mod state;+mod telemetry;mod tools;mod warnings;mod world_state;@@ -43,6 +44,7 @@ pub use provider::OrchestratorSkillProvider;pub use provider::SkillProvider;pub use sources::SkillProviderSource;pub use sources::SkillProviders;+pub use telemetry::record_plugin_turn_usage;/// Recognizes persisted explicit skill prompts without exposing their fragment implementation.pub fn is_skill_prompt_fragment(text: &str) -> bool {diff --git a/codex-rs/ext/skills/src/provider/orchestrator.rs b/codex-rs/ext/skills/src/provider/orchestrator.rsindex 07643b5ca7..e984693620 100644--- a/codex-rs/ext/skills/src/provider/orchestrator.rs+++ b/codex-rs/ext/skills/src/provider/orchestrator.rs@@ -276,6 +276,10 @@ fn catalog_entry_from_resource(resource: &Resource) -> Option<SkillCatalogEntry>).with_display_path(uri).with_alias_root(format!("skill://{namespace}"));+ entry.plugin_id = meta+ .get("plugin_id")+ .and_then(serde_json::Value::as_str)+ .map(str::to_owned);entry.canonical_skill_id = meta.get("skill_id").and_then(serde_json::Value::as_str)diff --git a/codex-rs/ext/skills/src/telemetry.rs b/codex-rs/ext/skills/src/telemetry.rsnew file mode 100644index 0000000000..070cbb9762--- /dev/null+++ b/codex-rs/ext/skills/src/telemetry.rs@@ -0,0 +1,159 @@+use std::collections::HashSet;+use std::sync::Arc;+use std::sync::Mutex;+use std::sync::Weak;+use std::time::Instant;++use codex_extension_api::ExtensionData;+use codex_extension_api::ExtensionFuture;+use codex_extension_api::TurnAbortInput;+use codex_extension_api::TurnErrorInput;+use codex_extension_api::TurnLifecycleContributor;+use codex_extension_api::TurnStartInput;+use codex_extension_api::TurnStopInput;+use codex_otel::sanitize_metric_tag_value;+use codex_protocol::openai_models::ModelInfo;++use crate::state::SkillsSessionState;++/// Attributes a successful host skill invocation to its turn's plugin latency metrics.+pub fn record_plugin_turn_usage(turn_store: &ExtensionData, plugin_id: Option<&str>) {+ if let Some(turn) = turn_store.get::<SkillTurnMetrics>() {+ turn.record_plugin(plugin_id);+ }+}++pub(crate) struct SkillTurnMetrics {+ pub(crate) turn_id: String,+ model_slug: String,+ pub(crate) reasoning_effort: String,+ started_at: Instant,+ usage: Mutex<Option<TurnUsage>>,+}++#[derive(Default)]+struct TurnUsage {+ plugins: HashSet<String>,+ failed: bool,+}++impl SkillTurnMetrics {+ pub(crate) fn record_plugin(&self, plugin_id: Option<&str>) {+ if let Some(usage) = self+ .usage+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ .as_mut()+ {+ usage+ .plugins+ .insert(plugin_id.unwrap_or("unattributed").to_string());+ }+ }++ fn finish(&self, session_store: &ExtensionData, status: &str) {+ let Some(usage) = self+ .usage+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ .take()+ else {+ return;+ };+ let Some(metrics) = session_store+ .get::<SkillsSessionState>()+ .and_then(|state| state.extension_metrics.clone())+ else {+ return;+ };+ let status = if status == "completed" && usage.failed {+ "error"+ } else {+ status+ };+ // Seconds keep multi-minute turns within the standard histogram buckets.+ // Round up so integer-second bucket boundaries classify fractional durations correctly.+ let duration_seconds = self.started_at.elapsed().as_secs_f64().ceil() as i64;+ for plugin_id in usage.plugins {+ let plugin_id = sanitize_metric_tag_value(&plugin_id);+ metrics.histogram(+ "codex.skill.turn.duration_seconds",+ duration_seconds,+ &[+ ("plugin_id", plugin_id.as_str()),+ ("model_slug", self.model_slug.as_str()),+ ("reasoning_effort", self.reasoning_effort.as_str()),+ ("status", status),+ ],+ );+ }+ }+}++#[derive(Default)]+pub(crate) struct ActiveSkillTurnMetrics(pub(crate) Mutex<Weak<SkillTurnMetrics>>);++pub(crate) struct SkillTelemetry;++impl TurnLifecycleContributor for SkillTelemetry {+ fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> {+ Box::pin(async move {+ let reasoning_effort = input+ .collaboration_mode+ .reasoning_effort()+ .or_else(|| {+ input+ .thread_store+ .get::<ModelInfo>()+ .and_then(|model| model.default_reasoning_level.clone())+ })+ .map(|effort| effort.to_string())+ .unwrap_or_else(|| "default".to_string());+ input.turn_store.insert(SkillTurnMetrics {+ turn_id: input.turn_id.to_string(),+ model_slug: sanitize_metric_tag_value(input.collaboration_mode.model()),+ reasoning_effort,+ started_at: Instant::now(),+ usage: Mutex::new(Some(TurnUsage::default())),+ });+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {+ *input+ .thread_store+ .get_or_init(ActiveSkillTurnMetrics::default)+ .0+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::downgrade(&turn);+ }+ })+ }++ fn on_turn_stop<'a>(&'a self, input: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> {+ Box::pin(async move {+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {+ turn.finish(input.session_store, "completed");+ }+ })+ }++ fn on_turn_abort<'a>(&'a self, input: TurnAbortInput<'a>) -> ExtensionFuture<'a, ()> {+ Box::pin(async move {+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {+ turn.finish(input.session_store, "aborted");+ }+ })+ }++ fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> {+ Box::pin(async move {+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>()+ && let Some(usage) = turn+ .usage+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ .as_mut()+ {+ usage.failed = true;+ }+ })+ }+}diff --git a/codex-rs/ext/skills/src/tools/mod.rs b/codex-rs/ext/skills/src/tools/mod.rsindex 4869c5f497..54d663e2a5 100644--- a/codex-rs/ext/skills/src/tools/mod.rs+++ b/codex-rs/ext/skills/src/tools/mod.rs@@ -11,6 +11,7 @@ use codex_analytics::SkillInvocationLocation;use codex_analytics::build_track_events_context;use codex_exec_server::FileSystemSandboxContext;use codex_extension_api::ExtensionData;+use codex_extension_api::ExtensionMetrics;use codex_extension_api::FunctionCallError;use codex_extension_api::JsonToolOutput;use codex_extension_api::ResponsesApiTool;@@ -24,6 +25,7 @@ use codex_extension_api::ToolSpec;use codex_extension_api::parse_tool_input_schema;use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;use codex_mcp::McpResourceClient;+use codex_otel::sanitize_metric_tag_value;use codex_tools::ResponsesApiNamespace;use codex_tools::ResponsesApiNamespaceTool;use codex_tools::default_namespace_description;@@ -43,6 +45,7 @@ use crate::shadow_selection_experiment::ShadowSelectionExperiment;use crate::sources::SkillProviders;use crate::state::SkillsSessionState;use crate::state::SkillsThreadState;+use crate::telemetry::ActiveSkillTurnMetrics;mod list;mod read;@@ -96,6 +99,8 @@ pub(crate) fn skill_tools(#[derive(Clone)]pub(crate) struct SkillAnalytics {client: AnalyticsEventsClient,+ metrics: Option<Arc<dyn ExtensionMetrics>>,+ active_turn: Arc<ActiveSkillTurnMetrics>,thread_id: String,product_client_id: String,}@@ -110,6 +115,10 @@ impl SkillAnalytics {Some(Self {client: client.as_ref().clone(),+ metrics: session_store+ .get::<SkillsSessionState>()+ .and_then(|state| state.extension_metrics.clone()),+ active_turn: thread_store.get_or_init(ActiveSkillTurnMetrics::default),thread_id: thread_store.level_id().to_string(),product_client_id: originator.0.clone(),})@@ -122,6 +131,42 @@ impl SkillAnalytics {turn_id: String,invocation_type: InvocationType,) {+ let turn_metrics = self+ .active_turn+ .0+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ .upgrade()+ .filter(|turn| turn.turn_id == turn_id);+ if let Some(turn_metrics) = &turn_metrics {+ turn_metrics.record_plugin(skill.plugin_id.as_deref());+ }+ if let Some(metrics) = &self.metrics {+ let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str());+ let plugin_id_tag =+ sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed"));+ let model_slug_tag = sanitize_metric_tag_value(model.as_str());+ let reasoning_effort = turn_metrics+ .as_ref()+ .map(|turn| turn.reasoning_effort.as_str())+ .unwrap_or("unknown");+ let invoke_type = match invocation_type {+ InvocationType::Explicit => "explicit",+ InvocationType::Implicit => "implicit",+ };+ metrics.counter(+ "codex.skill.injected",+ /*inc*/ 1,+ &[+ ("status", "ok"),+ ("skill", skill_name_tag.as_str()),+ ("invoke_type", invoke_type),+ ("plugin_id", plugin_id_tag.as_str()),+ ("model_slug", model_slug_tag.as_str()),+ ("reasoning_effort", reasoning_effort),+ ],+ );+ }self.client.track_skill_invocations(build_track_events_context(model,