Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add plugin-attributed skill telemetry (#40724)

jwang-openaic51e7b373647
diff --git a/codex-rs/core/src/skills.rs b/codex-rs/core/src/skills.rs
index 4d35dd403a..152073f39b 100644
--- a/codex-rs/core/src/skills.rs
+++ b/codex-rs/core/src/skills.rs
@@ -13,6 +13,7 @@ use codex_protocol::protocol::SkillScope;
use codex_skills::SkillMetadata;
use codex_skills_extension::HostSkillsLoadInput;
use codex_skills_extension::detect_implicit_skill_invocation;
+use codex_skills_extension::record_plugin_turn_usage;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
use codex_utils_plugins::PluginSkillRoot;
@@ -44,9 +45,17 @@ pub(crate) fn emit_explicit_skill_invocations(
.iter()
.map(|skill| &skill.path_to_skills_md)
.collect::<HashSet<_>>();
+ let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str());
+ let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing();
for skill in mentioned_skills {
let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str());
+ let plugin_id_tag =
+ sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed"));
let status = if injected_skill_paths.contains(&skill.path_to_skills_md) {
+ record_plugin_turn_usage(
+ turn_context.extension_data.as_ref(),
+ skill.plugin_id.as_deref(),
+ );
"ok"
} else {
"error"
@@ -58,6 +67,9 @@ pub(crate) fn emit_explicit_skill_invocations(
("status", status),
("skill", skill_name_tag.as_str()),
("invoke_type", "explicit"),
+ ("plugin_id", plugin_id_tag.as_str()),
+ ("model_slug", model_slug_tag.as_str()),
+ ("reasoning_effort", reasoning_effort.as_str()),
],
);
}
@@ -123,6 +135,14 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(
return;
}
let skill_name_tag = sanitize_metric_tag_value(skill_name.as_str());
+ let plugin_id_tag =
+ sanitize_metric_tag_value(invocation.plugin_id.as_deref().unwrap_or("unattributed"));
+ let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str());
+ let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing();
+ record_plugin_turn_usage(
+ turn_context.extension_data.as_ref(),
+ invocation.plugin_id.as_deref(),
+ );
for contributor in sess.services.extensions.skill_invocation_contributors() {
contributor
@@ -144,6 +164,9 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(
("status", "ok"),
("skill", skill_name_tag.as_str()),
("invoke_type", "implicit"),
+ ("plugin_id", plugin_id_tag.as_str()),
+ ("model_slug", model_slug_tag.as_str()),
+ ("reasoning_effort", reasoning_effort.as_str()),
],
);
sess.services
diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rs
index c77e26665a..99cefa6229 100644
--- a/codex-rs/ext/skills/src/extension.rs
+++ b/codex-rs/ext/skills/src/extension.rs
@@ -67,6 +67,7 @@ use crate::state::HostSkillsStepState;
use crate::state::SkillsSessionState;
use crate::state::SkillsThreadState;
use crate::state::SkillsTurnState;
+use crate::telemetry::SkillTelemetry;
use crate::tools::SkillAnalytics;
use crate::tools::SkillToolAuthority;
use crate::tools::skill_tools;
@@ -670,6 +671,7 @@ pub fn install_with_providers_and_metrics<C>(
shadow_selection: Arc::new(ShadowSelectionExperiment::new(metrics_client)),
});
registry.thread_lifecycle_contributor(extension.clone());
+ registry.turn_lifecycle_contributor(Arc::new(SkillTelemetry));
registry.config_contributor(extension.clone());
registry.prompt_contributor(extension.clone());
registry.turn_input_contributor(extension.clone());
diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rs
index 940aa9657c..b3fdc291b9 100644
--- a/codex-rs/ext/skills/src/lib.rs
+++ b/codex-rs/ext/skills/src/lib.rs
@@ -20,6 +20,7 @@ mod selection;
mod shadow_selection_experiment;
mod sources;
mod state;
+mod telemetry;
mod tools;
mod warnings;
mod world_state;
@@ -43,6 +44,7 @@ pub use provider::OrchestratorSkillProvider;
pub use provider::SkillProvider;
pub use sources::SkillProviderSource;
pub use sources::SkillProviders;
+pub use telemetry::record_plugin_turn_usage;
/// Recognizes persisted explicit skill prompts without exposing their fragment implementation.
pub fn is_skill_prompt_fragment(text: &str) -> bool {
diff --git a/codex-rs/ext/skills/src/provider/orchestrator.rs b/codex-rs/ext/skills/src/provider/orchestrator.rs
index 07643b5ca7..e984693620 100644
--- a/codex-rs/ext/skills/src/provider/orchestrator.rs
+++ b/codex-rs/ext/skills/src/provider/orchestrator.rs
@@ -276,6 +276,10 @@ fn catalog_entry_from_resource(resource: &Resource) -> Option<SkillCatalogEntry>
)
.with_display_path(uri)
.with_alias_root(format!("skill://{namespace}"));
+ entry.plugin_id = meta
+ .get("plugin_id")
+ .and_then(serde_json::Value::as_str)
+ .map(str::to_owned);
entry.canonical_skill_id = meta
.get("skill_id")
.and_then(serde_json::Value::as_str)
diff --git a/codex-rs/ext/skills/src/telemetry.rs b/codex-rs/ext/skills/src/telemetry.rs
new file mode 100644
index 0000000000..070cbb9762
--- /dev/null
+++ b/codex-rs/ext/skills/src/telemetry.rs
@@ -0,0 +1,159 @@
+use std::collections::HashSet;
+use std::sync::Arc;
+use std::sync::Mutex;
+use std::sync::Weak;
+use std::time::Instant;
+
+use codex_extension_api::ExtensionData;
+use codex_extension_api::ExtensionFuture;
+use codex_extension_api::TurnAbortInput;
+use codex_extension_api::TurnErrorInput;
+use codex_extension_api::TurnLifecycleContributor;
+use codex_extension_api::TurnStartInput;
+use codex_extension_api::TurnStopInput;
+use codex_otel::sanitize_metric_tag_value;
+use codex_protocol::openai_models::ModelInfo;
+
+use crate::state::SkillsSessionState;
+
+/// Attributes a successful host skill invocation to its turn's plugin latency metrics.
+pub fn record_plugin_turn_usage(turn_store: &ExtensionData, plugin_id: Option<&str>) {
+ if let Some(turn) = turn_store.get::<SkillTurnMetrics>() {
+ turn.record_plugin(plugin_id);
+ }
+}
+
+pub(crate) struct SkillTurnMetrics {
+ pub(crate) turn_id: String,
+ model_slug: String,
+ pub(crate) reasoning_effort: String,
+ started_at: Instant,
+ usage: Mutex<Option<TurnUsage>>,
+}
+
+#[derive(Default)]
+struct TurnUsage {
+ plugins: HashSet<String>,
+ failed: bool,
+}
+
+impl SkillTurnMetrics {
+ pub(crate) fn record_plugin(&self, plugin_id: Option<&str>) {
+ if let Some(usage) = self
+ .usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .as_mut()
+ {
+ usage
+ .plugins
+ .insert(plugin_id.unwrap_or("unattributed").to_string());
+ }
+ }
+
+ fn finish(&self, session_store: &ExtensionData, status: &str) {
+ let Some(usage) = self
+ .usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .take()
+ else {
+ return;
+ };
+ let Some(metrics) = session_store
+ .get::<SkillsSessionState>()
+ .and_then(|state| state.extension_metrics.clone())
+ else {
+ return;
+ };
+ let status = if status == "completed" && usage.failed {
+ "error"
+ } else {
+ status
+ };
+ // Seconds keep multi-minute turns within the standard histogram buckets.
+ // Round up so integer-second bucket boundaries classify fractional durations correctly.
+ let duration_seconds = self.started_at.elapsed().as_secs_f64().ceil() as i64;
+ for plugin_id in usage.plugins {
+ let plugin_id = sanitize_metric_tag_value(&plugin_id);
+ metrics.histogram(
+ "codex.skill.turn.duration_seconds",
+ duration_seconds,
+ &[
+ ("plugin_id", plugin_id.as_str()),
+ ("model_slug", self.model_slug.as_str()),
+ ("reasoning_effort", self.reasoning_effort.as_str()),
+ ("status", status),
+ ],
+ );
+ }
+ }
+}
+
+#[derive(Default)]
+pub(crate) struct ActiveSkillTurnMetrics(pub(crate) Mutex<Weak<SkillTurnMetrics>>);
+
+pub(crate) struct SkillTelemetry;
+
+impl TurnLifecycleContributor for SkillTelemetry {
+ fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> {
+ Box::pin(async move {
+ let reasoning_effort = input
+ .collaboration_mode
+ .reasoning_effort()
+ .or_else(|| {
+ input
+ .thread_store
+ .get::<ModelInfo>()
+ .and_then(|model| model.default_reasoning_level.clone())
+ })
+ .map(|effort| effort.to_string())
+ .unwrap_or_else(|| "default".to_string());
+ input.turn_store.insert(SkillTurnMetrics {
+ turn_id: input.turn_id.to_string(),
+ model_slug: sanitize_metric_tag_value(input.collaboration_mode.model()),
+ reasoning_effort,
+ started_at: Instant::now(),
+ usage: Mutex::new(Some(TurnUsage::default())),
+ });
+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {
+ *input
+ .thread_store
+ .get_or_init(ActiveSkillTurnMetrics::default)
+ .0
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::downgrade(&turn);
+ }
+ })
+ }
+
+ fn on_turn_stop<'a>(&'a self, input: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> {
+ Box::pin(async move {
+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {
+ turn.finish(input.session_store, "completed");
+ }
+ })
+ }
+
+ fn on_turn_abort<'a>(&'a self, input: TurnAbortInput<'a>) -> ExtensionFuture<'a, ()> {
+ Box::pin(async move {
+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {
+ turn.finish(input.session_store, "aborted");
+ }
+ })
+ }
+
+ fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> {
+ Box::pin(async move {
+ if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>()
+ && let Some(usage) = turn
+ .usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .as_mut()
+ {
+ usage.failed = true;
+ }
+ })
+ }
+}
diff --git a/codex-rs/ext/skills/src/tools/mod.rs b/codex-rs/ext/skills/src/tools/mod.rs
index 4869c5f497..54d663e2a5 100644
--- a/codex-rs/ext/skills/src/tools/mod.rs
+++ b/codex-rs/ext/skills/src/tools/mod.rs
@@ -11,6 +11,7 @@ use codex_analytics::SkillInvocationLocation;
use codex_analytics::build_track_events_context;
use codex_exec_server::FileSystemSandboxContext;
use codex_extension_api::ExtensionData;
+use codex_extension_api::ExtensionMetrics;
use codex_extension_api::FunctionCallError;
use codex_extension_api::JsonToolOutput;
use codex_extension_api::ResponsesApiTool;
@@ -24,6 +25,7 @@ use codex_extension_api::ToolSpec;
use codex_extension_api::parse_tool_input_schema;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_mcp::McpResourceClient;
+use codex_otel::sanitize_metric_tag_value;
use codex_tools::ResponsesApiNamespace;
use codex_tools::ResponsesApiNamespaceTool;
use codex_tools::default_namespace_description;
@@ -43,6 +45,7 @@ use crate::shadow_selection_experiment::ShadowSelectionExperiment;
use crate::sources::SkillProviders;
use crate::state::SkillsSessionState;
use crate::state::SkillsThreadState;
+use crate::telemetry::ActiveSkillTurnMetrics;
mod list;
mod read;
@@ -96,6 +99,8 @@ pub(crate) fn skill_tools(
#[derive(Clone)]
pub(crate) struct SkillAnalytics {
client: AnalyticsEventsClient,
+ metrics: Option<Arc<dyn ExtensionMetrics>>,
+ active_turn: Arc<ActiveSkillTurnMetrics>,
thread_id: String,
product_client_id: String,
}
@@ -110,6 +115,10 @@ impl SkillAnalytics {
Some(Self {
client: client.as_ref().clone(),
+ metrics: session_store
+ .get::<SkillsSessionState>()
+ .and_then(|state| state.extension_metrics.clone()),
+ active_turn: thread_store.get_or_init(ActiveSkillTurnMetrics::default),
thread_id: thread_store.level_id().to_string(),
product_client_id: originator.0.clone(),
})
@@ -122,6 +131,42 @@ impl SkillAnalytics {
turn_id: String,
invocation_type: InvocationType,
) {
+ let turn_metrics = self
+ .active_turn
+ .0
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .upgrade()
+ .filter(|turn| turn.turn_id == turn_id);
+ if let Some(turn_metrics) = &turn_metrics {
+ turn_metrics.record_plugin(skill.plugin_id.as_deref());
+ }
+ if let Some(metrics) = &self.metrics {
+ let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str());
+ let plugin_id_tag =
+ sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed"));
+ let model_slug_tag = sanitize_metric_tag_value(model.as_str());
+ let reasoning_effort = turn_metrics
+ .as_ref()
+ .map(|turn| turn.reasoning_effort.as_str())
+ .unwrap_or("unknown");
+ let invoke_type = match invocation_type {
+ InvocationType::Explicit => "explicit",
+ InvocationType::Implicit => "implicit",
+ };
+ metrics.counter(
+ "codex.skill.injected",
+ /*inc*/ 1,
+ &[
+ ("status", "ok"),
+ ("skill", skill_name_tag.as_str()),
+ ("invoke_type", invoke_type),
+ ("plugin_id", plugin_id_tag.as_str()),
+ ("model_slug", model_slug_tag.as_str()),
+ ("reasoning_effort", reasoning_effort),
+ ],
+ );
+ }
self.client.track_skill_invocations(
build_track_events_context(
model,