Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Improve plugin install failure analytics (#37645)

charlesgong-openai94937de51ba2
diff --git a/codex-rs/app-server/src/request_processors/plugins.rs b/codex-rs/app-server/src/request_processors/plugins.rs
index 45ff3c947a..d0f4f82458 100644
--- a/codex-rs/app-server/src/request_processors/plugins.rs
+++ b/codex-rs/app-server/src/request_processors/plugins.rs
@@ -1601,12 +1601,13 @@ impl PluginRequestProcessor {
.await
.map_err(|err| {
let error_type = remote_plugin_catalog_error_type(&err);
+ let sub_error_type = err.sub_error_type();
self.track_plugin_install_failed_for_remote_plugin(
&remote_plugin_id,
&remote_marketplace_name,
/*plugin_id*/ None,
error_type,
- /*sub_error_type*/ None,
+ sub_error_type,
err.to_string(),
);
remote_plugin_catalog_error_to_jsonrpc(
@@ -1623,14 +1624,29 @@ impl PluginRequestProcessor {
))
})?;
if remote_detail.summary.availability == PluginAvailability::DisabledByAdmin {
- return Err(invalid_request(format!(
- "remote plugin {remote_plugin_id} is disabled by admin"
- )));
+ let error_message = format!("remote plugin {remote_plugin_id} is disabled by admin");
+ self.track_plugin_install_failed_for_remote_plugin(
+ &remote_plugin_id,
+ &actual_remote_marketplace_name,
+ Some(&resolved_plugin_id),
+ "remote_plugin_not_available",
+ Some("disabled_by_admin".to_string()),
+ error_message.clone(),
+ );
+ return Err(invalid_request(error_message));
}
if remote_detail.summary.install_policy == PluginInstallPolicy::NotAvailable {
- return Err(invalid_request(format!(
- "remote plugin {remote_plugin_id} is not available for install"
- )));
+ let error_message =
+ format!("remote plugin {remote_plugin_id} is not available for install");
+ self.track_plugin_install_failed_for_remote_plugin(
+ &remote_plugin_id,
+ &actual_remote_marketplace_name,
+ Some(&resolved_plugin_id),
+ "remote_plugin_not_available",
+ Some("install_policy_not_available".to_string()),
+ error_message.clone(),
+ );
+ return Err(invalid_request(error_message));
}
// Direct install writes the same cache tree that installed-plugin sync
// prunes before the backend installed snapshot can include this plugin.
@@ -1694,12 +1710,13 @@ impl PluginRequestProcessor {
.await
.map_err(|err| {
let error_type = remote_plugin_catalog_error_type(&err);
+ let sub_error_type = err.sub_error_type();
self.track_plugin_install_failed_for_remote_plugin(
&remote_plugin_id,
&actual_remote_marketplace_name,
Some(&result.plugin_id),
error_type,
- /*sub_error_type*/ None,
+ sub_error_type,
err.to_string(),
);
remote_plugin_catalog_error_to_jsonrpc(err, "install remote plugin")
diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs
index 974d7de49b..38b85ee9cb 100644
--- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs
+++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs
@@ -589,6 +589,79 @@ async fn plugin_install_tracks_analytics_when_remote_detail_fetch_fails() -> Res
event_params["error_type"],
"remote_catalog_unexpected_status"
);
+ assert_eq!(event_params["sub_error_type"], "http_404");
+ Ok(())
+}
+
+#[tokio::test]
+async fn plugin_install_tracks_analytics_when_remote_install_is_rate_limited() -> Result<()> {
+ let codex_home = TempDir::new()?;
+ let server = MockServer::start().await;
+ let bundle_url = mount_remote_plugin_bundle(
+ &server,
+ /*status_code*/ 200,
+ remote_plugin_bundle_tar_gz_bytes("linear")?,
+ )
+ .await;
+ configure_remote_plugin_test(codex_home.path(), &server)?;
+ mount_remote_plugin_detail(&server, REMOTE_PLUGIN_ID, "1.2.3", Some(&bundle_url)).await;
+ mount_empty_remote_installed_plugins(&server).await;
+ Mock::given(method("POST"))
+ .and(path(format!(
+ "/backend-api/ps/plugins/{REMOTE_PLUGIN_ID}/install"
+ )))
+ .and(header("authorization", "Bearer chatgpt-token"))
+ .and(header("chatgpt-account-id", "account-123"))
+ .respond_with(ResponseTemplate::new(/*status_code*/ 429).set_body_string("rate limited"))
+ .mount(&server)
+ .await;
+ mount_backend_analytics_events(&server).await;
+
+ let mut mcp = TestAppServer::builder()
+ .with_codex_home(codex_home.path())
+ .without_auto_env()
+ .with_env_overrides(&[(TEST_ALLOW_HTTP_REMOTE_PLUGIN_BUNDLE_DOWNLOADS, Some("1"))])
+ .build_initialized_with_timeout(DEFAULT_TIMEOUT)
+ .await?;
+
+ let request_id = send_remote_plugin_install_request(&mut mcp, REMOTE_PLUGIN_ID).await?;
+ let err = timeout(
+ DEFAULT_TIMEOUT,
+ mcp.read_stream_until_error_message(RequestId::Integer(request_id)),
+ )
+ .await??;
+
+ assert_eq!(err.error.code, -32603);
+ assert!(err.error.message.contains("failed with status 429"));
+ wait_for_remote_plugin_request_count(
+ &server,
+ "GET",
+ "/bundles/linear.tar.gz",
+ /*expected_count*/ 1,
+ )
+ .await?;
+ wait_for_remote_plugin_request_count(
+ &server,
+ "POST",
+ &format!("/ps/plugins/{REMOTE_PLUGIN_ID}/install"),
+ /*expected_count*/ 1,
+ )
+ .await?;
+ let payload = wait_for_plugin_analytics_payload(&server).await?;
+ let event_params = &payload["events"][0]["event_params"];
+ assert_eq!(
+ payload["events"][0]["event_type"],
+ "codex_plugin_install_failed"
+ );
+ assert_eq!(event_params["plugin_id"], "linear@openai-curated-remote");
+ assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID);
+ assert_eq!(event_params["marketplace_name"], "openai-curated-remote");
+ assert_eq!(event_params["source"], "manual");
+ assert_eq!(
+ event_params["error_type"],
+ "remote_catalog_unexpected_status"
+ );
+ assert_eq!(event_params["sub_error_type"], "http_429");
Ok(())
}
@@ -612,6 +685,7 @@ async fn plugin_install_rejects_remote_plugin_disabled_by_admin_before_download(
)
.await;
mount_empty_remote_installed_plugins(&server).await;
+ mount_backend_analytics_events(&server).await;
let mut mcp = TestAppServer::builder()
.with_codex_home(codex_home.path())
@@ -649,6 +723,16 @@ async fn plugin_install_rejects_remote_plugin_disabled_by_admin_before_download(
.join("plugins/cache/openai-curated-remote/linear")
.exists()
);
+ let payload = wait_for_plugin_analytics_payload(&server).await?;
+ let event_params = &payload["events"][0]["event_params"];
+ assert_eq!(
+ payload["events"][0]["event_type"],
+ "codex_plugin_install_failed"
+ );
+ assert_eq!(event_params["plugin_id"], "linear@openai-curated-remote");
+ assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID);
+ assert_eq!(event_params["error_type"], "remote_plugin_not_available");
+ assert_eq!(event_params["sub_error_type"], "disabled_by_admin");
Ok(())
}
@@ -665,6 +749,7 @@ async fn plugin_install_rejects_remote_plugin_not_available() -> Result<()> {
)
.await;
mount_empty_remote_installed_plugins(&server).await;
+ mount_backend_analytics_events(&server).await;
let mut mcp = TestAppServer::builder()
.with_codex_home(codex_home.path())
@@ -688,6 +773,19 @@ async fn plugin_install_rejects_remote_plugin_not_available() -> Result<()> {
/*expected_count*/ 0,
)
.await?;
+ let payload = wait_for_plugin_analytics_payload(&server).await?;
+ let event_params = &payload["events"][0]["event_params"];
+ assert_eq!(
+ payload["events"][0]["event_type"],
+ "codex_plugin_install_failed"
+ );
+ assert_eq!(event_params["plugin_id"], "linear@openai-curated-remote");
+ assert_eq!(event_params["remote_plugin_id"], REMOTE_PLUGIN_ID);
+ assert_eq!(event_params["error_type"], "remote_plugin_not_available");
+ assert_eq!(
+ event_params["sub_error_type"],
+ "install_policy_not_available"
+ );
Ok(())
}
@@ -792,6 +890,64 @@ async fn plugin_install_returns_invalid_request_for_missing_marketplace_file() -
Ok(())
}
+#[tokio::test]
+async fn plugin_install_tracks_analytics_when_marketplace_file_cannot_be_read() -> Result<()> {
+ let analytics_server = start_analytics_events_server().await?;
+ let codex_home = TempDir::new()?;
+ write_analytics_config(codex_home.path(), &analytics_server.uri())?;
+ write_chatgpt_auth(
+ codex_home.path(),
+ ChatGptAuthFixture::new("chatgpt-token")
+ .account_id("account-123")
+ .chatgpt_user_id("user-123")
+ .chatgpt_account_id("account-123"),
+ AuthCredentialsStoreMode::File,
+ )?;
+ let marketplace_path = codex_home.path().join("marketplace-dir");
+ std::fs::create_dir_all(&marketplace_path)?;
+
+ let mut mcp = TestAppServer::builder()
+ .with_codex_home(codex_home.path())
+ .without_auto_env()
+ .build_initialized_with_timeout(DEFAULT_TIMEOUT)
+ .await?;
+
+ let request_id = mcp
+ .send_plugin_install_request(PluginInstallParams {
+ marketplace_path: Some(AbsolutePathBuf::try_from(marketplace_path)?),
+ remote_marketplace_name: None,
+ plugin_name: "sample-plugin".to_string(),
+ })
+ .await?;
+ let err = timeout(
+ DEFAULT_TIMEOUT,
+ mcp.read_stream_until_error_message(RequestId::Integer(request_id)),
+ )
+ .await??;
+
+ assert_eq!(err.error.code, -32603);
+ assert!(
+ err.error
+ .message
+ .contains("failed to read marketplace file")
+ );
+
+ let payload = wait_for_plugin_analytics_payload(&analytics_server).await?;
+ let event_params = &payload["events"][0]["event_params"];
+ assert_eq!(
+ payload["events"][0]["event_type"],
+ "codex_plugin_install_failed"
+ );
+ assert_eq!(event_params["plugin_id"], json!(null));
+ assert_eq!(event_params["remote_plugin_id"], json!(null));
+ assert_eq!(event_params["error_type"], "marketplace_io");
+ assert_eq!(
+ event_params["sub_error_type"],
+ "failed_to_read_marketplace_file"
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn plugin_install_returns_invalid_request_for_not_available_plugin() -> Result<()> {
let codex_home = TempDir::new()?;
@@ -1135,6 +1291,7 @@ async fn plugin_install_preserves_status_when_remote_bundle_error_body_is_too_la
assert_eq!(event_params["marketplace_name"], "openai-curated-remote");
assert_eq!(event_params["source"], "manual");
assert_eq!(event_params["error_type"], "remote_bundle_download_status");
+ assert_eq!(event_params["sub_error_type"], "http_5xx");
assert!(
!codex_home
.path()
diff --git a/codex-rs/core-plugins/src/error_subtype.rs b/codex-rs/core-plugins/src/error_subtype.rs
new file mode 100644
index 0000000000..130964f2d7
--- /dev/null
+++ b/codex-rs/core-plugins/src/error_subtype.rs
@@ -0,0 +1,13 @@
+use http::StatusCode;
+
+pub(crate) fn http_status_sub_error_type(status: StatusCode) -> &'static str {
+ match status.as_u16() {
+ 401 => "http_401",
+ 403 => "http_403",
+ 404 => "http_404",
+ 409 => "http_409",
+ 429 => "http_429",
+ _ if status.is_server_error() => "http_5xx",
+ _ => "http_other",
+ }
+}
diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs
index a31a0f6619..71d2ce70bb 100644
--- a/codex-rs/core-plugins/src/lib.rs
+++ b/codex-rs/core-plugins/src/lib.rs
@@ -1,6 +1,7 @@
mod app_mcp_routing;
mod command_migration;
mod discoverable;
+mod error_subtype;
mod http_client_selector;
pub mod installed_marketplaces;
pub mod loader;
diff --git a/codex-rs/core-plugins/src/manager.rs b/codex-rs/core-plugins/src/manager.rs
index 1519f6a0c2..48cb599929 100644
--- a/codex-rs/core-plugins/src/manager.rs
+++ b/codex-rs/core-plugins/src/manager.rs
@@ -66,6 +66,7 @@ use crate::startup_sync::sync_openai_plugins_repo;
use crate::store::PluginInstallResult as StorePluginInstallResult;
use crate::store::PluginStore;
use crate::store::PluginStoreError;
+use crate::store::error_context_sub_error_type;
use crate::tool_suggest_metadata::ToolSuggestMetadataCache;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::PluginInstallSource;
@@ -1555,6 +1556,7 @@ impl PluginsManager {
}
fn track_plugin_install_resolution_failed(&self, err: &MarketplaceError) {
+ let sub_error_type = marketplace_error_sub_error_type(err);
let plugin_id = match err {
MarketplaceError::PluginNotFound {
plugin_name,
@@ -1574,15 +1576,25 @@ impl PluginsManager {
self.track_plugin_install_failed(
&plugin_id,
marketplace_error_type(err),
- /*sub_error_type*/ None,
+ sub_error_type,
err.to_string(),
);
} else {
tracing::warn!(
error_type = %marketplace_error_type(err),
+ sub_error_type = sub_error_type.as_deref(),
error = %err,
"plugin install failed while resolving marketplace plugin"
);
+ self.emit_plugin_install_failed(
+ PluginTelemetryMetadata {
+ plugin_id: None,
+ remote_plugin_id: None,
+ capability_summary: None,
+ },
+ marketplace_error_type(err),
+ sub_error_type,
+ );
}
}
@@ -1600,13 +1612,26 @@ impl PluginsManager {
error = %error_message,
"plugin install failed"
);
+ self.emit_plugin_install_failed(
+ self.telemetry_metadata_for_plugin_id(plugin_id),
+ error_type,
+ sub_error_type,
+ );
+ }
+
+ fn emit_plugin_install_failed(
+ &self,
+ plugin: PluginTelemetryMetadata,
+ error_type: &'static str,
+ sub_error_type: Option<String>,
+ ) {
let analytics_events_client = match self.analytics_events_client.read() {
Ok(client) => client.clone(),
Err(err) => err.into_inner().clone(),
};
if let Some(analytics_events_client) = analytics_events_client {
analytics_events_client.track_plugin_install_failed(
- self.telemetry_metadata_for_plugin_id(plugin_id),
+ plugin,
self.plugin_install_source,
error_type.to_string(),
sub_error_type,
@@ -3053,8 +3078,11 @@ impl PluginInstallError {
pub fn sub_error_type(&self) -> Option<String> {
match self {
+ Self::Marketplace(err) => marketplace_error_sub_error_type(err),
+ Self::Remote(err) => err.sub_error_type(),
Self::Store(err) => err.sub_error_type(),
- Self::Marketplace(_) | Self::Remote(_) | Self::Config(_) | Self::Join(_) => None,
+ Self::Config(_) => Some("failed_to_enable_plugin".to_string()),
+ Self::Join(_) => Some("plugin_install_task_failed".to_string()),
}
}
}
@@ -3081,6 +3109,18 @@ fn marketplace_error_type(err: &MarketplaceError) -> &'static str {
}
}
+fn marketplace_error_sub_error_type(err: &MarketplaceError) -> Option<String> {
+ match err {
+ MarketplaceError::Io { context, .. } => Some(error_context_sub_error_type(context)),
+ MarketplaceError::MarketplaceNotFound { .. }
+ | MarketplaceError::InvalidMarketplaceFile { .. }
+ | MarketplaceError::PluginNotFound { .. }
+ | MarketplaceError::PluginNotAvailable { .. }
+ | MarketplaceError::PluginsDisabled
+ | MarketplaceError::InvalidPlugin(_) => None,
+ }
+}
+
fn remote_plugin_mutation_error_type(err: &RemotePluginMutationError) -> &'static str {
match err {
RemotePluginMutationError::AuthRequired => "remote_mutation_auth_required",
diff --git a/codex-rs/core-plugins/src/remote.rs b/codex-rs/core-plugins/src/remote.rs
index 457de514b2..f55fcf5f86 100644
--- a/codex-rs/core-plugins/src/remote.rs
+++ b/codex-rs/core-plugins/src/remote.rs
@@ -1,4 +1,5 @@
use crate::app_mcp_routing::apply_app_mcp_routing_policy;
+use crate::error_subtype::http_status_sub_error_type;
use crate::http_client_selector::HttpClientSelector;
use crate::loader::plugin_app_declarations_from_value;
use crate::store::PLUGINS_CACHE_DIR;
@@ -458,6 +459,36 @@ pub enum RemotePluginCatalogError {
CacheRemove(String),
}
+impl RemotePluginCatalogError {
+ /// Stable low-cardinality detail for plugin-install failure telemetry.
+ pub fn sub_error_type(&self) -> Option<String> {
+ match self {
+ Self::UnexpectedStatus { status, .. } => {
+ Some(http_status_sub_error_type(*status).to_string())
+ }
+ Self::AuthRequired
+ | Self::UnsupportedAuthMode
+ | Self::AuthToken(_)
+ | Self::Request { .. }
+ | Self::Decode { .. }
+ | Self::InvalidBaseUrl(_)
+ | Self::InvalidBaseUrlPath
+ | Self::UnknownMarketplace { .. }
+ | Self::UnexpectedPluginId { .. }
+ | Self::UnexpectedSkillName { .. }
+ | Self::UnexpectedEnabledState { .. }
+ | Self::InvalidPluginPath { .. }
+ | Self::PluginShareCheckoutNotAvailable { .. }
+ | Self::Archive { .. }
+ | Self::ArchiveJoin(_)
+ | Self::ArchiveTooLarge { .. }
+ | Self::MissingUploadEtag
+ | Self::UnexpectedResponse(_)
+ | Self::CacheRemove(_) => None,
+ }
+ }
+}
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
pub enum RemotePluginScope {
#[serde(rename = "GLOBAL")]
diff --git a/codex-rs/core-plugins/src/remote_bundle.rs b/codex-rs/core-plugins/src/remote_bundle.rs
index 25e136f8a1..8a3c1f47c6 100644
--- a/codex-rs/core-plugins/src/remote_bundle.rs
+++ b/codex-rs/core-plugins/src/remote_bundle.rs
@@ -1,3 +1,4 @@
+use crate::error_subtype::http_status_sub_error_type;
use crate::plugin_bundle_archive::PluginBundleUnpackError;
use crate::plugin_bundle_archive::unpack_plugin_bundle_tar_gz;
use crate::remote::REMOTE_GLOBAL_MARKETPLACE_NAME;
@@ -143,6 +144,9 @@ impl RemotePluginBundleInstallError {
match self {
Self::Io { context, .. } => Some(error_context_sub_error_type(context)),
Self::Store(err) => err.sub_error_type(),
+ Self::DownloadStatus { status, .. } => {
+ Some(http_status_sub_error_type(*status).to_string())
+ }
Self::MissingReleaseVersion { .. }
| Self::InvalidReleaseVersion { .. }
| Self::MissingBundleDownloadUrl { .. }
@@ -150,7 +154,6 @@ impl RemotePluginBundleInstallError {
| Self::UnsupportedBundleDownloadUrlScheme { .. }
| Self::InvalidPluginId { .. }
| Self::DownloadRequest { .. }
- | Self::DownloadStatus { .. }
| Self::DownloadBody { .. }
| Self::DownloadTooLarge { .. }
| Self::UnsupportedBundleDownloadFinalUrl { .. }
diff --git a/codex-rs/core-plugins/src/remote_legacy.rs b/codex-rs/core-plugins/src/remote_legacy.rs
index 5701637aae..65f1c04efd 100644
--- a/codex-rs/core-plugins/src/remote_legacy.rs
+++ b/codex-rs/core-plugins/src/remote_legacy.rs
@@ -1,3 +1,4 @@
+use crate::error_subtype::http_status_sub_error_type;
use crate::remote::RemotePluginServiceConfig;
use codex_http_client::RouteAwareRequestError;
use codex_login::CodexAuth;
@@ -73,6 +74,25 @@ pub enum RemotePluginMutationError {
},
}
+impl RemotePluginMutationError {
+ pub(crate) fn sub_error_type(&self) -> Option<String> {
+ match self {
+ Self::UnexpectedStatus { status, .. } => {
+ Some(http_status_sub_error_type(*status).to_string())
+ }
+ Self::AuthRequired
+ | Self::UnsupportedAuthMode
+ | Self::AuthToken(_)
+ | Self::InvalidBaseUrl(_)
+ | Self::InvalidBaseUrlPath
+ | Self::Request { .. }
+ | Self::Decode { .. }
+ | Self::UnexpectedPluginId { .. }
+ | Self::UnexpectedEnabledState { .. } => None,
+ }
+ }
+}
+
#[derive(Debug, thiserror::Error)]
pub enum RemotePluginFetchError {
#[error("invalid chatgpt base url for remote featured plugin request: {0}")]