Lightweight coding agent that runs in your terminal
Rust
Add a configurable skill catalog token budget (#38978)
jif632e35ce8d5d
diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rsindex 8494c2ba66..6173999e49 100644--- a/codex-rs/app-server/src/extensions.rs+++ b/codex-rs/app-server/src/extensions.rs@@ -115,6 +115,7 @@ wherecodex_otel::global(),|config: &Config| codex_skills_extension::SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: config.bundled_skills_enabled(),orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: configdiff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rsindex 102f40aec6..c2261f1c83 100644--- a/codex-rs/cli/src/main.rs+++ b/codex-rs/cli/src/main.rs@@ -2127,6 +2127,7 @@ async fn run_debug_prompt_input_command(codex_skills_extension::install(&mut extensions, |config: &Config| {codex_skills_extension::SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: config.bundled_skills_enabled(),orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: configdiff --git a/codex-rs/config/src/skills_config.rs b/codex-rs/config/src/skills_config.rsindex 8b0593c887..cc944d817c 100644--- a/codex-rs/config/src/skills_config.rs+++ b/codex-rs/config/src/skills_config.rs@@ -1,6 +1,7 @@//! Skill-related configuration types shared across crates.use std::collections::HashSet;+use std::num::NonZeroUsize;use crate::ConfigLayerSource;use crate::ConfigLayerStack;@@ -36,6 +37,11 @@ pub struct SkillsConfig {#[serde(default, skip_serializing_if = "Option::is_none")]pub include_instructions: Option<bool>,+ /// Maximum tokens used by the available-skills catalog. Defaults to 2% of+ /// the model context window and is capped at 10,000 tokens when set.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub max_context_tokens: Option<NonZeroUsize>,+#[serde(default, skip_serializing_if = "Vec::is_empty")]pub config: Vec<SkillConfig>,}diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.jsonindex d05a206097..a060ca3e33 100644--- a/codex-rs/core/config.schema.json+++ b/codex-rs/core/config.schema.json@@ -3231,6 +3231,12 @@"include_instructions": {"description": "Whether turns receive the automatic skills instructions block.","type": "boolean"+ },+ "max_context_tokens": {+ "description": "Maximum tokens used by the available-skills catalog. Defaults to 2% of the model context window and is capped at 10,000 tokens when set.",+ "format": "uint",+ "minimum": 1.0,+ "type": "integer"}},"type": "object"diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rsindex b32181c988..2ce99fed3c 100644--- a/codex-rs/core/src/config/config_tests.rs+++ b/codex-rs/core/src/config/config_tests.rs@@ -358,6 +358,7 @@ fn parses_bundled_skills_config() {r#"[skills]include_instructions = false+max_context_tokens = 1200[skills.bundled]enabled = false@@ -370,9 +371,12 @@ enabled = falseSome(SkillsConfig {bundled: Some(BundledSkillsConfig { enabled: false }),include_instructions: Some(false),+ max_context_tokens: std::num::NonZeroUsize::new(1_200),config: Vec::new(),}));++ assert!(toml::from_str::<ConfigToml>("[skills]\nmax_context_tokens = 0\n").is_err());}#[test]diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rsindex 2361fea0cd..d7716076c8 100644--- a/codex-rs/core/src/config/mod.rs+++ b/codex-rs/core/src/config/mod.rs@@ -130,6 +130,7 @@ use std::collections::BTreeMap;use std::collections::HashMap;use std::collections::HashSet;use std::io::ErrorKind;+use std::num::NonZeroUsize;use std::path::Path;use std::path::PathBuf;use std::sync::Arc;@@ -649,6 +650,9 @@ pub struct Config {/// Whether to inject the `<skills_instructions>` developer block.pub include_skill_instructions: bool,+ /// Optional token budget override for the available-skills catalog.+ pub skill_max_context_tokens: Option<NonZeroUsize>,+/// Whether orchestrator-owned skills are exposed to the model.pub orchestrator_skills_enabled: bool,@@ -3797,6 +3801,10 @@ impl Config {.as_ref().and_then(|skills| skills.include_instructions).unwrap_or(true);+ let skill_max_context_tokens = cfg+ .skills+ .as_ref()+ .and_then(|skills| skills.max_context_tokens);let include_environment_context = cfg.include_environment_context.unwrap_or(true);let guardian_policy_config =guardian_policy_config_from_requirements(config_layer_stack.requirements_toml())@@ -4001,6 +4009,7 @@ impl Config {include_apps_instructions,include_collaboration_mode_instructions,include_skill_instructions,+ skill_max_context_tokens,orchestrator_skills_enabled,orchestrator_mcp_enabled,include_environment_context,diff --git a/codex-rs/core/tests/suite/model_visible_layout.rs b/codex-rs/core/tests/suite/model_visible_layout.rsindex d2fafb3d13..57abdaac2b 100644--- a/codex-rs/core/tests/suite/model_visible_layout.rs+++ b/codex-rs/core/tests/suite/model_visible_layout.rs@@ -78,6 +78,7 @@ fn skills_extensions() -> Arc<ExtensionRegistry<Config>> {let mut extensions = ExtensionRegistryBuilder::<Config>::new();install(&mut extensions, |config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: config.bundled_skills_enabled(),orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: config.features.enabled(Feature::SkillSearch),diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rsindex c8041493a9..5ff15dff10 100644--- a/codex-rs/core/tests/suite/plugins.rs+++ b/codex-rs/core/tests/suite/plugins.rs@@ -75,6 +75,7 @@ fn skills_extensions() -> Arc<ExtensionRegistry<Config>> {let mut extensions = ExtensionRegistryBuilder::<Config>::new();install(&mut extensions, |config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: config.bundled_skills_enabled(),orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: config.features.enabled(Feature::SkillSearch),diff --git a/codex-rs/core/tests/suite/skills.rs b/codex-rs/core/tests/suite/skills.rsindex a21787cf4b..fcc632ed14 100644--- a/codex-rs/core/tests/suite/skills.rs+++ b/codex-rs/core/tests/suite/skills.rs@@ -157,6 +157,7 @@ async fn user_turn_selects_symlinked_skill_by_advertised_discovery_path() -> Reslet mut extensions = ExtensionRegistryBuilder::<Config>::new();install(&mut extensions, |config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: false,shadow_selection_enabled: false,diff --git a/codex-rs/core/tests/suite/skills_extension.rs b/codex-rs/core/tests/suite/skills_extension.rsindex 3e83163e5b..d7b0211e6c 100644--- a/codex-rs/core/tests/suite/skills_extension.rs+++ b/codex-rs/core/tests/suite/skills_extension.rs@@ -258,6 +258,7 @@ fn catalog_extensions(install_with_providers(&mut extensions, providers, |config: &Config| {SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: false,shadow_selection_enabled: false,@@ -534,6 +535,7 @@ async fn capability_sections_render_in_order_with_host_repo_and_plugin_skills()let mut extensions = ExtensionRegistryBuilder::<Config>::new();install(&mut extensions, |config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: config.bundled_skills_enabled(),orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: config.features.enabled(Feature::SkillSearch),@@ -960,6 +962,7 @@ async fn explicit_only_orchestrator_skill_is_hidden_but_can_be_invoked() -> Resu.with_orchestrator_provider(Arc::new(OrchestratorSkillProvider::new())),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: true,shadow_selection_enabled: false,@@ -1120,6 +1123,7 @@ async fn production_turn_aliases_discovered_singleton_orchestrator_root() -> Res.with_orchestrator_provider(Arc::new(OrchestratorSkillProvider::new())),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: true,shadow_selection_enabled: false,@@ -1412,6 +1416,7 @@ async fn production_turn_aliases_combined_skill_catalogs_under_shared_budget() -.with_host_provider(Arc::new(HostSkillProvider::new())),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: true,shadow_selection_enabled: false,@@ -1512,6 +1517,7 @@ async fn production_turn_scales_extension_catalog_from_resolved_model_window() -)),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: false,shadow_selection_enabled: false,@@ -1619,6 +1625,52 @@ async fn production_turn_shares_catalog_budget_across_host_and_executor_sectionsOk(())}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn production_turn_uses_configured_skill_catalog_token_budget() -> Result<()> {+ let server = responses::start_mock_server().await;+ let response = mount_sse_once(+ &server,+ sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),+ )+ .await;+ let codex_home = Arc::new(TempDir::new()?);+ std::fs::write(+ codex_home.path().join("config.toml"),+ "[skills]\nmax_context_tokens = 800\n",+ )?;+ write_host_skills(codex_home.path(), &HOST_CATALOG)?;+ let (extensions, _event_rx) = catalog_extensions(+ executor_catalog(&EXECUTOR_CATALOG),+ /*include_host_provider*/ true,+ );+ let mut builder = test_codex()+ .with_home(codex_home)+ .with_extensions(extensions)+ .with_model_info_override("gpt-5.5", |model_info| {+ model_info.context_window = Some(SHORTENING_CONTEXT_WINDOW);+ model_info.max_context_window = None;+ })+ .with_config(configure_catalog_test);+ let test = builder.build_with_auto_env(&server).await?;++ test.submit_turn("Inspect the available skills.").await?;+ let request = response.single_request();+ let developer_texts = request.message_input_texts("developer");+ let host_lines = skill_lines(catalog_text(&developer_texts, "host"), "host");+ let executor_lines = skill_lines(catalog_text(&developer_texts, "exec"), "exec");+ let combined_lines = host_lines+ .iter()+ .chain(executor_lines.iter())+ .copied()+ .collect::<Vec<_>>();++ assert_full_descriptions(&host_lines, &HOST_CATALOG);+ assert_full_descriptions(&executor_lines, &EXECUTOR_CATALOG);+ assert!(metadata_cost(&combined_lines) <= 800);++ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn production_turn_keeps_full_host_only_catalog_when_it_fits() -> Result<()> {let (developer_texts, _) =@@ -1785,6 +1837,7 @@ async fn production_turn_uses_provider_host_catalog_and_core_snapshot_injection(})),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: false,shadow_selection_enabled: false,@@ -1885,6 +1938,7 @@ async fn production_turn_suppresses_only_the_superseded_host_skill_prompt() -> R)),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: false,shadow_selection_enabled: false,@@ -2133,6 +2187,7 @@ async fn production_turn_keeps_orchestrator_world_state_incremental_across_turns.with_orchestrator_provider(Arc::new(CatalogSkillProvider { catalog })),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: false,@@ -2398,6 +2453,7 @@ async fn production_turn_fairly_shortens_extension_catalog_descriptions() -> Res)),|config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: false,orchestrator_skills_enabled: false,shadow_selection_enabled: false,diff --git a/codex-rs/core/tests/suite/token_budget.rs b/codex-rs/core/tests/suite/token_budget.rsindex ff7572c2eb..4406d7a426 100644--- a/codex-rs/core/tests/suite/token_budget.rs+++ b/codex-rs/core/tests/suite/token_budget.rs@@ -1436,6 +1436,7 @@ async fn new_context_tool_skips_auto_compact_fallback() -> Result<()> {let mut extensions = ExtensionRegistryBuilder::<Config>::new();install(&mut extensions, |config: &Config| SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: config.bundled_skills_enabled(),orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: config.features.enabled(Feature::SkillSearch),diff --git a/codex-rs/ext/skills/src/config.rs b/codex-rs/ext/skills/src/config.rsindex f0198145de..fe98df93bd 100644--- a/codex-rs/ext/skills/src/config.rs+++ b/codex-rs/ext/skills/src/config.rs@@ -1,8 +1,12 @@+use std::num::NonZeroUsize;+/// Host-supplied configuration used by the skills extension.#[derive(Clone, Debug, Eq, PartialEq)]pub struct SkillsExtensionConfig {/// Whether the available-skills catalog is included in model context.pub include_instructions: bool,+ /// Optional token budget override for the available-skills catalog.+ pub max_context_tokens: Option<NonZeroUsize>,/// Whether bundled skills are eligible for discovery.pub bundled_skills_enabled: bool,/// Whether orchestrator-owned skills are eligible for discovery.diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rsindex 2227688a97..efc58945f8 100644--- a/codex-rs/ext/skills/src/extension.rs+++ b/codex-rs/ext/skills/src/extension.rs@@ -228,7 +228,7 @@ where&catalog,include_usage,SkillCatalogRenderPolicy::ExtensionCompatible,- skill_metadata_budget(/*context_window*/ None),+ skill_metadata_budget(/*context_window*/ None, config.max_context_tokens),);if let Some(message) = rendered.warning_message {self.emit_warning(thread_store.level_id(), /*turn_id*/ None, message);@@ -420,7 +420,8 @@ wherelet context_window = model_info.as_deref().and_then(ModelInfo::resolved_context_window);- let metadata_budget = skill_metadata_budget(context_window);+ let metadata_budget =+ skill_metadata_budget(context_window, config.max_context_tokens);let rendered = render_catalog(extension_metrics.as_deref(),CatalogSurface::TurnInput,diff --git a/codex-rs/ext/skills/src/render.rs b/codex-rs/ext/skills/src/render.rsindex 4bed6c2987..92cba67ad6 100644--- a/codex-rs/ext/skills/src/render.rs+++ b/codex-rs/ext/skills/src/render.rs@@ -1,4 +1,5 @@use std::borrow::Cow;+use std::num::NonZeroUsize;use codex_protocol::protocol::SkillScope;use codex_utils_string::approx_token_count;@@ -14,6 +15,7 @@ use crate::fragments::AvailableSkillsInstructions;use crate::host_aliases::shared_host_alias_roots;const DEFAULT_SKILL_METADATA_CHAR_BUDGET: usize = 8_000;+const MAX_CONFIGURED_SKILL_METADATA_TOKEN_BUDGET: usize = 10_000;const MAX_SKILL_PROMPT_BYTES: usize = 8_000;const SKILL_METADATA_CONTEXT_WINDOW_PERCENT: usize = 2;const MAX_CATALOG_SKILL_DESCRIPTION_CHARS: usize = 1_024;@@ -122,7 +124,18 @@ impl SkillRenderReport {}}-pub(crate) fn skill_metadata_budget(context_window: Option<i64>) -> SkillMetadataBudget {+pub(crate) fn skill_metadata_budget(+ context_window: Option<i64>,+ max_context_tokens: Option<NonZeroUsize>,+) -> SkillMetadataBudget {+ if let Some(max_context_tokens) = max_context_tokens {+ return SkillMetadataBudget::Tokens(+ max_context_tokens+ .get()+ .min(MAX_CONFIGURED_SKILL_METADATA_TOKEN_BUDGET),+ );+ }+context_window.and_then(|window| usize::try_from(window).ok()).filter(|window| *window > 0)diff --git a/codex-rs/ext/skills/src/render_tests.rs b/codex-rs/ext/skills/src/render_tests.rsindex 93de8e65b0..090224fc58 100644--- a/codex-rs/ext/skills/src/render_tests.rs+++ b/codex-rs/ext/skills/src/render_tests.rs@@ -194,17 +194,27 @@ fn description_selection_follows_render_policy() {#[test]fn catalog_budget_uses_context_percentage_or_character_fallback() {assert_eq!(- skill_metadata_budget(Some(100_000)),+ skill_metadata_budget(Some(100_000), /*max_context_tokens*/ None),SkillMetadataBudget::Tokens(2_000));assert_eq!(- skill_metadata_budget(Some(400_000)),+ skill_metadata_budget(Some(400_000), /*max_context_tokens*/ None),SkillMetadataBudget::Tokens(8_000));assert_eq!(- skill_metadata_budget(/*context_window*/ None),+ skill_metadata_budget(+ /*context_window*/ None, /*max_context_tokens*/ None+ ),SkillMetadataBudget::Characters(8_000));+ assert_eq!(+ skill_metadata_budget(Some(100_000), NonZeroUsize::new(5_000)),+ SkillMetadataBudget::Tokens(5_000)+ );+ assert_eq!(+ skill_metadata_budget(/*context_window*/ None, NonZeroUsize::new(50_000)),+ SkillMetadataBudget::Tokens(10_000)+ );}#[test]diff --git a/codex-rs/ext/skills/src/world_state_catalogs.rs b/codex-rs/ext/skills/src/world_state_catalogs.rsindex 8c685a1bce..2e4941db08 100644--- a/codex-rs/ext/skills/src/world_state_catalogs.rs+++ b/codex-rs/ext/skills/src/world_state_catalogs.rs@@ -107,6 +107,7 @@ impl<'a> CatalogContext<'a> {let context_window = model_info.as_deref().and_then(ModelInfo::resolved_context_window);+ let metadata_budget = skill_metadata_budget(context_window, config.max_context_tokens);let emitted_warnings = input.turn_store.get_or_init(EmittedCatalogBudgetWarnings::default);@@ -127,7 +128,7 @@ impl<'a> CatalogContext<'a> {input,thread_state,config,- metadata_budget: skill_metadata_budget(context_window),+ metadata_budget,include_usage,warning_emitter,})diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rsindex b6fd8b4b83..52192c35de 100644--- a/codex-rs/ext/skills/tests/skills_extension.rs+++ b/codex-rs/ext/skills/tests/skills_extension.rs@@ -2393,6 +2393,7 @@ fn default_config() -> TestConfig {fn skills_extension_config(config: &TestConfig) -> SkillsExtensionConfig {SkillsExtensionConfig {include_instructions: config.include_instructions,+ max_context_tokens: None,bundled_skills_enabled: config.bundled_skills_enabled,orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: config.shadow_selection_enabled,diff --git a/codex-rs/mcp-server/src/message_processor.rs b/codex-rs/mcp-server/src/message_processor.rsindex 97cbd97356..25c5faa36c 100644--- a/codex-rs/mcp-server/src/message_processor.rs+++ b/codex-rs/mcp-server/src/message_processor.rs@@ -88,6 +88,7 @@ impl MessageProcessor {codex_otel::global(),|config: &Config| codex_skills_extension::SkillsExtensionConfig {include_instructions: config.include_skill_instructions,+ max_context_tokens: config.skill_max_context_tokens,bundled_skills_enabled: config.bundled_skills_enabled(),orchestrator_skills_enabled: config.orchestrator_skills_enabled,shadow_selection_enabled: configdiff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rsindex f01bc12251..f2591ca029 100644--- a/codex-rs/thread-manager-sample/src/main.rs+++ b/codex-rs/thread-manager-sample/src/main.rs@@ -215,6 +215,7 @@ fn new_config(model: Option<String>, arg0_paths: Arg0DispatchPaths) -> anyhow::Rinclude_apps_instructions: false,include_collaboration_mode_instructions: false,include_skill_instructions: false,+ skill_max_context_tokens: None,orchestrator_skills_enabled: false,orchestrator_mcp_enabled: false,include_environment_context: false,