Lightweight coding agent that runs in your terminal
Rust
Preserve URLs and email addresses during external agent migration (#53706)
jifcb7091b18037
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lockindex 65deecac1b..b4a21c2e4f 100644--- a/MODULE.bazel.lock+++ b/MODULE.bazel.lock@@ -1307,6 +1307,7 @@"libz-sys_1.1.25": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0.98\"},{\"kind\":\"build\",\"name\":\"cmake\",\"optional\":true,\"req\":\"^0.1.50\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.43\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.9\"},{\"kind\":\"build\",\"name\":\"vcpkg\",\"req\":\"^0.2.11\"}],\"features\":{\"asm\":[],\"default\":[\"libc\",\"stock-zlib\"],\"static\":[],\"stock-zlib\":[],\"zlib-ng\":[\"libc\",\"cmake\"],\"zlib-ng-no-cmake-experimental-community-maintained\":[\"libc\"]}}","line-clipping_0.3.7": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.11\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.26\"}],\"features\":{}}","link-section_0.17.2": "{\"dependencies\":[{\"features\":[\"link_section\"],\"name\":\"linktime-proc-macro\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"macrotest\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2\"}],\"features\":{\"default\":[\"proc_macro\",\"std\"],\"proc_macro\":[\"dep:linktime-proc-macro\"],\"std\":[]}}",+ "linkify_0.11.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"memchr\",\"req\":\"^2.0.1\"}],\"features\":{}}","linktime-proc-macro_0.1.0": "{\"dependencies\":[],\"features\":{\"ctor\":[],\"default\":[],\"dtor\":[],\"link_section\":[]}}","linux-keyutils_0.2.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4\"},{\"default_features\":false,\"features\":[\"std\",\"derive\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.4.11\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.132\"},{\"kind\":\"dev\",\"name\":\"zeroize\",\"req\":\"^1.5.7\"}],\"features\":{\"default\":[],\"std\":[\"bitflags/std\"]}}","linux-raw-sys_0.12.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.100\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"}],\"features\":{\"auxvec\":[],\"bootparam\":[],\"btrfs\":[],\"default\":[\"std\",\"general\",\"errno\"],\"elf\":[],\"elf_uapi\":[],\"errno\":[],\"general\":[],\"if_arp\":[],\"if_ether\":[],\"if_packet\":[],\"if_tun\":[],\"image\":[],\"io_uring\":[],\"ioctl\":[],\"landlock\":[],\"loop_device\":[],\"mempolicy\":[],\"net\":[],\"netlink\":[],\"no_std\":[],\"prctl\":[],\"ptrace\":[],\"rustc-dep-of-std\":[\"core\",\"no_std\"],\"std\":[],\"system\":[],\"vm_sockets\":[],\"xdp\":[]}}",diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lockindex 48ebe98c8a..72658b9087 100644--- a/codex-rs/Cargo.lock+++ b/codex-rs/Cargo.lock@@ -3403,6 +3403,7 @@ dependencies = ["codex-rollout","codex-thread-store","codex-utils-output-truncation",+ "linkify","pretty_assertions","serde","serde_json",@@ -10004,6 +10005,15 @@ version = "0.17.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4d1e908a416d6e9f725743b84a36feea40c4c131e805fbc26d61f9f451f36080"+[[package]]+name = "linkify"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "92f9ae2f3d31697697311f016320fe984127e636b45cf5eb5c175b7eb103dadb"+dependencies = [+ "memchr",+]+[[package]]name = "linktime-proc-macro"version = "0.1.0"diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.tomlindex 8fffe579a7..465e7aeb4f 100644--- a/codex-rs/Cargo.toml+++ b/codex-rs/Cargo.toml@@ -407,6 +407,7 @@ libc = "0.2.182"# Keep SQLx's bundled SQLite on a version containing the WAL-reset corruption fix:# https://www.sqlite.org/wal.html#the_wal_reset_buglibsqlite3-sys = { version = "0.37", default-features = false }+linkify = "0.11"log = "0.4"lru = "0.18.2"maplit = "1.0.2"diff --git a/codex-rs/app-server/tests/suite/v2/external_agent_config.rs b/codex-rs/app-server/tests/suite/v2/external_agent_config.rsindex 3620695b5c..fe05440a40 100644--- a/codex-rs/app-server/tests/suite/v2/external_agent_config.rs+++ b/codex-rs/app-server/tests/suite/v2/external_agent_config.rs@@ -401,7 +401,10 @@ async fn external_agent_config_import_source_remains_attribution_only() -> Resullet codex_home = TempDir::new()?;let source_home = external_agent_home(codex_home.path());std::fs::create_dir_all(&source_home)?;- std::fs::write(source_home.join("CLAUDE.md"), "Claude guidance")?;+ std::fs::write(+ source_home.join("CLAUDE.md"),+ "Claude guidance: [Claude](https://claude.ai/CLAUDE.md)",+ )?;let home_dir = codex_home.path().display().to_string();let mut mcp = TestAppServer::builder().with_codex_home(codex_home.path())@@ -447,7 +450,7 @@ async fn external_agent_config_import_source_remains_attribution_only() -> Resulassert_eq!(completed.item_type_results[0].failures, Vec::new());assert_eq!(std::fs::read_to_string(codex_home.path().join("AGENTS.md"))?,- "Codex guidance"+ "Codex guidance: [Codex](https://claude.ai/CLAUDE.md)");Ok(())diff --git a/codex-rs/external-agent-migration/Cargo.toml b/codex-rs/external-agent-migration/Cargo.tomlindex 5da3625c4d..3f49a1ae00 100644--- a/codex-rs/external-agent-migration/Cargo.toml+++ b/codex-rs/external-agent-migration/Cargo.toml@@ -27,6 +27,7 @@ codex-protocol = { workspace = true }codex-rollout = { workspace = true }codex-thread-store = { workspace = true }codex-utils-output-truncation = { workspace = true }+linkify = { workspace = true }serde = { workspace = true, features = ["derive"] }serde_json = { workspace = true }serde_yaml = { workspace = true }diff --git a/codex-rs/external-agent-migration/src/rewrite.rs b/codex-rs/external-agent-migration/src/rewrite.rsindex e1558850dd..d1036aca43 100644--- a/codex-rs/external-agent-migration/src/rewrite.rs+++ b/codex-rs/external-agent-migration/src/rewrite.rs@@ -1,3 +1,5 @@+use linkify::LinkFinder;+/// Describes source-specific terms that should be rewritten in migrated artifacts.#[derive(Clone, Copy)]pub struct RewriteProfile {@@ -35,8 +37,35 @@ impl RewriteProfile {self.case_sensitive_term_variants}- /// Rewrites source-specific documentation names and product terms to their Codex forms.+ /// Rewrites source-specific terms outside recognized URLs and email addresses.pub fn rewrite(self, content: &str) -> String {+ let mut finder = LinkFinder::new();+ finder.url_must_have_scheme(/* url_must_have_scheme */ false);+ let mut rewritten = String::with_capacity(content.len());+ let mut start = 0;+ for link in finder.links(content) {+ // The .md suffix is also a TLD, but this filename must still migrate,+ // including references with a :line or :line:column suffix.+ if link.as_str().split(['?', '#']).next().is_some_and(|path| {+ let mut parts = path.splitn(3, ':');+ parts+ .next()+ .is_some_and(|name| name.eq_ignore_ascii_case(self.doc_file_name))+ && parts.all(|part| {+ !part.is_empty() && part.as_bytes().iter().all(u8::is_ascii_digit)+ })+ }) {+ continue;+ }+ rewritten.push_str(&self.rewrite_terms(&content[start..link.start()]));+ rewritten.push_str(link.as_str());+ start = link.end();+ }+ rewritten.push_str(&self.rewrite_terms(&content[start..]));+ rewritten+ }++ fn rewrite_terms(self, content: &str) -> String {let mut rewritten =replace_case_insensitive_with_boundaries(content, self.doc_file_name, "AGENTS.md");for from in self.term_variants {diff --git a/codex-rs/external-agent-migration/src/rewrite_tests.rs b/codex-rs/external-agent-migration/src/rewrite_tests.rsindex 8b103cb3e1..c773c70c32 100644--- a/codex-rs/external-agent-migration/src/rewrite_tests.rs+++ b/codex-rs/external-agent-migration/src/rewrite_tests.rs@@ -7,7 +7,67 @@ const PROFILE: RewriteProfile = RewriteProfile::new("SOURCE.md", &["source agent#[test]fn rewrites_terms_only_at_word_boundaries() {assert_eq!(- PROFILE.rewrite("SOURCE.md Source source agent source_agent"),- "AGENTS.md Codex Codex source_agent"+ PROFILE.rewrite("SOURCE.md Source source agent source_agent Source: café ‘Source’"),+ "AGENTS.md Codex Codex source_agent Codex: café ‘Codex’"+ );+}++#[test]+fn preserves_urls_and_emails_while_rewriting_surrounding_text() {+ assert_eq!(+ PROFILE.rewrite(+ r#"Source: https://Source.example/SOURCE.md?product=Source#Source+[Source](HTTPS://Source.example/a_(Source)/source.md "Source")Source+<http://Source.example/SOURCE.md>,Source+“https://Source.example/é/SOURCE.md” Source+`file:///Source/SOURCE.md` Source://Source/SOURCE.md+[Source](mailto:[email protected]) [SOURCE.md](./SOURCE.md)+WWW.Source.example/SOURCE.md Source+https://Source.example/it's/SOURCE.md Source+[Source](/Source/reference) [SOURCE.md](./Source/SOURCE.md)+SOURCE.md?section=details+[Source](#Source)+<a id="Source"></a>++[![logo][SOURCE.md logo]](https://Source.example/Source)++[SOURCE.md logo]: ./logo.svg++| URL | Agent | Instructions |+| --- | --- | --- |+|https://example.com|Source|[docs](SOURCE.md)|++[Source][Source] [Source][] [Source]++[Source]: https://example.com/Source/SOURCE.md "Source"+"#,+ ),+ r#"Codex: https://Source.example/SOURCE.md?product=Source#Source+[Codex](HTTPS://Source.example/a_(Source)/source.md "Codex")Codex+<http://Source.example/SOURCE.md>,Codex+“https://Source.example/é/SOURCE.md” Codex+`file:///Source/SOURCE.md` Source://Source/SOURCE.md+[Codex](mailto:[email protected]) [AGENTS.md](./AGENTS.md)+WWW.Source.example/SOURCE.md Codex+https://Source.example/it's/SOURCE.md Codex+[Codex](/Codex/reference) [AGENTS.md](./Codex/AGENTS.md)+AGENTS.md?section=details+[Codex](#Codex)+<a id="Codex"></a>++[![logo][AGENTS.md logo]](https://Source.example/Source)++[AGENTS.md logo]: ./logo.svg++| URL | Agent | Instructions |+| --- | --- | --- |+|https://example.com|Codex|[docs](AGENTS.md)|++[Codex][Codex] [Codex][] [Codex]++[Codex]: https://example.com/Source/SOURCE.md "Codex"+"#);}diff --git a/codex-rs/external-agent-migration/src/service_tests/general/config_import.rs b/codex-rs/external-agent-migration/src/service_tests/general/config_import.rsindex 02e3cba393..cc66f1d648 100644--- a/codex-rs/external-agent-migration/src/service_tests/general/config_import.rs+++ b/codex-rs/external-agent-migration/src/service_tests/general/config_import.rs@@ -139,7 +139,10 @@ async fn import_home_migrates_supported_config_fields_skills_and_agents_md() {.join("skill-a").join("SKILL.md"),format!(- "Use {SOURCE_EXTERNAL_AGENT_PRODUCT_NAME} and {SOURCE_EXTERNAL_AGENT_UPPER_NAME} utilities."+ "Use {SOURCE_EXTERNAL_AGENT_PRODUCT_NAME} and {SOURCE_EXTERNAL_AGENT_UPPER_NAME} utilities.\n\+ Open [{SOURCE_EXTERNAL_AGENT_DISPLAY_NAME} settings](https://{SOURCE_EXTERNAL_AGENT_NAME}.ai/settings)\n\+ or <HTTPS://{SOURCE_EXTERNAL_AGENT_UPPER_NAME}.ai/{EXTERNAL_AGENT_CONFIG_MD}>.\n\+ Read [{EXTERNAL_AGENT_CONFIG_MD}](./{EXTERNAL_AGENT_CONFIG_MD})."),).expect("write skill");@@ -199,7 +202,12 @@ MY_TEAM = "codex"assert_eq!(fs::read_to_string(agents_skills.join("skill-a").join("SKILL.md")).expect("read copied skill"),- "Use Codex and Codex utilities."+ format!(+ "Use Codex and Codex utilities.\n\+ Open [Codex settings](https://{SOURCE_EXTERNAL_AGENT_NAME}.ai/settings)\n\+ or <HTTPS://{SOURCE_EXTERNAL_AGENT_UPPER_NAME}.ai/{EXTERNAL_AGENT_CONFIG_MD}>.\n\+ Read [AGENTS.md](./AGENTS.md)."+ ));}