Lightweight coding agent that runs in your terminal
Rust
Preserve UTF-8 byte semantics in network domain matching (#52277)
rafael-oai14ff9c3a43eb
diff --git a/codex-rs/network-proxy/README.md b/codex-rs/network-proxy/README.mdindex 88d677f16b..bfee26f55d 100644--- a/codex-rs/network-proxy/README.md+++ b/codex-rs/network-proxy/README.md@@ -241,9 +241,12 @@ what it can reasonably guarantee.- Allowlist-first policy: if `domains` has no `allow` entries, requests are blocked until an allowlist is configured.- Domain patterns: exact hosts are supported, `*.example.com` matches subdomains only, and `**.example.com` matches the apex plus subdomains; the global `*` wildcard is only accepted when explicitly enabled for allowlist compilation and is otherwise rejected.-- Within a domain pattern, `?` matches exactly one character, including a dot. For example,+- Within a domain pattern, `?` matches exactly one byte, including a dot. For example,`api?.example.com` matches `api1.example.com`, but not `api.example.com` or `api12.example.com`.It can be combined with `*`, `*.`, and `**.` in both allow and deny entries.+ Matching uses UTF-8 bytes with ASCII case insensitivity; it does not convert between Unicode+ domain names and IDNA/Punycode. Internationalized domains arriving as Punycode must be matched+ using their Punycode spelling.- Deny wins: `domains` entries marked `deny` always override the allowlist.- Local/private network protection: when `allow_local_binding = false`, the proxy blocks loopbackand common private/link-local ranges. Explicit allowlisting of local IP literals (or `localhost`)diff --git a/codex-rs/network-proxy/src/domain_matcher.rs b/codex-rs/network-proxy/src/domain_matcher.rsindex 3fcd3b90d8..bf71f8a276 100644--- a/codex-rs/network-proxy/src/domain_matcher.rs+++ b/codex-rs/network-proxy/src/domain_matcher.rs@@ -1,6 +1,7 @@-//! Network-domain wildcards over Unicode scalar values, independent of filesystem semantics.+//! Network-domain wildcards over UTF-8 bytes, independent of filesystem semantics.//! Construction validates and normalizes patterns and expands domain prefixes.-//! Callers normalize candidate hosts before matching.+//! Callers normalize candidate hosts before matching. No IDNA conversion is performed.+//! Byte matching preserves the historical globset wildcard semantics.use anyhow::Result;use anyhow::bail;@@ -18,7 +19,7 @@ pub(crate) enum GlobalWildcard {#[derive(Clone)]pub struct DomainPatternSet {- patterns: Arc<[Vec<char>]>,+ patterns: Arc<[Vec<u8>]>,}impl DomainPatternSet {@@ -65,14 +66,14 @@ impl DomainPatternSet {// - "example.com": match the exact host// - "*.example.com": match any subdomain (not the apex)// - "**.example.com": match the apex and any subdomain- // - "api?.example.com": match exactly one Unicode scalar value after "api"+ // - "api?.example.com": match exactly one byte after "api"// - "*": match every host when explicitly enabled for allowlist compilationfor candidate in expand_domain_pattern(&pattern) {if !seen.insert(candidate.clone()) {continue;}ensure!(!candidate.is_empty(), "domain pattern is empty");- compiled.push(candidate.chars().collect());+ compiled.push(candidate.into_bytes());}}Ok(Self {@@ -81,8 +82,8 @@ impl DomainPatternSet {}pub fn is_match(&self, host: impl AsRef<str>) -> bool {- let host: Vec<char> = host.as_ref().chars().collect();- self.patterns.iter().any(|pattern| matches(pattern, &host))+ let host = host.as_ref().as_bytes();+ self.patterns.iter().any(|pattern| matches(pattern, host))}}@@ -186,16 +187,16 @@ fn expand_domain_pattern(pattern: &str) -> Vec<String> {// Only the latest star needs retrying: it can absorb any extension that an earlier star could.// Its retry position advances monotonically, bounding work by O(pattern length * host length).-fn matches(pattern: &[char], host: &[char]) -> bool {+fn matches(pattern: &[u8], host: &[u8]) -> bool {let (mut p, mut h) = (0, 0);let mut star = None;while h < host.len() {match pattern.get(p) {- Some('*') => {+ Some(b'*') => {star = Some((p + 1, h));p += 1;}- Some(c) if *c == '?' || c.eq_ignore_ascii_case(&host[h]) => {+ Some(c) if *c == b'?' || c.eq_ignore_ascii_case(&host[h]) => {p += 1;h += 1;}@@ -209,7 +210,7 @@ fn matches(pattern: &[char], host: &[char]) -> bool {},}}- pattern[p..].iter().all(|c| *c == '*')+ pattern[p..].iter().all(|c| *c == b'*')}#[cfg(test)]diff --git a/codex-rs/network-proxy/src/domain_matcher_tests.rs b/codex-rs/network-proxy/src/domain_matcher_tests.rsindex c38562b07d..83c0eec3bf 100644--- a/codex-rs/network-proxy/src/domain_matcher_tests.rs+++ b/codex-rs/network-proxy/src/domain_matcher_tests.rs@@ -5,19 +5,36 @@ use crate::policy::normalize_host;use pretty_assertions::assert_eq;#[test]-fn unicode_wildcards_count_scalars_not_bytes_or_graphemes() {+fn unicode_wildcards_preserve_utf8_byte_matching() {for (pattern, host, expected) in [- ("api?.com", "apié.com", true),- ("api?.com", "api😀.com", true),- ("api??.com", "apié.com", false),- ("api?.com", "apie\u{301}.com", false),- ("api??.com", "apie\u{301}.com", true),- ("*.例?.com", "a.例子.com", true),- ("*.例?.com", "例子.com", false),- ("**.例?.com", "例子.com", true),+ ("api?.com", "api1.com", true),+ ("api??.com", "apié.com", true),+ ("api????.com", "api😀.com", true),+ ("api???.com", "apie\u{301}.com", true),+ ("api?*.com", "apié.com", true),+ ("*.例???.com", "a.例子.com", true),+ ("*.例???.com", "例子.com", false),+ ("**.例???.com", "例子.com", true),+ ("büch?.de", "büch1.de", true),+ ("bücher.de", "bücher.de", true),+ ("bücher.de", "xn--bcher-kva.de", false),+ ("xn--*.de", "xn--bcher-kva.de", true),("É.com", "é.com", false),("É.COM", "É.com", true),] {+ let mut legacy = globset::GlobSetBuilder::new();+ for expanded in expand_domain_pattern(&normalize_pattern(pattern)) {+ legacy.add(+ globset::GlobBuilder::new(&expanded)+ .case_insensitive(true)+ .build()+ .unwrap(),+ );+ }+ assert_eq!(+ legacy.build().unwrap().is_match(normalize_host(host)),+ expected+ );for compile in [compile_allowlist, compile_denylist] {assert_eq!(compile(&[pattern.to_string()])@@ -105,7 +122,7 @@ fn bracketed_patterns_preserve_port_and_trailing_dot_normalization() {#[test]fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {- fn words(alphabet: &[char], max_len: usize) -> Vec<Vec<char>> {+ fn words(alphabet: &[u8], max_len: usize) -> Vec<Vec<u8>> {let mut all = vec![vec![]];let mut level = vec![vec![]];for _ in 0..max_len {@@ -123,23 +140,23 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {}all}- fn reference(pattern: &[char], host: &[char]) -> bool {+ fn reference(pattern: &[u8], host: &[u8]) -> bool {let mut rows = vec![vec![false; host.len() + 1]; pattern.len() + 1];rows[0][0] = true;for (p, c) in pattern.iter().enumerate() {- rows[p + 1][0] = *c == '*' && rows[p][0];+ rows[p + 1][0] = *c == b'*' && rows[p][0];for (h, candidate) in host.iter().enumerate() {- rows[p + 1][h + 1] = if *c == '*' {+ rows[p + 1][h + 1] = if *c == b'*' {rows[p][h + 1] || rows[p + 1][h]} else {- rows[p][h] && (*c == '?' || c.eq_ignore_ascii_case(candidate))+ rows[p][h] && (*c == b'?' || c.eq_ignore_ascii_case(candidate))};}}rows[pattern.len()][host.len()]}- for pattern in words(&['a', 'é', '?', '*'], /*max_len*/ 4) {- for host in words(&['a', 'é'], /*max_len*/ 5) {+ for pattern in words(&[b'a', 0xc3, 0xa9, b'?', b'*'], /*max_len*/ 4) {+ for host in words(&[b'a', 0xc3, 0xa9], /*max_len*/ 5) {assert_eq!(matches(&pattern, &host),reference(&pattern, &host),@@ -151,17 +168,13 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {#[test]fn adversarial_retries_and_empty_boundaries() {- let pattern: Vec<char> = format!("*{}b", "a".repeat(1024)).chars().collect();- let host = vec!['a'; 4096];+ let pattern = format!("*{}b", "a".repeat(1024)).into_bytes();+ let host = vec![b'a'; 4096];assert!(!matches(&pattern, &host));- let pattern: Vec<char> = format!("{}b", "*a".repeat(1024)).chars().collect();+ let pattern = format!("{}b", "*a".repeat(1024)).into_bytes();assert!(!matches(&pattern, &host));assert_eq!(- (- matches(&[], &[]),- matches(&['*'], &[]),- matches(&['?'], &[])- ),+ (matches(&[], &[]), matches(b"*", &[]), matches(b"?", &[])),(true, true, false));assert!(!compile_allowlist(&[]).unwrap().is_match("host"));