Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Preserve UTF-8 byte semantics in network domain matching (#52277)

rafael-oai14ff9c3a43eb
diff --git a/codex-rs/network-proxy/README.md b/codex-rs/network-proxy/README.md
index 88d677f16b..bfee26f55d 100644
--- a/codex-rs/network-proxy/README.md
+++ b/codex-rs/network-proxy/README.md
@@ -241,9 +241,12 @@ what it can reasonably guarantee.
- Allowlist-first policy: if `domains` has no `allow` entries, requests are blocked until an allowlist is configured.
- Domain patterns: exact hosts are supported, `*.example.com` matches subdomains only, and `**.example.com` matches the apex plus subdomains; the global `*` wildcard is only accepted when explicitly enabled for allowlist compilation and is otherwise rejected.
-- Within a domain pattern, `?` matches exactly one character, including a dot. For example,
+- Within a domain pattern, `?` matches exactly one byte, including a dot. For example,
`api?.example.com` matches `api1.example.com`, but not `api.example.com` or `api12.example.com`.
It can be combined with `*`, `*.`, and `**.` in both allow and deny entries.
+ Matching uses UTF-8 bytes with ASCII case insensitivity; it does not convert between Unicode
+ domain names and IDNA/Punycode. Internationalized domains arriving as Punycode must be matched
+ using their Punycode spelling.
- Deny wins: `domains` entries marked `deny` always override the allowlist.
- Local/private network protection: when `allow_local_binding = false`, the proxy blocks loopback
and common private/link-local ranges. Explicit allowlisting of local IP literals (or `localhost`)
diff --git a/codex-rs/network-proxy/src/domain_matcher.rs b/codex-rs/network-proxy/src/domain_matcher.rs
index 3fcd3b90d8..bf71f8a276 100644
--- a/codex-rs/network-proxy/src/domain_matcher.rs
+++ b/codex-rs/network-proxy/src/domain_matcher.rs
@@ -1,6 +1,7 @@
-//! Network-domain wildcards over Unicode scalar values, independent of filesystem semantics.
+//! Network-domain wildcards over UTF-8 bytes, independent of filesystem semantics.
//! Construction validates and normalizes patterns and expands domain prefixes.
-//! Callers normalize candidate hosts before matching.
+//! Callers normalize candidate hosts before matching. No IDNA conversion is performed.
+//! Byte matching preserves the historical globset wildcard semantics.
use anyhow::Result;
use anyhow::bail;
@@ -18,7 +19,7 @@ pub(crate) enum GlobalWildcard {
#[derive(Clone)]
pub struct DomainPatternSet {
- patterns: Arc<[Vec<char>]>,
+ patterns: Arc<[Vec<u8>]>,
}
impl DomainPatternSet {
@@ -65,14 +66,14 @@ impl DomainPatternSet {
// - "example.com": match the exact host
// - "*.example.com": match any subdomain (not the apex)
// - "**.example.com": match the apex and any subdomain
- // - "api?.example.com": match exactly one Unicode scalar value after "api"
+ // - "api?.example.com": match exactly one byte after "api"
// - "*": match every host when explicitly enabled for allowlist compilation
for candidate in expand_domain_pattern(&pattern) {
if !seen.insert(candidate.clone()) {
continue;
}
ensure!(!candidate.is_empty(), "domain pattern is empty");
- compiled.push(candidate.chars().collect());
+ compiled.push(candidate.into_bytes());
}
}
Ok(Self {
@@ -81,8 +82,8 @@ impl DomainPatternSet {
}
pub fn is_match(&self, host: impl AsRef<str>) -> bool {
- let host: Vec<char> = host.as_ref().chars().collect();
- self.patterns.iter().any(|pattern| matches(pattern, &host))
+ let host = host.as_ref().as_bytes();
+ self.patterns.iter().any(|pattern| matches(pattern, host))
}
}
@@ -186,16 +187,16 @@ fn expand_domain_pattern(pattern: &str) -> Vec<String> {
// Only the latest star needs retrying: it can absorb any extension that an earlier star could.
// Its retry position advances monotonically, bounding work by O(pattern length * host length).
-fn matches(pattern: &[char], host: &[char]) -> bool {
+fn matches(pattern: &[u8], host: &[u8]) -> bool {
let (mut p, mut h) = (0, 0);
let mut star = None;
while h < host.len() {
match pattern.get(p) {
- Some('*') => {
+ Some(b'*') => {
star = Some((p + 1, h));
p += 1;
}
- Some(c) if *c == '?' || c.eq_ignore_ascii_case(&host[h]) => {
+ Some(c) if *c == b'?' || c.eq_ignore_ascii_case(&host[h]) => {
p += 1;
h += 1;
}
@@ -209,7 +210,7 @@ fn matches(pattern: &[char], host: &[char]) -> bool {
},
}
}
- pattern[p..].iter().all(|c| *c == '*')
+ pattern[p..].iter().all(|c| *c == b'*')
}
#[cfg(test)]
diff --git a/codex-rs/network-proxy/src/domain_matcher_tests.rs b/codex-rs/network-proxy/src/domain_matcher_tests.rs
index c38562b07d..83c0eec3bf 100644
--- a/codex-rs/network-proxy/src/domain_matcher_tests.rs
+++ b/codex-rs/network-proxy/src/domain_matcher_tests.rs
@@ -5,19 +5,36 @@ use crate::policy::normalize_host;
use pretty_assertions::assert_eq;
#[test]
-fn unicode_wildcards_count_scalars_not_bytes_or_graphemes() {
+fn unicode_wildcards_preserve_utf8_byte_matching() {
for (pattern, host, expected) in [
- ("api?.com", "apié.com", true),
- ("api?.com", "api😀.com", true),
- ("api??.com", "apié.com", false),
- ("api?.com", "apie\u{301}.com", false),
- ("api??.com", "apie\u{301}.com", true),
- ("*.例?.com", "a.例子.com", true),
- ("*.例?.com", "例子.com", false),
- ("**.例?.com", "例子.com", true),
+ ("api?.com", "api1.com", true),
+ ("api??.com", "apié.com", true),
+ ("api????.com", "api😀.com", true),
+ ("api???.com", "apie\u{301}.com", true),
+ ("api?*.com", "apié.com", true),
+ ("*.例???.com", "a.例子.com", true),
+ ("*.例???.com", "例子.com", false),
+ ("**.例???.com", "例子.com", true),
+ ("büch?.de", "büch1.de", true),
+ ("bücher.de", "bücher.de", true),
+ ("bücher.de", "xn--bcher-kva.de", false),
+ ("xn--*.de", "xn--bcher-kva.de", true),
("É.com", "é.com", false),
("É.COM", "É.com", true),
] {
+ let mut legacy = globset::GlobSetBuilder::new();
+ for expanded in expand_domain_pattern(&normalize_pattern(pattern)) {
+ legacy.add(
+ globset::GlobBuilder::new(&expanded)
+ .case_insensitive(true)
+ .build()
+ .unwrap(),
+ );
+ }
+ assert_eq!(
+ legacy.build().unwrap().is_match(normalize_host(host)),
+ expected
+ );
for compile in [compile_allowlist, compile_denylist] {
assert_eq!(
compile(&[pattern.to_string()])
@@ -105,7 +122,7 @@ fn bracketed_patterns_preserve_port_and_trailing_dot_normalization() {
#[test]
fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {
- fn words(alphabet: &[char], max_len: usize) -> Vec<Vec<char>> {
+ fn words(alphabet: &[u8], max_len: usize) -> Vec<Vec<u8>> {
let mut all = vec![vec![]];
let mut level = vec![vec![]];
for _ in 0..max_len {
@@ -123,23 +140,23 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {
}
all
}
- fn reference(pattern: &[char], host: &[char]) -> bool {
+ fn reference(pattern: &[u8], host: &[u8]) -> bool {
let mut rows = vec![vec![false; host.len() + 1]; pattern.len() + 1];
rows[0][0] = true;
for (p, c) in pattern.iter().enumerate() {
- rows[p + 1][0] = *c == '*' && rows[p][0];
+ rows[p + 1][0] = *c == b'*' && rows[p][0];
for (h, candidate) in host.iter().enumerate() {
- rows[p + 1][h + 1] = if *c == '*' {
+ rows[p + 1][h + 1] = if *c == b'*' {
rows[p][h + 1] || rows[p + 1][h]
} else {
- rows[p][h] && (*c == '?' || c.eq_ignore_ascii_case(candidate))
+ rows[p][h] && (*c == b'?' || c.eq_ignore_ascii_case(candidate))
};
}
}
rows[pattern.len()][host.len()]
}
- for pattern in words(&['a', 'é', '?', '*'], /*max_len*/ 4) {
- for host in words(&['a', 'é'], /*max_len*/ 5) {
+ for pattern in words(&[b'a', 0xc3, 0xa9, b'?', b'*'], /*max_len*/ 4) {
+ for host in words(&[b'a', 0xc3, 0xa9], /*max_len*/ 5) {
assert_eq!(
matches(&pattern, &host),
reference(&pattern, &host),
@@ -151,17 +168,13 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() {
#[test]
fn adversarial_retries_and_empty_boundaries() {
- let pattern: Vec<char> = format!("*{}b", "a".repeat(1024)).chars().collect();
- let host = vec!['a'; 4096];
+ let pattern = format!("*{}b", "a".repeat(1024)).into_bytes();
+ let host = vec![b'a'; 4096];
assert!(!matches(&pattern, &host));
- let pattern: Vec<char> = format!("{}b", "*a".repeat(1024)).chars().collect();
+ let pattern = format!("{}b", "*a".repeat(1024)).into_bytes();
assert!(!matches(&pattern, &host));
assert_eq!(
- (
- matches(&[], &[]),
- matches(&['*'], &[]),
- matches(&['?'], &[])
- ),
+ (matches(&[], &[]), matches(b"*", &[]), matches(b"?", &[])),
(true, true, false)
);
assert!(!compile_allowlist(&[]).unwrap().is_match("host"));