Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(ragfs): make s3 key normalization chars configurable (#1767)
Jiahui Zhouc0ecbe3096a0
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rsindex 540017e20..96049005e 100644--- a/crates/ragfs/src/plugins/s3fs/client.rs+++ b/crates/ragfs/src/plugins/s3fs/client.rs@@ -13,35 +13,39 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};const ENCODED_SEGMENT_PREFIX: char = '!';const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF";-fn encode_path(path: &str, normalize_encoding: bool) -> String {- if !normalize_encoding {+fn encode_path(path: &str, normalize_encoding_chars: &str) -> String {+ if normalize_encoding_chars.is_empty() {return path.to_string();}- if path.bytes().all(|byte| byte == b'/' || is_safe_segment_byte(byte)) {+ let target_chars: Vec<char> = normalize_encoding_chars.chars().collect();+ if !path+ .chars()+ .any(|ch| ch != '/' && ch.is_ascii() && target_chars.contains(&ch))+ {return path.to_string();}let extra = path- .bytes()- .filter(|&byte| byte != b'/' && !is_safe_segment_byte(byte))+ .chars()+ .filter(|ch| *ch != '/' && ch.is_ascii() && target_chars.contains(ch)).count()* 2;let mut encoded = String::with_capacity(path.len() + extra);- for &byte in path.as_bytes() {- if byte == b'/' || is_safe_segment_byte(byte) {- encoded.push(byte as char);+ for ch in path.chars() {+ if ch == '/' || !ch.is_ascii() || !target_chars.contains(&ch) {+ encoded.push(ch);} else {- push_encoded_byte(&mut encoded, byte);+ push_encoded_byte(&mut encoded, ch as u8);}}encoded}-fn decode_path(path: &str, normalize_encoding: bool) -> String {- if !normalize_encoding {+fn decode_path(path: &str, normalize_encoding_chars: &str) -> String {+ if normalize_encoding_chars.is_empty() {return path.to_string();}@@ -75,11 +79,6 @@ fn decode_segment(segment: &str) -> String {String::from_utf8(decoded).unwrap_or_else(|_| segment.to_string())}-fn is_safe_segment_byte(byte: u8) -> bool {- byte.is_ascii_alphanumeric()- || matches!(byte, b'!' | b'-' | b'_' | b'.' | b'*' | b'\'' | b'(' | b')')-}-fn push_encoded_byte(output: &mut String, byte: u8) {output.push(ENCODED_SEGMENT_PREFIX);output.push(HEX_UPPER[(byte >> 4) as usize] as char);@@ -163,13 +162,13 @@ pub struct S3Client {client: Client,bucket: String,prefix: String,- normalize_encoding: bool,+ normalize_encoding_chars: String,marker_mode: DirectoryMarkerMode,disable_batch_delete: bool,}impl S3Client {- fn strip_prefix_with_codec(prefix: &str, normalize_encoding: bool, key: &str) -> String {+ fn strip_prefix_with_codec(prefix: &str, normalize_encoding_chars: &str, key: &str) -> String {let stripped = if prefix.is_empty() {key} else {@@ -177,7 +176,7 @@ impl S3Client {key.strip_prefix(&prefix).unwrap_or(key)};- decode_path(stripped, normalize_encoding)+ decode_path(stripped, normalize_encoding_chars)}/// Create a new S3 client from configuration@@ -233,10 +232,11 @@ impl S3Client {.unwrap_or("").to_string();- let normalize_encoding = config- .get("normalize_encoding")- .and_then(|v| v.as_bool())- .unwrap_or(false);+ let normalize_encoding_chars = config+ .get("normalize_encoding_chars")+ .and_then(|v| v.as_string())+ .unwrap_or("?#%+@")+ .to_string();let marker_mode = config.get("directory_marker_mode")@@ -273,7 +273,7 @@ impl S3Client {client,bucket,prefix,- normalize_encoding,+ normalize_encoding_chars,marker_mode,disable_batch_delete,})@@ -281,7 +281,7 @@ impl S3Client {/// Build the full S3 key from a filesystem pathpub fn build_key(&self, path: &str) -> String {- let clean = encode_path(path.trim_start_matches('/'), self.normalize_encoding);+ let clean = encode_path(path.trim_start_matches('/'), &self.normalize_encoding_chars);if self.prefix.is_empty() {clean} else {@@ -296,7 +296,7 @@ impl S3Client {/// Strip the prefix from an S3 key to get the filesystem pathpub fn strip_prefix(&self, key: &str) -> String {- Self::strip_prefix_with_codec(&self.prefix, self.normalize_encoding, key)+ Self::strip_prefix_with_codec(&self.prefix, &self.normalize_encoding_chars, key)}/// Get an object's contents@@ -663,7 +663,7 @@ impl S3Client {mod tests {use super::*;- fn test_client(prefix: &str, normalize_encoding: bool) -> S3Client {+ fn test_client(prefix: &str, normalize_encoding_chars: &str) -> S3Client {S3Client {client: Client::from_conf(aws_sdk_s3::Config::builder()@@ -673,7 +673,7 @@ mod tests {),bucket: "test-bucket".to_string(),prefix: prefix.to_string(),- normalize_encoding,+ normalize_encoding_chars: normalize_encoding_chars.to_string(),marker_mode: DirectoryMarkerMode::Empty,disable_batch_delete: false,}@@ -682,57 +682,65 @@ mod tests {#[test]fn test_key_codec_keeps_safe_segments() {let path = "dir/ok-name_1.2!*'()";- assert_eq!(encode_path(path, true), path);- assert_eq!(decode_path(path, true), path);+ assert_eq!(encode_path(path, "?#%+@"), path);+ assert_eq!(decode_path(path, "?#%+@"), path);}#[test]- fn test_key_codec_encodes_reserved_and_unsafe_segments() {- let encoded = encode_path("a b/c?d/#frag/%raw", true);+ fn test_key_codec_encodes_only_target_characters() {+ let encoded = encode_path("a b/c?d/#frag/%raw", "?#%+@");assert_eq!(encoded, "a!20b/c!3Fd/!23frag/!25raw");- assert_eq!(decode_path(&encoded, true), "a b/c?d/#frag/%raw");+ assert_eq!(decode_path(&encoded, "?#%+@"), "a b/c?d/#frag/%raw");}#[test]fn test_key_codec_encodes_segments_with_reserved_prefix() {- let encoded = encode_path("!literal/normal", true);+ let encoded = encode_path("!literal/normal", "?#%+@");assert_eq!(encoded, "!literal/normal");- assert_eq!(decode_path(&encoded, true), "!literal/normal");+ assert_eq!(decode_path(&encoded, "?#%+@"), "!literal/normal");}#[test]fn test_key_codec_disabled_is_passthrough() {let path = "a b/c?d";- assert_eq!(encode_path(path, false), path);- assert_eq!(decode_path(path, false), path);+ assert_eq!(encode_path(path, ""), path);+ assert_eq!(decode_path(path, ""), path);+ }++ #[test]+ fn test_key_codec_preserves_non_target_characters_including_unicode() {+ let path = "dir//safe-_.*/@scope+pkg/客户看板 file/";+ let encoded = encode_path(path, "?#%+@");++ assert_eq!(encoded, "dir//safe-_.*/!40scope!2Bpkg/客户看板!20file/");+ assert_eq!(decode_path(&encoded, "?#%+@"), path);}#[test]- fn test_key_codec_preserves_path_structure_with_mixed_segments() {- let path = "dir//safe-_.*/@scope+pkg/file name/";- let encoded = encode_path(path, true);+ fn test_key_codec_leaves_other_characters_unchanged() {+ let path = "目录/@scope&name=1";- assert_eq!(encoded, "dir//safe-_.*/!40scope!2Bpkg/file!20name/");- assert_eq!(decode_path(&encoded, true), path);+ assert_eq!(encode_path(path, "?#%+@"), "目录/!40scope&name=1");+ assert_eq!(decode_path("目录/!40scope&name=1", "?#%+@"), path);}#[test]fn test_build_key_applies_normalized_encoding_per_segment() {- let key = test_client("ns", true).build_key("/dir/a b/c?d.txt");+ let key = test_client("ns", "?#%+@").build_key("/dir/a b/c?d.txt");assert_eq!(key, "ns/dir/a!20b/c!3Fd.txt");}#[test]fn test_strip_prefix_decodes_normalized_segments() {- let client = test_client("ns", true);+ let client = test_client("ns", "?#%+@");let key = client.build_key("/dir/a b/c?d.txt");assert_eq!(client.strip_prefix(&key), "dir/a b/c?d.txt".to_string());}#[test]fn test_build_key_preserves_segments_when_normalization_disabled() {- assert_eq!(test_client("", false).build_key("/a b"), "a b");+ assert_eq!(test_client("", "").build_key("/a b"), "a b");}}diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rsindex 320e2dd36..b1aef5bf1 100644--- a/crates/ragfs/src/plugins/s3fs/mod.rs+++ b/crates/ragfs/src/plugins/s3fs/mod.rs@@ -527,10 +527,10 @@ impl S3FSPlugin {"Key prefix for namespace isolation (e.g. 'agfs/')",),ConfigParameter::optional(- "normalize_encoding",- "bool",- "false",- "Normalize unsafe path segments by base32-encoding URL-unsafe characters in S3 object keys",+ "normalize_encoding_chars",+ "string",+ "?#%+@",+ "Characters to escape in S3 object keys as !HH hexadecimal bytes; empty string disables normalization",),ConfigParameter::optional("directory_marker_mode",@@ -606,7 +606,7 @@ A file system backed by Amazon S3 or S3-compatible object storage.- Dual-layer caching (directory listings + stat metadata)- Range-based reads for partial file access- Configurable directory marker modes-- Optional URL-safe key normalization for unsafe path segments+- Optional configurable key normalization for selected characters## Configuration@@ -647,7 +647,7 @@ plugins:endpoint: https://tos-cn-beijing.volces.comuse_path_style: falsedirectory_marker_mode: nonempty- normalize_encoding: true+ normalize_encoding_chars: "?#%+@"```### Alibaba Cloud OSS@@ -671,10 +671,9 @@ plugins:## Key Normalization-- `normalize_encoding: false`: keep original path segments in object keys-- `normalize_encoding: true` (default): escape only URL-reserved or URL-unsafe bytes as `!HH`-- Allowed special characters without encoding: `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, `)`-- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole path segment+- `normalize_encoding_chars: "?#%+@"` (default): escape only `?`, `#`, `%`, `+`, and `@` as `!HH`+- `normalize_encoding_chars: ""`: keep original path segments in object keys+- Characters not listed in `normalize_encoding_chars`, including Chinese and other Unicode characters, remain unchanged## Notes@@ -709,10 +708,10 @@ plugins:}}- if let Some(value) = config.params.get("normalize_encoding") {- if value.as_bool().is_none() {+ if let Some(value) = config.params.get("normalize_encoding_chars") {+ if value.as_string().is_none() {return Err(Error::config(- "invalid normalize_encoding: expected bool",+ "invalid normalize_encoding_chars: expected string",));}}@@ -817,7 +816,7 @@ mod tests {}#[tokio::test]- async fn test_plugin_validate_normalize_encoding() {+ async fn test_plugin_validate_normalize_encoding_chars() {let plugin = S3FSPlugin::new();let mut params = std::collections::HashMap::new();@@ -826,8 +825,8 @@ mod tests {crate::core::ConfigValue::String("test".to_string()),);params.insert(- "normalize_encoding".to_string(),- crate::core::ConfigValue::String("true".to_string()),+ "normalize_encoding_chars".to_string(),+ crate::core::ConfigValue::Bool(true),);let config = PluginConfig {name: "s3fs".to_string(),@@ -842,8 +841,8 @@ mod tests {crate::core::ConfigValue::String("test".to_string()),);params.insert(- "normalize_encoding".to_string(),- crate::core::ConfigValue::Bool(true),+ "normalize_encoding_chars".to_string(),+ crate::core::ConfigValue::String("?#%+@".to_string()),);let config = PluginConfig {name: "s3fs".to_string(),diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 82783b3dd..cf00d482a 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -778,7 +778,7 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro| `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections. Also controls the scheme auto-prefixed onto bare-hostname `endpoint` values | true || `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true || `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` |-| `normalize_encoding` | bool | Escape only URL-reserved or URL-unsafe characters in S3 object keys as `!HH` hexadecimal bytes while preserving safe characters | `true` |+| `normalize_encoding_chars` | str | Characters to escape in S3 object keys as `!HH` hexadecimal bytes; empty string disables normalization | `"?#%+@"` |`directory_marker_mode` controls how RAGFS materializes directory objects in S3:@@ -792,13 +792,12 @@ Typical choices:- For TOS or other VirtualHostStyle backends that reject zero-byte directory markers, use `nonempty`.- If you want pure prefix-style behavior and do not need persisted empty directories, use `none`.-`normalize_encoding` controls whether RAGFS rewrites unsafe path segments before issuing S3 requests:+`normalize_encoding_chars` controls which characters RAGFS rewrites before issuing S3 requests:-- `true` is the default. RAGFS preserves safe path characters and rewrites only unsafe bytes.-- Unsafe bytes are encoded as `!HH`, where `HH` is the uppercase hexadecimal value of the byte.-- The characters `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, and `)` remain unescaped.-- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole segment.-- Set `false` to keep original path segments in object keys.+- The default value is `"?#%+@"`, so only `?`, `#`, `%`, `+`, and `@` are escaped.+- Escaped bytes are encoded as `!HH`, where `HH` is the uppercase hexadecimal value of the byte.+- Characters not listed in `normalize_encoding_chars`, including Chinese and other Unicode characters, remain unchanged.+- Set `normalize_encoding_chars` to `""` to keep original path segments in object keys.<details><summary><b>PathStyle S3</b></summary>@@ -815,7 +814,7 @@ Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS."region": "us-east-1","access_key": "your-ak","secret_key": "your-sk",- "normalize_encoding": true+ "normalize_encoding_chars": "?#%+@"}}}@@ -841,7 +840,7 @@ Supports S3 storage in VirtualHostStyle mode, such as TOS."secret_key": "your-sk","use_path_style": false,"directory_marker_mode": "nonempty",- "normalize_encoding": true+ "normalize_encoding_chars": "?#%+@"}}}diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex da08f9bf5..2f4aa8cbf 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -751,7 +751,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系| `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS)。也用于决定 `endpoint` 仅填主机名时自动补的协议前缀 | true || `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true || `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` |-| `normalize_encoding` | bool | 仅把 S3 object key 中 URL 保留字符或不安全字符转义为 `!HH` 十六进制字节,同时保留安全字符 | `true` |+| `normalize_encoding_chars` | str | 需要在 S3 object key 中转义为 `!HH` 十六进制字节的字符集合;空字符串表示关闭编码 | `"?#%+@"` |`directory_marker_mode` 用来控制 RAGFS 在 S3 中如何落目录对象:@@ -765,13 +765,12 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系- 对 TOS 或其他拒绝 0 字节目录 marker 的 VirtualHostStyle 后端,使用 `nonempty`。- 如果你想完全使用 prefix 风格行为,并且不需要持久化空目录,可以使用 `none`。-`normalize_encoding` 用来控制 RAGFS 在发起 S3 请求前是否重写不安全路径段:+`normalize_encoding_chars` 用来控制 RAGFS 在发起 S3 请求前需要重写哪些字符:-- `true` 是默认值。RAGFS 会保留安全路径字符,只重写不安全字节。-- 不安全字节会被编码成 `!HH`,其中 `HH` 是该字节的大写十六进制值。-- `/`、`!`、`-`、`_`、`.`、`*`、`'`、`(`、`)` 这些字符会保留,不触发编码。-- `?`、`&`、`#`、空格、`%`、`@`、`+` 等字符会在原位被转义,不会导致整个路径段一起重写。-- 设为 `false` 时,会在 object key 中保留原始路径段。+- 默认值是 `"?#%+@"`,所以只会转义 `?`、`#`、`%`、`+`、`@`。+- 被转义的字节会编码成 `!HH`,其中 `HH` 是该字节的大写十六进制值。+- 没有列在 `normalize_encoding_chars` 里的字符,包括中文和其他 Unicode 字符,都会保持原样。+- 设为 `""` 时,会在 object key 中保留原始路径段。<details><summary><b>PathStyle S3</b></summary>@@ -788,7 +787,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系"region": "us-east-1","access_key": "your-ak","secret_key": "your-sk",- "normalize_encoding": true+ "normalize_encoding_chars": "?#%+@"}}}@@ -814,7 +813,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系"secret_key": "your-sk","use_path_style": false,"directory_marker_mode": "nonempty",- "normalize_encoding": true+ "normalize_encoding_chars": "?#%+@"}}}diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.pyindex 1520498a6..5788abf38 100644--- a/openviking/utils/agfs_utils.py+++ b/openviking/utils/agfs_utils.py@@ -66,7 +66,7 @@ def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]if hasattr(s3_config.directory_marker_mode, "value")else s3_config.directory_marker_mode,"disable_batch_delete": s3_config.disable_batch_delete,- "normalize_encoding": s3_config.normalize_encoding,+ "normalize_encoding_chars": s3_config.normalize_encoding_chars,}config["s3fs"] = {diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex f50dac14f..c6e026e5e 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -71,11 +71,10 @@ class S3Config(BaseModel):"for DeleteObjects but AWS SDK v2 does not send it by default. Defaults to False.",)- normalize_encoding: bool = Field(- default=True,- description="Normalize URL-unsafe path segments before writing S3 object keys. "- "When enabled, reserved or unsafe URL bytes such as ?, &, #, spaces, %, @, or + are "- "escaped in place as !HH hexadecimal bytes, while / ! - _ . * ' ( ) remain unescaped.",+ normalize_encoding_chars: str = Field(+ default="?#%+@",+ description="Characters to escape in S3 object keys as !HH hexadecimal bytes. "+ "Set to an empty string to disable key normalization. Defaults to ?#%+@.",)model_config = {"extra": "forbid"}diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex 0be0b9e8a..1acf2432a 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -28,7 +28,7 @@ def test_agfs_validation():print(f" Fail: {e}")-def test_agfs_s3_normalize_encoding_defaults_to_false():+def test_agfs_s3_normalize_encoding_chars_defaults_to_target_set():config = AGFSConfig(backend="s3",s3=S3Config(@@ -40,10 +40,10 @@ def test_agfs_s3_normalize_encoding_defaults_to_false():),)- assert config.s3.normalize_encoding is False+ assert config.s3.normalize_encoding_chars == "?#%+@"-def test_agfs_s3_normalize_encoding_is_forwarded_to_ragfs_plugin_config():+def test_agfs_s3_normalize_encoding_chars_is_forwarded_to_ragfs_plugin_config():config = AGFSConfig(path="/tmp/ov-test",backend="s3",@@ -53,13 +53,13 @@ def test_agfs_s3_normalize_encoding_is_forwarded_to_ragfs_plugin_config():access_key="fake-access-key-for-testing",secret_key="fake-secret-key-for-testing-12345",endpoint="https://s3.amazonaws.com",- normalize_encoding=True,+ normalize_encoding_chars="?#",),)plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))- assert plugins["s3fs"]["config"]["normalize_encoding"] is True+ assert plugins["s3fs"]["config"]["normalize_encoding_chars"] == "?#"# Test 2: invalid backendprint("\n2. Test invalid backend...")