Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(ragfs): make s3 key normalization chars configurable (#1767)

Jiahui Zhouc0ecbe3096a0
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rs
index 540017e20..96049005e 100644
--- a/crates/ragfs/src/plugins/s3fs/client.rs
+++ b/crates/ragfs/src/plugins/s3fs/client.rs
@@ -13,35 +13,39 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
const ENCODED_SEGMENT_PREFIX: char = '!';
const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF";
-fn encode_path(path: &str, normalize_encoding: bool) -> String {
- if !normalize_encoding {
+fn encode_path(path: &str, normalize_encoding_chars: &str) -> String {
+ if normalize_encoding_chars.is_empty() {
return path.to_string();
}
- if path.bytes().all(|byte| byte == b'/' || is_safe_segment_byte(byte)) {
+ let target_chars: Vec<char> = normalize_encoding_chars.chars().collect();
+ if !path
+ .chars()
+ .any(|ch| ch != '/' && ch.is_ascii() && target_chars.contains(&ch))
+ {
return path.to_string();
}
let extra = path
- .bytes()
- .filter(|&byte| byte != b'/' && !is_safe_segment_byte(byte))
+ .chars()
+ .filter(|ch| *ch != '/' && ch.is_ascii() && target_chars.contains(ch))
.count()
* 2;
let mut encoded = String::with_capacity(path.len() + extra);
- for &byte in path.as_bytes() {
- if byte == b'/' || is_safe_segment_byte(byte) {
- encoded.push(byte as char);
+ for ch in path.chars() {
+ if ch == '/' || !ch.is_ascii() || !target_chars.contains(&ch) {
+ encoded.push(ch);
} else {
- push_encoded_byte(&mut encoded, byte);
+ push_encoded_byte(&mut encoded, ch as u8);
}
}
encoded
}
-fn decode_path(path: &str, normalize_encoding: bool) -> String {
- if !normalize_encoding {
+fn decode_path(path: &str, normalize_encoding_chars: &str) -> String {
+ if normalize_encoding_chars.is_empty() {
return path.to_string();
}
@@ -75,11 +79,6 @@ fn decode_segment(segment: &str) -> String {
String::from_utf8(decoded).unwrap_or_else(|_| segment.to_string())
}
-fn is_safe_segment_byte(byte: u8) -> bool {
- byte.is_ascii_alphanumeric()
- || matches!(byte, b'!' | b'-' | b'_' | b'.' | b'*' | b'\'' | b'(' | b')')
-}
-
fn push_encoded_byte(output: &mut String, byte: u8) {
output.push(ENCODED_SEGMENT_PREFIX);
output.push(HEX_UPPER[(byte >> 4) as usize] as char);
@@ -163,13 +162,13 @@ pub struct S3Client {
client: Client,
bucket: String,
prefix: String,
- normalize_encoding: bool,
+ normalize_encoding_chars: String,
marker_mode: DirectoryMarkerMode,
disable_batch_delete: bool,
}
impl S3Client {
- fn strip_prefix_with_codec(prefix: &str, normalize_encoding: bool, key: &str) -> String {
+ fn strip_prefix_with_codec(prefix: &str, normalize_encoding_chars: &str, key: &str) -> String {
let stripped = if prefix.is_empty() {
key
} else {
@@ -177,7 +176,7 @@ impl S3Client {
key.strip_prefix(&prefix).unwrap_or(key)
};
- decode_path(stripped, normalize_encoding)
+ decode_path(stripped, normalize_encoding_chars)
}
/// Create a new S3 client from configuration
@@ -233,10 +232,11 @@ impl S3Client {
.unwrap_or("")
.to_string();
- let normalize_encoding = config
- .get("normalize_encoding")
- .and_then(|v| v.as_bool())
- .unwrap_or(false);
+ let normalize_encoding_chars = config
+ .get("normalize_encoding_chars")
+ .and_then(|v| v.as_string())
+ .unwrap_or("?#%+@")
+ .to_string();
let marker_mode = config
.get("directory_marker_mode")
@@ -273,7 +273,7 @@ impl S3Client {
client,
bucket,
prefix,
- normalize_encoding,
+ normalize_encoding_chars,
marker_mode,
disable_batch_delete,
})
@@ -281,7 +281,7 @@ impl S3Client {
/// Build the full S3 key from a filesystem path
pub fn build_key(&self, path: &str) -> String {
- let clean = encode_path(path.trim_start_matches('/'), self.normalize_encoding);
+ let clean = encode_path(path.trim_start_matches('/'), &self.normalize_encoding_chars);
if self.prefix.is_empty() {
clean
} else {
@@ -296,7 +296,7 @@ impl S3Client {
/// Strip the prefix from an S3 key to get the filesystem path
pub fn strip_prefix(&self, key: &str) -> String {
- Self::strip_prefix_with_codec(&self.prefix, self.normalize_encoding, key)
+ Self::strip_prefix_with_codec(&self.prefix, &self.normalize_encoding_chars, key)
}
/// Get an object's contents
@@ -663,7 +663,7 @@ impl S3Client {
mod tests {
use super::*;
- fn test_client(prefix: &str, normalize_encoding: bool) -> S3Client {
+ fn test_client(prefix: &str, normalize_encoding_chars: &str) -> S3Client {
S3Client {
client: Client::from_conf(
aws_sdk_s3::Config::builder()
@@ -673,7 +673,7 @@ mod tests {
),
bucket: "test-bucket".to_string(),
prefix: prefix.to_string(),
- normalize_encoding,
+ normalize_encoding_chars: normalize_encoding_chars.to_string(),
marker_mode: DirectoryMarkerMode::Empty,
disable_batch_delete: false,
}
@@ -682,57 +682,65 @@ mod tests {
#[test]
fn test_key_codec_keeps_safe_segments() {
let path = "dir/ok-name_1.2!*'()";
- assert_eq!(encode_path(path, true), path);
- assert_eq!(decode_path(path, true), path);
+ assert_eq!(encode_path(path, "?#%+@"), path);
+ assert_eq!(decode_path(path, "?#%+@"), path);
}
#[test]
- fn test_key_codec_encodes_reserved_and_unsafe_segments() {
- let encoded = encode_path("a b/c?d/#frag/%raw", true);
+ fn test_key_codec_encodes_only_target_characters() {
+ let encoded = encode_path("a b/c?d/#frag/%raw", "?#%+@");
assert_eq!(encoded, "a!20b/c!3Fd/!23frag/!25raw");
- assert_eq!(decode_path(&encoded, true), "a b/c?d/#frag/%raw");
+ assert_eq!(decode_path(&encoded, "?#%+@"), "a b/c?d/#frag/%raw");
}
#[test]
fn test_key_codec_encodes_segments_with_reserved_prefix() {
- let encoded = encode_path("!literal/normal", true);
+ let encoded = encode_path("!literal/normal", "?#%+@");
assert_eq!(encoded, "!literal/normal");
- assert_eq!(decode_path(&encoded, true), "!literal/normal");
+ assert_eq!(decode_path(&encoded, "?#%+@"), "!literal/normal");
}
#[test]
fn test_key_codec_disabled_is_passthrough() {
let path = "a b/c?d";
- assert_eq!(encode_path(path, false), path);
- assert_eq!(decode_path(path, false), path);
+ assert_eq!(encode_path(path, ""), path);
+ assert_eq!(decode_path(path, ""), path);
+ }
+
+ #[test]
+ fn test_key_codec_preserves_non_target_characters_including_unicode() {
+ let path = "dir//safe-_.*/@scope+pkg/客户看板 file/";
+ let encoded = encode_path(path, "?#%+@");
+
+ assert_eq!(encoded, "dir//safe-_.*/!40scope!2Bpkg/客户看板!20file/");
+ assert_eq!(decode_path(&encoded, "?#%+@"), path);
}
#[test]
- fn test_key_codec_preserves_path_structure_with_mixed_segments() {
- let path = "dir//safe-_.*/@scope+pkg/file name/";
- let encoded = encode_path(path, true);
+ fn test_key_codec_leaves_other_characters_unchanged() {
+ let path = "目录/@scope&name=1";
- assert_eq!(encoded, "dir//safe-_.*/!40scope!2Bpkg/file!20name/");
- assert_eq!(decode_path(&encoded, true), path);
+ assert_eq!(encode_path(path, "?#%+@"), "目录/!40scope&name=1");
+ assert_eq!(decode_path("目录/!40scope&name=1", "?#%+@"), path);
}
#[test]
fn test_build_key_applies_normalized_encoding_per_segment() {
- let key = test_client("ns", true).build_key("/dir/a b/c?d.txt");
+ let key = test_client("ns", "?#%+@").build_key("/dir/a b/c?d.txt");
assert_eq!(key, "ns/dir/a!20b/c!3Fd.txt");
}
#[test]
fn test_strip_prefix_decodes_normalized_segments() {
- let client = test_client("ns", true);
+ let client = test_client("ns", "?#%+@");
let key = client.build_key("/dir/a b/c?d.txt");
assert_eq!(client.strip_prefix(&key), "dir/a b/c?d.txt".to_string());
}
#[test]
fn test_build_key_preserves_segments_when_normalization_disabled() {
- assert_eq!(test_client("", false).build_key("/a b"), "a b");
+ assert_eq!(test_client("", "").build_key("/a b"), "a b");
}
}
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index 320e2dd36..b1aef5bf1 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -527,10 +527,10 @@ impl S3FSPlugin {
"Key prefix for namespace isolation (e.g. 'agfs/')",
),
ConfigParameter::optional(
- "normalize_encoding",
- "bool",
- "false",
- "Normalize unsafe path segments by base32-encoding URL-unsafe characters in S3 object keys",
+ "normalize_encoding_chars",
+ "string",
+ "?#%+@",
+ "Characters to escape in S3 object keys as !HH hexadecimal bytes; empty string disables normalization",
),
ConfigParameter::optional(
"directory_marker_mode",
@@ -606,7 +606,7 @@ A file system backed by Amazon S3 or S3-compatible object storage.
- Dual-layer caching (directory listings + stat metadata)
- Range-based reads for partial file access
- Configurable directory marker modes
-- Optional URL-safe key normalization for unsafe path segments
+- Optional configurable key normalization for selected characters
## Configuration
@@ -647,7 +647,7 @@ plugins:
endpoint: https://tos-cn-beijing.volces.com
use_path_style: false
directory_marker_mode: nonempty
- normalize_encoding: true
+ normalize_encoding_chars: "?#%+@"
```
### Alibaba Cloud OSS
@@ -671,10 +671,9 @@ plugins:
## Key Normalization
-- `normalize_encoding: false`: keep original path segments in object keys
-- `normalize_encoding: true` (default): escape only URL-reserved or URL-unsafe bytes as `!HH`
-- Allowed special characters without encoding: `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, `)`
-- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole path segment
+- `normalize_encoding_chars: "?#%+@"` (default): escape only `?`, `#`, `%`, `+`, and `@` as `!HH`
+- `normalize_encoding_chars: ""`: keep original path segments in object keys
+- Characters not listed in `normalize_encoding_chars`, including Chinese and other Unicode characters, remain unchanged
## Notes
@@ -709,10 +708,10 @@ plugins:
}
}
- if let Some(value) = config.params.get("normalize_encoding") {
- if value.as_bool().is_none() {
+ if let Some(value) = config.params.get("normalize_encoding_chars") {
+ if value.as_string().is_none() {
return Err(Error::config(
- "invalid normalize_encoding: expected bool",
+ "invalid normalize_encoding_chars: expected string",
));
}
}
@@ -817,7 +816,7 @@ mod tests {
}
#[tokio::test]
- async fn test_plugin_validate_normalize_encoding() {
+ async fn test_plugin_validate_normalize_encoding_chars() {
let plugin = S3FSPlugin::new();
let mut params = std::collections::HashMap::new();
@@ -826,8 +825,8 @@ mod tests {
crate::core::ConfigValue::String("test".to_string()),
);
params.insert(
- "normalize_encoding".to_string(),
- crate::core::ConfigValue::String("true".to_string()),
+ "normalize_encoding_chars".to_string(),
+ crate::core::ConfigValue::Bool(true),
);
let config = PluginConfig {
name: "s3fs".to_string(),
@@ -842,8 +841,8 @@ mod tests {
crate::core::ConfigValue::String("test".to_string()),
);
params.insert(
- "normalize_encoding".to_string(),
- crate::core::ConfigValue::Bool(true),
+ "normalize_encoding_chars".to_string(),
+ crate::core::ConfigValue::String("?#%+@".to_string()),
);
let config = PluginConfig {
name: "s3fs".to_string(),
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 82783b3dd..cf00d482a 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -778,7 +778,7 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro
| `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections. Also controls the scheme auto-prefixed onto bare-hostname `endpoint` values | true |
| `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true |
| `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` |
-| `normalize_encoding` | bool | Escape only URL-reserved or URL-unsafe characters in S3 object keys as `!HH` hexadecimal bytes while preserving safe characters | `true` |
+| `normalize_encoding_chars` | str | Characters to escape in S3 object keys as `!HH` hexadecimal bytes; empty string disables normalization | `"?#%+@"` |
`directory_marker_mode` controls how RAGFS materializes directory objects in S3:
@@ -792,13 +792,12 @@ Typical choices:
- For TOS or other VirtualHostStyle backends that reject zero-byte directory markers, use `nonempty`.
- If you want pure prefix-style behavior and do not need persisted empty directories, use `none`.
-`normalize_encoding` controls whether RAGFS rewrites unsafe path segments before issuing S3 requests:
+`normalize_encoding_chars` controls which characters RAGFS rewrites before issuing S3 requests:
-- `true` is the default. RAGFS preserves safe path characters and rewrites only unsafe bytes.
-- Unsafe bytes are encoded as `!HH`, where `HH` is the uppercase hexadecimal value of the byte.
-- The characters `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, and `)` remain unescaped.
-- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole segment.
-- Set `false` to keep original path segments in object keys.
+- The default value is `"?#%+@"`, so only `?`, `#`, `%`, `+`, and `@` are escaped.
+- Escaped bytes are encoded as `!HH`, where `HH` is the uppercase hexadecimal value of the byte.
+- Characters not listed in `normalize_encoding_chars`, including Chinese and other Unicode characters, remain unchanged.
+- Set `normalize_encoding_chars` to `""` to keep original path segments in object keys.
<details>
<summary><b>PathStyle S3</b></summary>
@@ -815,7 +814,7 @@ Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS.
"region": "us-east-1",
"access_key": "your-ak",
"secret_key": "your-sk",
- "normalize_encoding": true
+ "normalize_encoding_chars": "?#%+@"
}
}
}
@@ -841,7 +840,7 @@ Supports S3 storage in VirtualHostStyle mode, such as TOS.
"secret_key": "your-sk",
"use_path_style": false,
"directory_marker_mode": "nonempty",
- "normalize_encoding": true
+ "normalize_encoding_chars": "?#%+@"
}
}
}
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index da08f9bf5..2f4aa8cbf 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -751,7 +751,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
| `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS)。也用于决定 `endpoint` 仅填主机名时自动补的协议前缀 | true |
| `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true |
| `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` |
-| `normalize_encoding` | bool | 仅把 S3 object key 中 URL 保留字符或不安全字符转义为 `!HH` 十六进制字节,同时保留安全字符 | `true` |
+| `normalize_encoding_chars` | str | 需要在 S3 object key 中转义为 `!HH` 十六进制字节的字符集合;空字符串表示关闭编码 | `"?#%+@"` |
`directory_marker_mode` 用来控制 RAGFS 在 S3 中如何落目录对象:
@@ -765,13 +765,12 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
- 对 TOS 或其他拒绝 0 字节目录 marker 的 VirtualHostStyle 后端,使用 `nonempty`。
- 如果你想完全使用 prefix 风格行为,并且不需要持久化空目录,可以使用 `none`。
-`normalize_encoding` 用来控制 RAGFS 在发起 S3 请求前是否重写不安全路径段:
+`normalize_encoding_chars` 用来控制 RAGFS 在发起 S3 请求前需要重写哪些字符:
-- `true` 是默认值。RAGFS 会保留安全路径字符,只重写不安全字节。
-- 不安全字节会被编码成 `!HH`,其中 `HH` 是该字节的大写十六进制值。
-- `/`、`!`、`-`、`_`、`.`、`*`、`'`、`(`、`)` 这些字符会保留,不触发编码。
-- `?`、`&`、`#`、空格、`%`、`@`、`+` 等字符会在原位被转义,不会导致整个路径段一起重写。
-- 设为 `false` 时,会在 object key 中保留原始路径段。
+- 默认值是 `"?#%+@"`,所以只会转义 `?`、`#`、`%`、`+`、`@`。
+- 被转义的字节会编码成 `!HH`,其中 `HH` 是该字节的大写十六进制值。
+- 没有列在 `normalize_encoding_chars` 里的字符,包括中文和其他 Unicode 字符,都会保持原样。
+- 设为 `""` 时,会在 object key 中保留原始路径段。
<details>
<summary><b>PathStyle S3</b></summary>
@@ -788,7 +787,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
"region": "us-east-1",
"access_key": "your-ak",
"secret_key": "your-sk",
- "normalize_encoding": true
+ "normalize_encoding_chars": "?#%+@"
}
}
}
@@ -814,7 +813,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
"secret_key": "your-sk",
"use_path_style": false,
"directory_marker_mode": "nonempty",
- "normalize_encoding": true
+ "normalize_encoding_chars": "?#%+@"
}
}
}
diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py
index 1520498a6..5788abf38 100644
--- a/openviking/utils/agfs_utils.py
+++ b/openviking/utils/agfs_utils.py
@@ -66,7 +66,7 @@ def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]
if hasattr(s3_config.directory_marker_mode, "value")
else s3_config.directory_marker_mode,
"disable_batch_delete": s3_config.disable_batch_delete,
- "normalize_encoding": s3_config.normalize_encoding,
+ "normalize_encoding_chars": s3_config.normalize_encoding_chars,
}
config["s3fs"] = {
diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py
index f50dac14f..c6e026e5e 100644
--- a/openviking_cli/utils/config/agfs_config.py
+++ b/openviking_cli/utils/config/agfs_config.py
@@ -71,11 +71,10 @@ class S3Config(BaseModel):
"for DeleteObjects but AWS SDK v2 does not send it by default. Defaults to False.",
)
- normalize_encoding: bool = Field(
- default=True,
- description="Normalize URL-unsafe path segments before writing S3 object keys. "
- "When enabled, reserved or unsafe URL bytes such as ?, &, #, spaces, %, @, or + are "
- "escaped in place as !HH hexadecimal bytes, while / ! - _ . * ' ( ) remain unescaped.",
+ normalize_encoding_chars: str = Field(
+ default="?#%+@",
+ description="Characters to escape in S3 object keys as !HH hexadecimal bytes. "
+ "Set to an empty string to disable key normalization. Defaults to ?#%+@.",
)
model_config = {"extra": "forbid"}
diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py
index 0be0b9e8a..1acf2432a 100644
--- a/tests/misc/test_config_validation.py
+++ b/tests/misc/test_config_validation.py
@@ -28,7 +28,7 @@ def test_agfs_validation():
print(f" Fail: {e}")
-def test_agfs_s3_normalize_encoding_defaults_to_false():
+def test_agfs_s3_normalize_encoding_chars_defaults_to_target_set():
config = AGFSConfig(
backend="s3",
s3=S3Config(
@@ -40,10 +40,10 @@ def test_agfs_s3_normalize_encoding_defaults_to_false():
),
)
- assert config.s3.normalize_encoding is False
+ assert config.s3.normalize_encoding_chars == "?#%+@"
-def test_agfs_s3_normalize_encoding_is_forwarded_to_ragfs_plugin_config():
+def test_agfs_s3_normalize_encoding_chars_is_forwarded_to_ragfs_plugin_config():
config = AGFSConfig(
path="/tmp/ov-test",
backend="s3",
@@ -53,13 +53,13 @@ def test_agfs_s3_normalize_encoding_is_forwarded_to_ragfs_plugin_config():
access_key="fake-access-key-for-testing",
secret_key="fake-secret-key-for-testing-12345",
endpoint="https://s3.amazonaws.com",
- normalize_encoding=True,
+ normalize_encoding_chars="?#",
),
)
plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
- assert plugins["s3fs"]["config"]["normalize_encoding"] is True
+ assert plugins["s3fs"]["config"]["normalize_encoding_chars"] == "?#"
# Test 2: invalid backend
print("\n2. Test invalid backend...")