Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(ragfs): add s3 key normalization encoding (#1685)

Jiahui Zhouce090be8267f
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rs
index a89e67fb3..4136469b0 100644
--- a/crates/ragfs/src/plugins/s3fs/client.rs
+++ b/crates/ragfs/src/plugins/s3fs/client.rs
@@ -10,6 +10,91 @@ use aws_sdk_s3::Client;
use std::collections::HashMap;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
+const ENCODED_SEGMENT_PREFIX: char = '!';
+const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF";
+
+fn encode_path(path: &str, normalize_encoding: bool) -> String {
+ if !normalize_encoding {
+ return path.to_string();
+ }
+
+ if path.bytes().all(|byte| byte == b'/' || is_safe_segment_byte(byte)) {
+ return path.to_string();
+ }
+
+ let extra = path
+ .bytes()
+ .filter(|&byte| byte != b'/' && !is_safe_segment_byte(byte))
+ .count()
+ * 2;
+ let mut encoded = String::with_capacity(path.len() + extra);
+
+ for &byte in path.as_bytes() {
+ if byte == b'/' || is_safe_segment_byte(byte) {
+ encoded.push(byte as char);
+ } else {
+ push_encoded_byte(&mut encoded, byte);
+ }
+ }
+
+ encoded
+}
+
+fn decode_path(path: &str, normalize_encoding: bool) -> String {
+ if !normalize_encoding {
+ return path.to_string();
+ }
+
+ decode_segment(path)
+}
+
+fn decode_segment(segment: &str) -> String {
+ if segment.is_empty() {
+ return String::new();
+ }
+
+ let bytes = segment.as_bytes();
+ let mut decoded = Vec::with_capacity(bytes.len());
+ let mut idx = 0usize;
+
+ while idx < bytes.len() {
+ if bytes[idx] == ENCODED_SEGMENT_PREFIX as u8 && idx + 2 < bytes.len() {
+ let hi = decode_hex_nibble(bytes[idx + 1]);
+ let lo = decode_hex_nibble(bytes[idx + 2]);
+ if let (Some(hi), Some(lo)) = (hi, lo) {
+ decoded.push((hi << 4) | lo);
+ idx += 3;
+ continue;
+ }
+ }
+
+ decoded.push(bytes[idx]);
+ idx += 1;
+ }
+
+ String::from_utf8(decoded).unwrap_or_else(|_| segment.to_string())
+}
+
+fn is_safe_segment_byte(byte: u8) -> bool {
+ byte.is_ascii_alphanumeric()
+ || matches!(byte, b'!' | b'-' | b'_' | b'.' | b'*' | b'\'' | b'(' | b')')
+}
+
+fn push_encoded_byte(output: &mut String, byte: u8) {
+ output.push(ENCODED_SEGMENT_PREFIX);
+ output.push(HEX_UPPER[(byte >> 4) as usize] as char);
+ output.push(HEX_UPPER[(byte & 0x0f) as usize] as char);
+}
+
+fn decode_hex_nibble(byte: u8) -> Option<u8> {
+ match byte {
+ b'0'..=b'9' => Some(byte - b'0'),
+ b'A'..=b'F' => Some(byte - b'A' + 10),
+ b'a'..=b'f' => Some(byte - b'a' + 10),
+ _ => None,
+ }
+}
+
/// Directory marker mode
#[derive(Debug, Clone, PartialEq)]
pub enum DirectoryMarkerMode {
@@ -78,11 +163,23 @@ pub struct S3Client {
client: Client,
bucket: String,
prefix: String,
+ normalize_encoding: bool,
marker_mode: DirectoryMarkerMode,
disable_batch_delete: bool,
}
impl S3Client {
+ fn strip_prefix_with_codec(prefix: &str, normalize_encoding: bool, key: &str) -> String {
+ let stripped = if prefix.is_empty() {
+ key
+ } else {
+ let prefix = format!("{}/", prefix.trim_end_matches('/'));
+ key.strip_prefix(&prefix).unwrap_or(key)
+ };
+
+ decode_path(stripped, normalize_encoding)
+ }
+
/// Create a new S3 client from configuration
pub async fn new(config: &HashMap<String, ConfigValue>) -> Result<Self> {
let bucket = config
@@ -120,6 +217,11 @@ impl S3Client {
.unwrap_or("")
.to_string();
+ let normalize_encoding = config
+ .get("normalize_encoding")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
+
let marker_mode = config
.get("directory_marker_mode")
.and_then(|v| v.as_string())
@@ -155,6 +257,7 @@ impl S3Client {
client,
bucket,
prefix,
+ normalize_encoding,
marker_mode,
disable_batch_delete,
})
@@ -162,9 +265,9 @@ impl S3Client {
/// Build the full S3 key from a filesystem path
pub fn build_key(&self, path: &str) -> String {
- let clean = path.trim_start_matches('/');
+ let clean = encode_path(path.trim_start_matches('/'), self.normalize_encoding);
if self.prefix.is_empty() {
- clean.to_string()
+ clean
} else {
let prefix = self.prefix.trim_end_matches('/');
if clean.is_empty() {
@@ -176,13 +279,8 @@ impl S3Client {
}
/// Strip the prefix from an S3 key to get the filesystem path
- pub fn strip_prefix<'a>(&self, key: &'a str) -> &'a str {
- if self.prefix.is_empty() {
- key
- } else {
- let prefix = format!("{}/", self.prefix.trim_end_matches('/'));
- key.strip_prefix(&prefix).unwrap_or(key)
- }
+ pub fn strip_prefix(&self, key: &str) -> String {
+ Self::strip_prefix_with_codec(&self.prefix, self.normalize_encoding, key)
}
/// Get an object's contents
@@ -544,3 +642,81 @@ impl S3Client {
&self.bucket
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn test_client(prefix: &str, normalize_encoding: bool) -> S3Client {
+ S3Client {
+ client: Client::from_conf(
+ aws_sdk_s3::Config::builder()
+ .behavior_version(BehaviorVersion::latest())
+ .region(Region::new("us-east-1"))
+ .build(),
+ ),
+ bucket: "test-bucket".to_string(),
+ prefix: prefix.to_string(),
+ normalize_encoding,
+ marker_mode: DirectoryMarkerMode::Empty,
+ disable_batch_delete: false,
+ }
+ }
+
+ #[test]
+ fn test_key_codec_keeps_safe_segments() {
+ let path = "dir/ok-name_1.2!*'()";
+ assert_eq!(encode_path(path, true), path);
+ assert_eq!(decode_path(path, true), path);
+ }
+
+ #[test]
+ fn test_key_codec_encodes_reserved_and_unsafe_segments() {
+ let encoded = encode_path("a b/c?d/#frag/%raw", true);
+
+ assert_eq!(encoded, "a!20b/c!3Fd/!23frag/!25raw");
+ assert_eq!(decode_path(&encoded, true), "a b/c?d/#frag/%raw");
+ }
+
+ #[test]
+ fn test_key_codec_encodes_segments_with_reserved_prefix() {
+ let encoded = encode_path("!literal/normal", true);
+
+ assert_eq!(encoded, "!literal/normal");
+ assert_eq!(decode_path(&encoded, true), "!literal/normal");
+ }
+
+ #[test]
+ fn test_key_codec_disabled_is_passthrough() {
+ let path = "a b/c?d";
+ assert_eq!(encode_path(path, false), path);
+ assert_eq!(decode_path(path, false), path);
+ }
+
+ #[test]
+ fn test_key_codec_preserves_path_structure_with_mixed_segments() {
+ let path = "dir//safe-_.*/@scope+pkg/file name/";
+ let encoded = encode_path(path, true);
+
+ assert_eq!(encoded, "dir//safe-_.*/!40scope!2Bpkg/file!20name/");
+ assert_eq!(decode_path(&encoded, true), path);
+ }
+
+ #[test]
+ fn test_build_key_applies_normalized_encoding_per_segment() {
+ let key = test_client("ns", true).build_key("/dir/a b/c?d.txt");
+ assert_eq!(key, "ns/dir/a!20b/c!3Fd.txt");
+ }
+
+ #[test]
+ fn test_strip_prefix_decodes_normalized_segments() {
+ let client = test_client("ns", true);
+ let key = client.build_key("/dir/a b/c?d.txt");
+ assert_eq!(client.strip_prefix(&key), "dir/a b/c?d.txt".to_string());
+ }
+
+ #[test]
+ fn test_build_key_preserves_segments_when_normalization_disabled() {
+ assert_eq!(test_client("", false).build_key("/a b"), "a b");
+ }
+}
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index 000076f79..005c96901 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -289,7 +289,7 @@ impl FileSystem for S3FileSystem {
// Add files
for obj in &listing.files {
let rel_path = self.client.strip_prefix(&obj.key);
- let name = rel_path.rsplit('/').next().unwrap_or(rel_path);
+ let name = rel_path.rsplit('/').next().unwrap_or(&rel_path);
if name.is_empty() {
continue;
@@ -307,7 +307,7 @@ impl FileSystem for S3FileSystem {
// Add directories
for dir_key in &listing.directories {
let rel_path = self.client.strip_prefix(dir_key);
- let name = rel_path.rsplit('/').next().unwrap_or(rel_path);
+ let name = rel_path.rsplit('/').next().unwrap_or(&rel_path);
if name.is_empty() {
continue;
@@ -545,6 +545,12 @@ impl S3FSPlugin {
"",
"Key prefix for namespace isolation (e.g. 'agfs/')",
),
+ ConfigParameter::optional(
+ "normalize_encoding",
+ "bool",
+ "false",
+ "Normalize unsafe path segments by base32-encoding URL-unsafe characters in S3 object keys",
+ ),
ConfigParameter::optional(
"directory_marker_mode",
"string",
@@ -619,6 +625,7 @@ A file system backed by Amazon S3 or S3-compatible object storage.
- Dual-layer caching (directory listings + stat metadata)
- Range-based reads for partial file access
- Configurable directory marker modes
+- Optional URL-safe key normalization for unsafe path segments
## Configuration
@@ -659,6 +666,7 @@ plugins:
endpoint: https://tos-cn-beijing.volces.com
use_path_style: false
directory_marker_mode: nonempty
+ normalize_encoding: true
```
### Alibaba Cloud OSS
@@ -680,6 +688,13 @@ plugins:
- `nonempty`: Single-byte marker (for TOS and services that reject zero-byte objects)
- `none`: No markers, pure prefix-based directory detection
+## Key Normalization
+
+- `normalize_encoding: false`: keep original path segments in object keys
+- `normalize_encoding: true` (default): escape only URL-reserved or URL-unsafe bytes as `!HH`
+- Allowed special characters without encoding: `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, `)`
+- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole path segment
+
## Notes
- S3 does not support partial/offset writes (always full object replacement)
@@ -713,6 +728,14 @@ plugins:
}
}
+ if let Some(value) = config.params.get("normalize_encoding") {
+ if value.as_bool().is_none() {
+ return Err(Error::config(
+ "invalid normalize_encoding: expected bool",
+ ));
+ }
+ }
+
Ok(())
}
@@ -822,4 +845,41 @@ mod tests {
};
assert!(plugin.validate(&config).await.is_ok());
}
+
+ #[tokio::test]
+ async fn test_plugin_validate_normalize_encoding() {
+ let plugin = S3FSPlugin::new();
+
+ let mut params = std::collections::HashMap::new();
+ params.insert(
+ "bucket".to_string(),
+ crate::core::ConfigValue::String("test".to_string()),
+ );
+ params.insert(
+ "normalize_encoding".to_string(),
+ crate::core::ConfigValue::String("true".to_string()),
+ );
+ let config = PluginConfig {
+ name: "s3fs".to_string(),
+ mount_path: "/s3".to_string(),
+ params,
+ };
+ assert!(plugin.validate(&config).await.is_err());
+
+ let mut params = std::collections::HashMap::new();
+ params.insert(
+ "bucket".to_string(),
+ crate::core::ConfigValue::String("test".to_string()),
+ );
+ params.insert(
+ "normalize_encoding".to_string(),
+ crate::core::ConfigValue::Bool(true),
+ );
+ let config = PluginConfig {
+ name: "s3fs".to_string(),
+ mount_path: "/s3".to_string(),
+ params,
+ };
+ assert!(plugin.validate(&config).await.is_ok());
+ }
}
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index ea8bfb196..518cfbcfc 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -777,6 +777,7 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro
| `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections | true |
| `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true |
| `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` |
+| `normalize_encoding` | bool | Escape only URL-reserved or URL-unsafe characters in S3 object keys as `!HH` hexadecimal bytes while preserving safe characters | `true` |
`directory_marker_mode` controls how RAGFS materializes directory objects in S3:
@@ -790,6 +791,15 @@ Typical choices:
- For TOS or other VirtualHostStyle backends that reject zero-byte directory markers, use `nonempty`.
- If you want pure prefix-style behavior and do not need persisted empty directories, use `none`.
+`normalize_encoding` controls whether RAGFS rewrites unsafe path segments before issuing S3 requests:
+
+- `true` is the default. RAGFS preserves safe path characters and rewrites only unsafe bytes.
+- Unsafe bytes are encoded as `!HH`, where `HH` is the uppercase hexadecimal value of the byte.
+- The characters `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, and `)` remain unescaped.
+- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole segment.
+- Set `false` to keep original path segments in object keys.
+
+</details>
<details>
<summary><b>PathStyle S3</b></summary>
Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS.
@@ -804,7 +814,8 @@ Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS.
"endpoint": "s3.amazonaws.com",
"region": "us-east-1",
"access_key": "your-ak",
- "secret_key": "your-sk"
+ "secret_key": "your-sk",
+ "normalize_encoding": true
}
}
}
@@ -829,7 +840,8 @@ Supports S3 storage in VirtualHostStyle mode, such as TOS.
"access_key": "your-ak",
"secret_key": "your-sk",
"use_path_style": false,
- "directory_marker_mode": "nonempty"
+ "directory_marker_mode": "nonempty",
+ "normalize_encoding": true
}
}
}
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index 75a3ad767..b033965a2 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -750,6 +750,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
| `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS) | true |
| `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true |
| `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` |
+| `normalize_encoding` | bool | 仅把 S3 object key 中 URL 保留字符或不安全字符转义为 `!HH` 十六进制字节,同时保留安全字符 | `true` |
`directory_marker_mode` 用来控制 RAGFS 在 S3 中如何落目录对象:
@@ -763,6 +764,15 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
- 对 TOS 或其他拒绝 0 字节目录 marker 的 VirtualHostStyle 后端,使用 `nonempty`。
- 如果你想完全使用 prefix 风格行为,并且不需要持久化空目录,可以使用 `none`。
+`normalize_encoding` 用来控制 RAGFS 在发起 S3 请求前是否重写不安全路径段:
+
+- `true` 是默认值。RAGFS 会保留安全路径字符,只重写不安全字节。
+- 不安全字节会被编码成 `!HH`,其中 `HH` 是该字节的大写十六进制值。
+- `/`、`!`、`-`、`_`、`.`、`*`、`'`、`(`、`)` 这些字符会保留,不触发编码。
+- `?`、`&`、`#`、空格、`%`、`@`、`+` 等字符会在原位被转义,不会导致整个路径段一起重写。
+- 设为 `false` 时,会在 object key 中保留原始路径段。
+
+</details>
<details>
<summary><b>PathStyle S3</b></summary>
支持 PathStyle 模式的 S3 存储, 如 MinIO、SeaweedFS.
@@ -777,7 +787,8 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
"endpoint": "s3.amazonaws.com",
"region": "us-east-1",
"access_key": "your-ak",
- "secret_key": "your-sk"
+ "secret_key": "your-sk",
+ "normalize_encoding": true
}
}
}
@@ -802,7 +813,8 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
"access_key": "your-ak",
"secret_key": "your-sk",
"use_path_style": false,
- "directory_marker_mode": "nonempty"
+ "directory_marker_mode": "nonempty",
+ "normalize_encoding": true
}
}
}
diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py
index 2b70be8c0..0a8e85eae 100644
--- a/openviking/utils/agfs_utils.py
+++ b/openviking/utils/agfs_utils.py
@@ -59,6 +59,7 @@ def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]
if hasattr(s3_config.directory_marker_mode, "value")
else s3_config.directory_marker_mode,
"disable_batch_delete": s3_config.disable_batch_delete,
+ "normalize_encoding": s3_config.normalize_encoding,
}
config["s3fs"] = {
diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py
index ff7024a9d..11863695b 100644
--- a/openviking_cli/utils/config/agfs_config.py
+++ b/openviking_cli/utils/config/agfs_config.py
@@ -71,6 +71,13 @@ class S3Config(BaseModel):
"for DeleteObjects but AWS SDK v2 does not send it by default. Defaults to False.",
)
+ normalize_encoding: bool = Field(
+ default=True,
+ description="Normalize URL-unsafe path segments before writing S3 object keys. "
+ "When enabled, reserved or unsafe URL bytes such as ?, &, #, spaces, %, @, or + are "
+ "escaped in place as !HH hexadecimal bytes, while / ! - _ . * ' ( ) remain unescaped.",
+ )
+
model_config = {"extra": "forbid"}
def validate_config(self):
diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py
index a0a66d4cb..0be0b9e8a 100644
--- a/tests/misc/test_config_validation.py
+++ b/tests/misc/test_config_validation.py
@@ -4,10 +4,12 @@
"""Test if config validators work correctly"""
import sys
+from pathlib import Path
+from openviking.utils.agfs_utils import _generate_plugin_config
from openviking_cli.utils.config.agfs_config import AGFSConfig, S3Config
from openviking_cli.utils.config.embedding_config import EmbeddingConfig, EmbeddingModelConfig
-from openviking_cli.utils.config.vectordb_config import VolcengineConfig, VectorDBBackendConfig
+from openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig, VolcengineConfig
from openviking_cli.utils.config.vlm_config import VLMConfig
@@ -25,6 +27,40 @@ def test_agfs_validation():
except ValueError as e:
print(f" Fail: {e}")
+
+def test_agfs_s3_normalize_encoding_defaults_to_false():
+ config = AGFSConfig(
+ backend="s3",
+ s3=S3Config(
+ bucket="my-bucket",
+ region="us-west-1",
+ access_key="fake-access-key-for-testing",
+ secret_key="fake-secret-key-for-testing-12345",
+ endpoint="https://s3.amazonaws.com",
+ ),
+ )
+
+ assert config.s3.normalize_encoding is False
+
+
+def test_agfs_s3_normalize_encoding_is_forwarded_to_ragfs_plugin_config():
+ config = AGFSConfig(
+ path="/tmp/ov-test",
+ backend="s3",
+ s3=S3Config(
+ bucket="my-bucket",
+ region="us-west-1",
+ access_key="fake-access-key-for-testing",
+ secret_key="fake-secret-key-for-testing-12345",
+ endpoint="https://s3.amazonaws.com",
+ normalize_encoding=True,
+ ),
+ )
+
+ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
+
+ assert plugins["s3fs"]["config"]["normalize_encoding"] is True
+
# Test 2: invalid backend
print("\n2. Test invalid backend...")
try: