Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(ragfs): add s3 key normalization encoding (#1685)
Jiahui Zhouce090be8267f
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rsindex a89e67fb3..4136469b0 100644--- a/crates/ragfs/src/plugins/s3fs/client.rs+++ b/crates/ragfs/src/plugins/s3fs/client.rs@@ -10,6 +10,91 @@ use aws_sdk_s3::Client;use std::collections::HashMap;use std::time::{Duration, SystemTime, UNIX_EPOCH};+const ENCODED_SEGMENT_PREFIX: char = '!';+const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF";++fn encode_path(path: &str, normalize_encoding: bool) -> String {+ if !normalize_encoding {+ return path.to_string();+ }++ if path.bytes().all(|byte| byte == b'/' || is_safe_segment_byte(byte)) {+ return path.to_string();+ }++ let extra = path+ .bytes()+ .filter(|&byte| byte != b'/' && !is_safe_segment_byte(byte))+ .count()+ * 2;+ let mut encoded = String::with_capacity(path.len() + extra);++ for &byte in path.as_bytes() {+ if byte == b'/' || is_safe_segment_byte(byte) {+ encoded.push(byte as char);+ } else {+ push_encoded_byte(&mut encoded, byte);+ }+ }++ encoded+}++fn decode_path(path: &str, normalize_encoding: bool) -> String {+ if !normalize_encoding {+ return path.to_string();+ }++ decode_segment(path)+}++fn decode_segment(segment: &str) -> String {+ if segment.is_empty() {+ return String::new();+ }++ let bytes = segment.as_bytes();+ let mut decoded = Vec::with_capacity(bytes.len());+ let mut idx = 0usize;++ while idx < bytes.len() {+ if bytes[idx] == ENCODED_SEGMENT_PREFIX as u8 && idx + 2 < bytes.len() {+ let hi = decode_hex_nibble(bytes[idx + 1]);+ let lo = decode_hex_nibble(bytes[idx + 2]);+ if let (Some(hi), Some(lo)) = (hi, lo) {+ decoded.push((hi << 4) | lo);+ idx += 3;+ continue;+ }+ }++ decoded.push(bytes[idx]);+ idx += 1;+ }++ String::from_utf8(decoded).unwrap_or_else(|_| segment.to_string())+}++fn is_safe_segment_byte(byte: u8) -> bool {+ byte.is_ascii_alphanumeric()+ || matches!(byte, b'!' | b'-' | b'_' | b'.' | b'*' | b'\'' | b'(' | b')')+}++fn push_encoded_byte(output: &mut String, byte: u8) {+ output.push(ENCODED_SEGMENT_PREFIX);+ output.push(HEX_UPPER[(byte >> 4) as usize] as char);+ output.push(HEX_UPPER[(byte & 0x0f) as usize] as char);+}++fn decode_hex_nibble(byte: u8) -> Option<u8> {+ match byte {+ b'0'..=b'9' => Some(byte - b'0'),+ b'A'..=b'F' => Some(byte - b'A' + 10),+ b'a'..=b'f' => Some(byte - b'a' + 10),+ _ => None,+ }+}+/// Directory marker mode#[derive(Debug, Clone, PartialEq)]pub enum DirectoryMarkerMode {@@ -78,11 +163,23 @@ pub struct S3Client {client: Client,bucket: String,prefix: String,+ normalize_encoding: bool,marker_mode: DirectoryMarkerMode,disable_batch_delete: bool,}impl S3Client {+ fn strip_prefix_with_codec(prefix: &str, normalize_encoding: bool, key: &str) -> String {+ let stripped = if prefix.is_empty() {+ key+ } else {+ let prefix = format!("{}/", prefix.trim_end_matches('/'));+ key.strip_prefix(&prefix).unwrap_or(key)+ };++ decode_path(stripped, normalize_encoding)+ }+/// Create a new S3 client from configurationpub async fn new(config: &HashMap<String, ConfigValue>) -> Result<Self> {let bucket = config@@ -120,6 +217,11 @@ impl S3Client {.unwrap_or("").to_string();+ let normalize_encoding = config+ .get("normalize_encoding")+ .and_then(|v| v.as_bool())+ .unwrap_or(false);+let marker_mode = config.get("directory_marker_mode").and_then(|v| v.as_string())@@ -155,6 +257,7 @@ impl S3Client {client,bucket,prefix,+ normalize_encoding,marker_mode,disable_batch_delete,})@@ -162,9 +265,9 @@ impl S3Client {/// Build the full S3 key from a filesystem pathpub fn build_key(&self, path: &str) -> String {- let clean = path.trim_start_matches('/');+ let clean = encode_path(path.trim_start_matches('/'), self.normalize_encoding);if self.prefix.is_empty() {- clean.to_string()+ clean} else {let prefix = self.prefix.trim_end_matches('/');if clean.is_empty() {@@ -176,13 +279,8 @@ impl S3Client {}/// Strip the prefix from an S3 key to get the filesystem path- pub fn strip_prefix<'a>(&self, key: &'a str) -> &'a str {- if self.prefix.is_empty() {- key- } else {- let prefix = format!("{}/", self.prefix.trim_end_matches('/'));- key.strip_prefix(&prefix).unwrap_or(key)- }+ pub fn strip_prefix(&self, key: &str) -> String {+ Self::strip_prefix_with_codec(&self.prefix, self.normalize_encoding, key)}/// Get an object's contents@@ -544,3 +642,81 @@ impl S3Client {&self.bucket}}++#[cfg(test)]+mod tests {+ use super::*;++ fn test_client(prefix: &str, normalize_encoding: bool) -> S3Client {+ S3Client {+ client: Client::from_conf(+ aws_sdk_s3::Config::builder()+ .behavior_version(BehaviorVersion::latest())+ .region(Region::new("us-east-1"))+ .build(),+ ),+ bucket: "test-bucket".to_string(),+ prefix: prefix.to_string(),+ normalize_encoding,+ marker_mode: DirectoryMarkerMode::Empty,+ disable_batch_delete: false,+ }+ }++ #[test]+ fn test_key_codec_keeps_safe_segments() {+ let path = "dir/ok-name_1.2!*'()";+ assert_eq!(encode_path(path, true), path);+ assert_eq!(decode_path(path, true), path);+ }++ #[test]+ fn test_key_codec_encodes_reserved_and_unsafe_segments() {+ let encoded = encode_path("a b/c?d/#frag/%raw", true);++ assert_eq!(encoded, "a!20b/c!3Fd/!23frag/!25raw");+ assert_eq!(decode_path(&encoded, true), "a b/c?d/#frag/%raw");+ }++ #[test]+ fn test_key_codec_encodes_segments_with_reserved_prefix() {+ let encoded = encode_path("!literal/normal", true);++ assert_eq!(encoded, "!literal/normal");+ assert_eq!(decode_path(&encoded, true), "!literal/normal");+ }++ #[test]+ fn test_key_codec_disabled_is_passthrough() {+ let path = "a b/c?d";+ assert_eq!(encode_path(path, false), path);+ assert_eq!(decode_path(path, false), path);+ }++ #[test]+ fn test_key_codec_preserves_path_structure_with_mixed_segments() {+ let path = "dir//safe-_.*/@scope+pkg/file name/";+ let encoded = encode_path(path, true);++ assert_eq!(encoded, "dir//safe-_.*/!40scope!2Bpkg/file!20name/");+ assert_eq!(decode_path(&encoded, true), path);+ }++ #[test]+ fn test_build_key_applies_normalized_encoding_per_segment() {+ let key = test_client("ns", true).build_key("/dir/a b/c?d.txt");+ assert_eq!(key, "ns/dir/a!20b/c!3Fd.txt");+ }++ #[test]+ fn test_strip_prefix_decodes_normalized_segments() {+ let client = test_client("ns", true);+ let key = client.build_key("/dir/a b/c?d.txt");+ assert_eq!(client.strip_prefix(&key), "dir/a b/c?d.txt".to_string());+ }++ #[test]+ fn test_build_key_preserves_segments_when_normalization_disabled() {+ assert_eq!(test_client("", false).build_key("/a b"), "a b");+ }+}diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rsindex 000076f79..005c96901 100644--- a/crates/ragfs/src/plugins/s3fs/mod.rs+++ b/crates/ragfs/src/plugins/s3fs/mod.rs@@ -289,7 +289,7 @@ impl FileSystem for S3FileSystem {// Add filesfor obj in &listing.files {let rel_path = self.client.strip_prefix(&obj.key);- let name = rel_path.rsplit('/').next().unwrap_or(rel_path);+ let name = rel_path.rsplit('/').next().unwrap_or(&rel_path);if name.is_empty() {continue;@@ -307,7 +307,7 @@ impl FileSystem for S3FileSystem {// Add directoriesfor dir_key in &listing.directories {let rel_path = self.client.strip_prefix(dir_key);- let name = rel_path.rsplit('/').next().unwrap_or(rel_path);+ let name = rel_path.rsplit('/').next().unwrap_or(&rel_path);if name.is_empty() {continue;@@ -545,6 +545,12 @@ impl S3FSPlugin {"","Key prefix for namespace isolation (e.g. 'agfs/')",),+ ConfigParameter::optional(+ "normalize_encoding",+ "bool",+ "false",+ "Normalize unsafe path segments by base32-encoding URL-unsafe characters in S3 object keys",+ ),ConfigParameter::optional("directory_marker_mode","string",@@ -619,6 +625,7 @@ A file system backed by Amazon S3 or S3-compatible object storage.- Dual-layer caching (directory listings + stat metadata)- Range-based reads for partial file access- Configurable directory marker modes+- Optional URL-safe key normalization for unsafe path segments## Configuration@@ -659,6 +666,7 @@ plugins:endpoint: https://tos-cn-beijing.volces.comuse_path_style: falsedirectory_marker_mode: nonempty+ normalize_encoding: true```### Alibaba Cloud OSS@@ -680,6 +688,13 @@ plugins:- `nonempty`: Single-byte marker (for TOS and services that reject zero-byte objects)- `none`: No markers, pure prefix-based directory detection+## Key Normalization++- `normalize_encoding: false`: keep original path segments in object keys+- `normalize_encoding: true` (default): escape only URL-reserved or URL-unsafe bytes as `!HH`+- Allowed special characters without encoding: `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, `)`+- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole path segment+## Notes- S3 does not support partial/offset writes (always full object replacement)@@ -713,6 +728,14 @@ plugins:}}+ if let Some(value) = config.params.get("normalize_encoding") {+ if value.as_bool().is_none() {+ return Err(Error::config(+ "invalid normalize_encoding: expected bool",+ ));+ }+ }+Ok(())}@@ -822,4 +845,41 @@ mod tests {};assert!(plugin.validate(&config).await.is_ok());}++ #[tokio::test]+ async fn test_plugin_validate_normalize_encoding() {+ let plugin = S3FSPlugin::new();++ let mut params = std::collections::HashMap::new();+ params.insert(+ "bucket".to_string(),+ crate::core::ConfigValue::String("test".to_string()),+ );+ params.insert(+ "normalize_encoding".to_string(),+ crate::core::ConfigValue::String("true".to_string()),+ );+ let config = PluginConfig {+ name: "s3fs".to_string(),+ mount_path: "/s3".to_string(),+ params,+ };+ assert!(plugin.validate(&config).await.is_err());++ let mut params = std::collections::HashMap::new();+ params.insert(+ "bucket".to_string(),+ crate::core::ConfigValue::String("test".to_string()),+ );+ params.insert(+ "normalize_encoding".to_string(),+ crate::core::ConfigValue::Bool(true),+ );+ let config = PluginConfig {+ name: "s3fs".to_string(),+ mount_path: "/s3".to_string(),+ params,+ };+ assert!(plugin.validate(&config).await.is_ok());+ }}diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex ea8bfb196..518cfbcfc 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -777,6 +777,7 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro| `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections | true || `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true || `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` |+| `normalize_encoding` | bool | Escape only URL-reserved or URL-unsafe characters in S3 object keys as `!HH` hexadecimal bytes while preserving safe characters | `true` |`directory_marker_mode` controls how RAGFS materializes directory objects in S3:@@ -790,6 +791,15 @@ Typical choices:- For TOS or other VirtualHostStyle backends that reject zero-byte directory markers, use `nonempty`.- If you want pure prefix-style behavior and do not need persisted empty directories, use `none`.+`normalize_encoding` controls whether RAGFS rewrites unsafe path segments before issuing S3 requests:++- `true` is the default. RAGFS preserves safe path characters and rewrites only unsafe bytes.+- Unsafe bytes are encoded as `!HH`, where `HH` is the uppercase hexadecimal value of the byte.+- The characters `/`, `!`, `-`, `_`, `.`, `*`, `'`, `(`, and `)` remain unescaped.+- Characters such as `?`, `&`, `#`, spaces, `%`, `@`, and `+` are escaped in place without rewriting the whole segment.+- Set `false` to keep original path segments in object keys.++</details><details><summary><b>PathStyle S3</b></summary>Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS.@@ -804,7 +814,8 @@ Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS."endpoint": "s3.amazonaws.com","region": "us-east-1","access_key": "your-ak",- "secret_key": "your-sk"+ "secret_key": "your-sk",+ "normalize_encoding": true}}}@@ -829,7 +840,8 @@ Supports S3 storage in VirtualHostStyle mode, such as TOS."access_key": "your-ak","secret_key": "your-sk","use_path_style": false,- "directory_marker_mode": "nonempty"+ "directory_marker_mode": "nonempty",+ "normalize_encoding": true}}}diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex 75a3ad767..b033965a2 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -750,6 +750,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系| `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS) | true || `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true || `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` |+| `normalize_encoding` | bool | 仅把 S3 object key 中 URL 保留字符或不安全字符转义为 `!HH` 十六进制字节,同时保留安全字符 | `true` |`directory_marker_mode` 用来控制 RAGFS 在 S3 中如何落目录对象:@@ -763,6 +764,15 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系- 对 TOS 或其他拒绝 0 字节目录 marker 的 VirtualHostStyle 后端,使用 `nonempty`。- 如果你想完全使用 prefix 风格行为,并且不需要持久化空目录,可以使用 `none`。+`normalize_encoding` 用来控制 RAGFS 在发起 S3 请求前是否重写不安全路径段:++- `true` 是默认值。RAGFS 会保留安全路径字符,只重写不安全字节。+- 不安全字节会被编码成 `!HH`,其中 `HH` 是该字节的大写十六进制值。+- `/`、`!`、`-`、`_`、`.`、`*`、`'`、`(`、`)` 这些字符会保留,不触发编码。+- `?`、`&`、`#`、空格、`%`、`@`、`+` 等字符会在原位被转义,不会导致整个路径段一起重写。+- 设为 `false` 时,会在 object key 中保留原始路径段。++</details><details><summary><b>PathStyle S3</b></summary>支持 PathStyle 模式的 S3 存储, 如 MinIO、SeaweedFS.@@ -777,7 +787,8 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系"endpoint": "s3.amazonaws.com","region": "us-east-1","access_key": "your-ak",- "secret_key": "your-sk"+ "secret_key": "your-sk",+ "normalize_encoding": true}}}@@ -802,7 +813,8 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系"access_key": "your-ak","secret_key": "your-sk","use_path_style": false,- "directory_marker_mode": "nonempty"+ "directory_marker_mode": "nonempty",+ "normalize_encoding": true}}}diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.pyindex 2b70be8c0..0a8e85eae 100644--- a/openviking/utils/agfs_utils.py+++ b/openviking/utils/agfs_utils.py@@ -59,6 +59,7 @@ def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]if hasattr(s3_config.directory_marker_mode, "value")else s3_config.directory_marker_mode,"disable_batch_delete": s3_config.disable_batch_delete,+ "normalize_encoding": s3_config.normalize_encoding,}config["s3fs"] = {diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex ff7024a9d..11863695b 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -71,6 +71,13 @@ class S3Config(BaseModel):"for DeleteObjects but AWS SDK v2 does not send it by default. Defaults to False.",)+ normalize_encoding: bool = Field(+ default=True,+ description="Normalize URL-unsafe path segments before writing S3 object keys. "+ "When enabled, reserved or unsafe URL bytes such as ?, &, #, spaces, %, @, or + are "+ "escaped in place as !HH hexadecimal bytes, while / ! - _ . * ' ( ) remain unescaped.",+ )+model_config = {"extra": "forbid"}def validate_config(self):diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex a0a66d4cb..0be0b9e8a 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -4,10 +4,12 @@"""Test if config validators work correctly"""import sys+from pathlib import Path+from openviking.utils.agfs_utils import _generate_plugin_configfrom openviking_cli.utils.config.agfs_config import AGFSConfig, S3Configfrom openviking_cli.utils.config.embedding_config import EmbeddingConfig, EmbeddingModelConfig-from openviking_cli.utils.config.vectordb_config import VolcengineConfig, VectorDBBackendConfig+from openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig, VolcengineConfigfrom openviking_cli.utils.config.vlm_config import VLMConfig@@ -25,6 +27,40 @@ def test_agfs_validation():except ValueError as e:print(f" Fail: {e}")++def test_agfs_s3_normalize_encoding_defaults_to_false():+ config = AGFSConfig(+ backend="s3",+ s3=S3Config(+ bucket="my-bucket",+ region="us-west-1",+ access_key="fake-access-key-for-testing",+ secret_key="fake-secret-key-for-testing-12345",+ endpoint="https://s3.amazonaws.com",+ ),+ )++ assert config.s3.normalize_encoding is False+++def test_agfs_s3_normalize_encoding_is_forwarded_to_ragfs_plugin_config():+ config = AGFSConfig(+ path="/tmp/ov-test",+ backend="s3",+ s3=S3Config(+ bucket="my-bucket",+ region="us-west-1",+ access_key="fake-access-key-for-testing",+ secret_key="fake-secret-key-for-testing-12345",+ endpoint="https://s3.amazonaws.com",+ normalize_encoding=True,+ ),+ )++ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))++ assert plugins["s3fs"]["config"]["normalize_encoding"] is True+# Test 2: invalid backendprint("\n2. Test invalid backend...")try: