Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(storage): support content-type auto-detecting in S3 case (#2668)
baojun-zhangeff7e670377a
diff --git a/Cargo.lock b/Cargo.lockindex 0ba9e32b9..d79c3a35b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3713,6 +3713,7 @@ dependencies = ["hkdf","ignore","lru",+ "mime_guess","path-clean","radix_trie","rand 0.8.5",diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.tomlindex 5f2f828f8..86cfcb1fd 100644--- a/crates/ragfs/Cargo.toml+++ b/crates/ragfs/Cargo.toml@@ -63,6 +63,7 @@ lru = "0.12"# Regular expressions for grepregex = "1.10"+mime_guess = "2.0"# Encryption (envelope encryption: AES-256-GCM + HKDF-SHA256)aes-gcm = "0.10"diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rsindex a39178f0d..76d122e46 100644--- a/crates/ragfs/src/plugins/s3fs/client.rs+++ b/crates/ragfs/src/plugins/s3fs/client.rs@@ -255,6 +255,20 @@ fn aws_datetime_to_systemtime(dt: &aws_sdk_s3::primitives::DateTime) -> SystemTi}}+/// The Content-Type to be written to S3 is inferred from the filename suffix of the object key.+fn detect_content_type_for_key(key: &str) -> Option<String> {+ if key.ends_with('/') {+ return None;+ }++ Some(+ mime_guess::from_path(key)+ .first_or_octet_stream()+ .essence_str()+ .to_string(),+ )+}+/// S3 Client wrapperpub struct S3Client {client: Client,@@ -263,6 +277,7 @@ pub struct S3Client {normalize_encoding_chars: String,marker_mode: DirectoryMarkerMode,disable_batch_delete: bool,+ auto_detect_content_type: bool,}impl S3Client {@@ -347,6 +362,11 @@ impl S3Client {.and_then(|v| v.as_bool()).unwrap_or(false);+ let auto_detect_content_type = config+ .get("auto_detect_content_type")+ .and_then(|v| v.as_bool())+ .unwrap_or(false);+// Build S3 configlet mut s3_config_builder = aws_sdk_s3::Config::builder().behavior_version(BehaviorVersion::latest())@@ -374,6 +394,7 @@ impl S3Client {normalize_encoding_chars,marker_mode,disable_batch_delete,+ auto_detect_content_type,})}@@ -472,14 +493,26 @@ impl S3Client {/// Upload an objectpub async fn put_object(&self, key: &str, data: Vec<u8>) -> Result<()> {- self.client+ let mut request = self+ .client.put_object().bucket(&self.bucket).key(key)- .body(ByteStream::from(data))- .send()- .await- .map_err(|e| format_sdk_s3_error("PutObject", &format!("bucket={} key={key}", self.bucket), &e))?;+ .body(ByteStream::from(data));++ if self.auto_detect_content_type {+ if let Some(content_type) = detect_content_type_for_key(key) {+ request = request.content_type(content_type);+ }+ }++ request.send().await.map_err(|e| {+ format_sdk_s3_error(+ "PutObject",+ &format!("bucket={} key={key}", self.bucket),+ &e,+ )+ })?;Ok(())}@@ -621,16 +654,13 @@ impl S3Client {req = req.continuation_token(token);}- let resp = req- .send()- .await- .map_err(|e| {- format_sdk_s3_error(- "ListObjectsV2",- &format!("bucket={} prefix={prefix}", self.bucket),- &e,- )- })?;+ let resp = req.send().await.map_err(|e| {+ format_sdk_s3_error(+ "ListObjectsV2",+ &format!("bucket={} prefix={prefix}", self.bucket),+ &e,+ )+ })?;// Process files (contents)for obj in resp.contents() {@@ -922,6 +952,7 @@ mod tests {normalize_encoding_chars: normalize_encoding_chars.to_string(),marker_mode: DirectoryMarkerMode::Empty,disable_batch_delete: false,+ auto_detect_content_type: false,}}@@ -990,6 +1021,31 @@ mod tests {assert_eq!(test_client("", "").build_key("/a b"), "a b");}+ #[test]+ fn test_detect_content_type_for_key_uses_filename_extension() {+ assert_eq!(+ detect_content_type_for_key("tenant/docs/readme.md").as_deref(),+ Some("text/markdown")+ );+ assert_eq!(+ detect_content_type_for_key("tenant/docs/data.json").as_deref(),+ Some("application/json")+ );+ assert_eq!(+ detect_content_type_for_key("tenant/images/logo.png").as_deref(),+ Some("image/png")+ );+ }++ #[test]+ fn test_detect_content_type_for_key_handles_unknown_and_directory_marker() {+ assert_eq!(+ detect_content_type_for_key("tenant/blob/no-extension").as_deref(),+ Some("application/octet-stream")+ );+ assert_eq!(detect_content_type_for_key("tenant/dir/"), None);+ }+#[test]fn test_format_generic_s3_error_includes_operation_bucket_key_and_raw_error() {let err = format_generic_s3_error(diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rsindex a7e75322d..0d3646687 100644--- a/crates/ragfs/src/plugins/s3fs/mod.rs+++ b/crates/ragfs/src/plugins/s3fs/mod.rs@@ -900,6 +900,12 @@ impl S3FSPlugin {"false","Disable batch delete (DeleteObjects) for S3-compatible services like OSS",),+ ConfigParameter::optional(+ "auto_detect_content_type",+ "bool",+ "false",+ "Infer S3 object Content-Type from the object key filename extension",+ ),ConfigParameter::optional("cache_enabled","bool",@@ -1072,6 +1078,14 @@ plugins:}}+ if let Some(value) = config.params.get("auto_detect_content_type") {+ if value.as_bool().is_none() {+ return Err(Error::config(+ "invalid auto_detect_content_type: expected bool",+ ));+ }+ }+Ok(())}@@ -1214,6 +1228,45 @@ mod tests {assert!(plugin.validate(&config).await.is_ok());}+ #[tokio::test]+ async fn test_plugin_validate_auto_detect_content_type() {+ let plugin = S3FSPlugin::new();++ let mut params = std::collections::HashMap::new();+ params.insert(+ "bucket".to_string(),+ crate::core::ConfigValue::String("test".to_string()),+ );+ params.insert(+ "auto_detect_content_type".to_string(),+ crate::core::ConfigValue::String("true".to_string()),+ );+ let config = PluginConfig {+ name: "s3fs".to_string(),+ mount_path: "/s3".to_string(),+ params,+ ..PluginConfig::default()+ };+ assert!(plugin.validate(&config).await.is_err());++ let mut params = std::collections::HashMap::new();+ params.insert(+ "bucket".to_string(),+ crate::core::ConfigValue::String("test".to_string()),+ );+ params.insert(+ "auto_detect_content_type".to_string(),+ crate::core::ConfigValue::Bool(true),+ );+ let config = PluginConfig {+ name: "s3fs".to_string(),+ mount_path: "/s3".to_string(),+ params,+ ..PluginConfig::default()+ };+ assert!(plugin.validate(&config).await.is_ok());+ }+#[test]fn test_s3_is_excluded_path() {assert!(diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 06f38e445..59a83e69d 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -1082,6 +1082,7 @@ Legacy compatibility example:| `prefix` | str | Optional key prefix for namespace isolation | "" || `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections. Also controls the scheme auto-prefixed onto bare-hostname `endpoint` values | true || `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true |+| `auto_detect_content_type` | bool | Automatically infer MIME type from the object key / filename extension and set the S3 object `Content-Type` header during upload | false || `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` || `normalize_encoding_chars` | str | Characters to escape in S3 object keys as `!HH` hexadecimal bytes; empty string disables normalization | `"?#%+@"` |@@ -1104,6 +1105,32 @@ Typical choices:- Characters not listed in `normalize_encoding_chars`, including Chinese and other Unicode characters, remain unchanged.- Set `normalize_encoding_chars` to `""` to keep original path segments in object keys.+`auto_detect_content_type` is disabled by default for backward compatibility. When enabled, RAGFS infers the MIME type from the object key / filename extension and writes it to the S3 object `Content-Type`:++- Detection is based on the object key / filename extension, not file content sniffing.+- Directory markers whose keys end with `/` do not get a `Content-Type`.+- Unknown extensions fall back to `application/octet-stream`.++Example:++```json+{+ "storage": {+ "agfs": {+ "backend": "s3",+ "s3": {+ "bucket": "my-bucket",+ "endpoint": "s3.amazonaws.com",+ "region": "us-east-1",+ "access_key": "your-ak",+ "secret_key": "your-sk",+ "auto_detect_content_type": true+ }+ }+ }+}+```+<details><summary><b>PathStyle S3</b></summary>Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS.diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex e945500a2..93b32e9ed 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -1054,6 +1054,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系| `prefix` | str | 用于命名空间隔离的可选键前缀 | "" || `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS)。也用于决定 `endpoint` 仅填主机名时自动补的协议前缀 | true || `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true |+| `auto_detect_content_type` | bool | 上传时根据 object key / 文件名后缀自动推断 MIME 类型,并写入 S3 对象的 `Content-Type` | false || `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` || `normalize_encoding_chars` | str | 需要在 S3 object key 中转义为 `!HH` 十六进制字节的字符集合;空字符串表示关闭编码 | `"?#%+@"` |@@ -1076,6 +1077,32 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系- 没有列在 `normalize_encoding_chars` 里的字符,包括中文和其他 Unicode 字符,都会保持原样。- 设为 `""` 时,会在 object key 中保留原始路径段。+`auto_detect_content_type` 默认关闭,以兼容历史行为。开启后,RAGFS 会根据 object key / 文件名后缀推断 MIME 类型,并写入 S3 对象的 `Content-Type`:++- 探测依据是 object key / 文件名后缀,不做文件内容 sniff。+- key 以 `/` 结尾的目录 marker 不会写 `Content-Type`。+- 无法识别的后缀会回退到 `application/octet-stream`。++示例:++```json+{+ "storage": {+ "agfs": {+ "backend": "s3",+ "s3": {+ "bucket": "my-bucket",+ "endpoint": "s3.amazonaws.com",+ "region": "us-east-1",+ "access_key": "your-ak",+ "secret_key": "your-sk",+ "auto_detect_content_type": true+ }+ }+ }+}+```+<details><summary><b>PathStyle S3</b></summary>支持 PathStyle 模式的 S3 存储, 如 MinIO、SeaweedFS.diff --git a/examples/ov.conf.example b/examples/ov.conf.exampleindex 48ad9aa15..a86d31f3a 100644--- a/examples/ov.conf.example+++ b/examples/ov.conf.example@@ -69,6 +69,7 @@"endpoint": null,"prefix": "","use_ssl": true,+ "auto_detect_content_type": false,"disable_batch_delete": false,},},diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.pyindex 04b507f74..92ca27cb5 100644--- a/openviking/utils/agfs_utils.py+++ b/openviking/utils/agfs_utils.py@@ -230,6 +230,7 @@ def _serialize_s3_plugin_params(s3_config: Any) -> Dict[str, Any]:"normalize_encoding_chars": _get_config_value(s3_config, "normalize_encoding_chars", "?#%+@"),+ "auto_detect_content_type": _get_config_value(s3_config, "auto_detect_content_type", False),}diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex d7652dab8..e956ddd10 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -79,6 +79,12 @@ class S3Config(BaseModel):"Set to an empty string to disable key normalization. Defaults to ?#%+@.",)+ auto_detect_content_type: bool = Field(+ default=False,+ description="Automatically infer S3 object Content-Type from the object key filename extension "+ "during uploads. Disabled by default for backward compatibility.",+ )+model_config = {"extra": "forbid"}def validate_config(self):diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex 60fdd7b26..40880a1f7 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -68,6 +68,40 @@ def test_agfs_s3_normalize_encoding_chars_is_forwarded_to_ragfs_plugin_config():assert plugins["s3fs"]["config"]["normalize_encoding_chars"] == "?#"++def test_agfs_s3_auto_detect_content_type_defaults_to_false():+ config = AGFSConfig(+ backend="s3",+ s3=S3Config(+ bucket="my-bucket",+ region="us-west-1",+ access_key="fake-access-key-for-testing",+ secret_key="fake-secret-key-for-testing-12345",+ endpoint="https://s3.amazonaws.com",+ ),+ )++ assert config.s3.auto_detect_content_type is False+++def test_agfs_s3_auto_detect_content_type_is_forwarded_to_ragfs_plugin_config():+ config = AGFSConfig(+ path="/tmp/ov-test",+ backend="s3",+ s3=S3Config(+ bucket="my-bucket",+ region="us-west-1",+ access_key="fake-access-key-for-testing",+ secret_key="fake-secret-key-for-testing-12345",+ endpoint="https://s3.amazonaws.com",+ auto_detect_content_type=True,+ ),+ )++ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))++ assert plugins["s3fs"]["config"]["auto_detect_content_type"] is True+# Test 2: invalid backendprint("\n2. Test invalid backend...")try:@@ -364,6 +398,7 @@ def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):"directory_marker_mode": None,"disable_batch_delete": False,"normalize_encoding_chars": "#?",+ "auto_detect_content_type": False,}