Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(storage): support content-type auto-detecting in S3 case (#2668)

baojun-zhangeff7e670377a
diff --git a/Cargo.lock b/Cargo.lock
index 0ba9e32b9..d79c3a35b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3713,6 +3713,7 @@ dependencies = [
"hkdf",
"ignore",
"lru",
+ "mime_guess",
"path-clean",
"radix_trie",
"rand 0.8.5",
diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml
index 5f2f828f8..86cfcb1fd 100644
--- a/crates/ragfs/Cargo.toml
+++ b/crates/ragfs/Cargo.toml
@@ -63,6 +63,7 @@ lru = "0.12"
# Regular expressions for grep
regex = "1.10"
+mime_guess = "2.0"
# Encryption (envelope encryption: AES-256-GCM + HKDF-SHA256)
aes-gcm = "0.10"
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rs
index a39178f0d..76d122e46 100644
--- a/crates/ragfs/src/plugins/s3fs/client.rs
+++ b/crates/ragfs/src/plugins/s3fs/client.rs
@@ -255,6 +255,20 @@ fn aws_datetime_to_systemtime(dt: &aws_sdk_s3::primitives::DateTime) -> SystemTi
}
}
+/// The Content-Type to be written to S3 is inferred from the filename suffix of the object key.
+fn detect_content_type_for_key(key: &str) -> Option<String> {
+ if key.ends_with('/') {
+ return None;
+ }
+
+ Some(
+ mime_guess::from_path(key)
+ .first_or_octet_stream()
+ .essence_str()
+ .to_string(),
+ )
+}
+
/// S3 Client wrapper
pub struct S3Client {
client: Client,
@@ -263,6 +277,7 @@ pub struct S3Client {
normalize_encoding_chars: String,
marker_mode: DirectoryMarkerMode,
disable_batch_delete: bool,
+ auto_detect_content_type: bool,
}
impl S3Client {
@@ -347,6 +362,11 @@ impl S3Client {
.and_then(|v| v.as_bool())
.unwrap_or(false);
+ let auto_detect_content_type = config
+ .get("auto_detect_content_type")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
+
// Build S3 config
let mut s3_config_builder = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
@@ -374,6 +394,7 @@ impl S3Client {
normalize_encoding_chars,
marker_mode,
disable_batch_delete,
+ auto_detect_content_type,
})
}
@@ -472,14 +493,26 @@ impl S3Client {
/// Upload an object
pub async fn put_object(&self, key: &str, data: Vec<u8>) -> Result<()> {
- self.client
+ let mut request = self
+ .client
.put_object()
.bucket(&self.bucket)
.key(key)
- .body(ByteStream::from(data))
- .send()
- .await
- .map_err(|e| format_sdk_s3_error("PutObject", &format!("bucket={} key={key}", self.bucket), &e))?;
+ .body(ByteStream::from(data));
+
+ if self.auto_detect_content_type {
+ if let Some(content_type) = detect_content_type_for_key(key) {
+ request = request.content_type(content_type);
+ }
+ }
+
+ request.send().await.map_err(|e| {
+ format_sdk_s3_error(
+ "PutObject",
+ &format!("bucket={} key={key}", self.bucket),
+ &e,
+ )
+ })?;
Ok(())
}
@@ -621,16 +654,13 @@ impl S3Client {
req = req.continuation_token(token);
}
- let resp = req
- .send()
- .await
- .map_err(|e| {
- format_sdk_s3_error(
- "ListObjectsV2",
- &format!("bucket={} prefix={prefix}", self.bucket),
- &e,
- )
- })?;
+ let resp = req.send().await.map_err(|e| {
+ format_sdk_s3_error(
+ "ListObjectsV2",
+ &format!("bucket={} prefix={prefix}", self.bucket),
+ &e,
+ )
+ })?;
// Process files (contents)
for obj in resp.contents() {
@@ -922,6 +952,7 @@ mod tests {
normalize_encoding_chars: normalize_encoding_chars.to_string(),
marker_mode: DirectoryMarkerMode::Empty,
disable_batch_delete: false,
+ auto_detect_content_type: false,
}
}
@@ -990,6 +1021,31 @@ mod tests {
assert_eq!(test_client("", "").build_key("/a b"), "a b");
}
+ #[test]
+ fn test_detect_content_type_for_key_uses_filename_extension() {
+ assert_eq!(
+ detect_content_type_for_key("tenant/docs/readme.md").as_deref(),
+ Some("text/markdown")
+ );
+ assert_eq!(
+ detect_content_type_for_key("tenant/docs/data.json").as_deref(),
+ Some("application/json")
+ );
+ assert_eq!(
+ detect_content_type_for_key("tenant/images/logo.png").as_deref(),
+ Some("image/png")
+ );
+ }
+
+ #[test]
+ fn test_detect_content_type_for_key_handles_unknown_and_directory_marker() {
+ assert_eq!(
+ detect_content_type_for_key("tenant/blob/no-extension").as_deref(),
+ Some("application/octet-stream")
+ );
+ assert_eq!(detect_content_type_for_key("tenant/dir/"), None);
+ }
+
#[test]
fn test_format_generic_s3_error_includes_operation_bucket_key_and_raw_error() {
let err = format_generic_s3_error(
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index a7e75322d..0d3646687 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -900,6 +900,12 @@ impl S3FSPlugin {
"false",
"Disable batch delete (DeleteObjects) for S3-compatible services like OSS",
),
+ ConfigParameter::optional(
+ "auto_detect_content_type",
+ "bool",
+ "false",
+ "Infer S3 object Content-Type from the object key filename extension",
+ ),
ConfigParameter::optional(
"cache_enabled",
"bool",
@@ -1072,6 +1078,14 @@ plugins:
}
}
+ if let Some(value) = config.params.get("auto_detect_content_type") {
+ if value.as_bool().is_none() {
+ return Err(Error::config(
+ "invalid auto_detect_content_type: expected bool",
+ ));
+ }
+ }
+
Ok(())
}
@@ -1214,6 +1228,45 @@ mod tests {
assert!(plugin.validate(&config).await.is_ok());
}
+ #[tokio::test]
+ async fn test_plugin_validate_auto_detect_content_type() {
+ let plugin = S3FSPlugin::new();
+
+ let mut params = std::collections::HashMap::new();
+ params.insert(
+ "bucket".to_string(),
+ crate::core::ConfigValue::String("test".to_string()),
+ );
+ params.insert(
+ "auto_detect_content_type".to_string(),
+ crate::core::ConfigValue::String("true".to_string()),
+ );
+ let config = PluginConfig {
+ name: "s3fs".to_string(),
+ mount_path: "/s3".to_string(),
+ params,
+ ..PluginConfig::default()
+ };
+ assert!(plugin.validate(&config).await.is_err());
+
+ let mut params = std::collections::HashMap::new();
+ params.insert(
+ "bucket".to_string(),
+ crate::core::ConfigValue::String("test".to_string()),
+ );
+ params.insert(
+ "auto_detect_content_type".to_string(),
+ crate::core::ConfigValue::Bool(true),
+ );
+ let config = PluginConfig {
+ name: "s3fs".to_string(),
+ mount_path: "/s3".to_string(),
+ params,
+ ..PluginConfig::default()
+ };
+ assert!(plugin.validate(&config).await.is_ok());
+ }
+
#[test]
fn test_s3_is_excluded_path() {
assert!(
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 06f38e445..59a83e69d 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -1082,6 +1082,7 @@ Legacy compatibility example:
| `prefix` | str | Optional key prefix for namespace isolation | "" |
| `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections. Also controls the scheme auto-prefixed onto bare-hostname `endpoint` values | true |
| `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true |
+| `auto_detect_content_type` | bool | Automatically infer MIME type from the object key / filename extension and set the S3 object `Content-Type` header during upload | false |
| `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` |
| `normalize_encoding_chars` | str | Characters to escape in S3 object keys as `!HH` hexadecimal bytes; empty string disables normalization | `"?#%+@"` |
@@ -1104,6 +1105,32 @@ Typical choices:
- Characters not listed in `normalize_encoding_chars`, including Chinese and other Unicode characters, remain unchanged.
- Set `normalize_encoding_chars` to `""` to keep original path segments in object keys.
+`auto_detect_content_type` is disabled by default for backward compatibility. When enabled, RAGFS infers the MIME type from the object key / filename extension and writes it to the S3 object `Content-Type`:
+
+- Detection is based on the object key / filename extension, not file content sniffing.
+- Directory markers whose keys end with `/` do not get a `Content-Type`.
+- Unknown extensions fall back to `application/octet-stream`.
+
+Example:
+
+```json
+{
+ "storage": {
+ "agfs": {
+ "backend": "s3",
+ "s3": {
+ "bucket": "my-bucket",
+ "endpoint": "s3.amazonaws.com",
+ "region": "us-east-1",
+ "access_key": "your-ak",
+ "secret_key": "your-sk",
+ "auto_detect_content_type": true
+ }
+ }
+ }
+}
+```
+
<details>
<summary><b>PathStyle S3</b></summary>
Supports S3 storage in PathStyle mode, such as MinIO, SeaweedFS.
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index e945500a2..93b32e9ed 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -1054,6 +1054,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
| `prefix` | str | 用于命名空间隔离的可选键前缀 | "" |
| `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS)。也用于决定 `endpoint` 仅填主机名时自动补的协议前缀 | true |
| `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true |
+| `auto_detect_content_type` | bool | 上传时根据 object key / 文件名后缀自动推断 MIME 类型,并写入 S3 对象的 `Content-Type` | false |
| `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` |
| `normalize_encoding_chars` | str | 需要在 S3 object key 中转义为 `!HH` 十六进制字节的字符集合;空字符串表示关闭编码 | `"?#%+@"` |
@@ -1076,6 +1077,32 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
- 没有列在 `normalize_encoding_chars` 里的字符,包括中文和其他 Unicode 字符,都会保持原样。
- 设为 `""` 时,会在 object key 中保留原始路径段。
+`auto_detect_content_type` 默认关闭,以兼容历史行为。开启后,RAGFS 会根据 object key / 文件名后缀推断 MIME 类型,并写入 S3 对象的 `Content-Type`:
+
+- 探测依据是 object key / 文件名后缀,不做文件内容 sniff。
+- key 以 `/` 结尾的目录 marker 不会写 `Content-Type`。
+- 无法识别的后缀会回退到 `application/octet-stream`。
+
+示例:
+
+```json
+{
+ "storage": {
+ "agfs": {
+ "backend": "s3",
+ "s3": {
+ "bucket": "my-bucket",
+ "endpoint": "s3.amazonaws.com",
+ "region": "us-east-1",
+ "access_key": "your-ak",
+ "secret_key": "your-sk",
+ "auto_detect_content_type": true
+ }
+ }
+ }
+}
+```
+
<details>
<summary><b>PathStyle S3</b></summary>
支持 PathStyle 模式的 S3 存储, 如 MinIO、SeaweedFS.
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index 48ad9aa15..a86d31f3a 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -69,6 +69,7 @@
"endpoint": null,
"prefix": "",
"use_ssl": true,
+ "auto_detect_content_type": false,
"disable_batch_delete": false,
},
},
diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py
index 04b507f74..92ca27cb5 100644
--- a/openviking/utils/agfs_utils.py
+++ b/openviking/utils/agfs_utils.py
@@ -230,6 +230,7 @@ def _serialize_s3_plugin_params(s3_config: Any) -> Dict[str, Any]:
"normalize_encoding_chars": _get_config_value(
s3_config, "normalize_encoding_chars", "?#%+@"
),
+ "auto_detect_content_type": _get_config_value(s3_config, "auto_detect_content_type", False),
}
diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py
index d7652dab8..e956ddd10 100644
--- a/openviking_cli/utils/config/agfs_config.py
+++ b/openviking_cli/utils/config/agfs_config.py
@@ -79,6 +79,12 @@ class S3Config(BaseModel):
"Set to an empty string to disable key normalization. Defaults to ?#%+@.",
)
+ auto_detect_content_type: bool = Field(
+ default=False,
+ description="Automatically infer S3 object Content-Type from the object key filename extension "
+ "during uploads. Disabled by default for backward compatibility.",
+ )
+
model_config = {"extra": "forbid"}
def validate_config(self):
diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py
index 60fdd7b26..40880a1f7 100644
--- a/tests/misc/test_config_validation.py
+++ b/tests/misc/test_config_validation.py
@@ -68,6 +68,40 @@ def test_agfs_s3_normalize_encoding_chars_is_forwarded_to_ragfs_plugin_config():
assert plugins["s3fs"]["config"]["normalize_encoding_chars"] == "?#"
+
+def test_agfs_s3_auto_detect_content_type_defaults_to_false():
+ config = AGFSConfig(
+ backend="s3",
+ s3=S3Config(
+ bucket="my-bucket",
+ region="us-west-1",
+ access_key="fake-access-key-for-testing",
+ secret_key="fake-secret-key-for-testing-12345",
+ endpoint="https://s3.amazonaws.com",
+ ),
+ )
+
+ assert config.s3.auto_detect_content_type is False
+
+
+def test_agfs_s3_auto_detect_content_type_is_forwarded_to_ragfs_plugin_config():
+ config = AGFSConfig(
+ path="/tmp/ov-test",
+ backend="s3",
+ s3=S3Config(
+ bucket="my-bucket",
+ region="us-west-1",
+ access_key="fake-access-key-for-testing",
+ secret_key="fake-secret-key-for-testing-12345",
+ endpoint="https://s3.amazonaws.com",
+ auto_detect_content_type=True,
+ ),
+ )
+
+ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
+
+ assert plugins["s3fs"]["config"]["auto_detect_content_type"] is True
+
# Test 2: invalid backend
print("\n2. Test invalid backend...")
try:
@@ -364,6 +398,7 @@ def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):
"directory_marker_mode": None,
"disable_batch_delete": False,
"normalize_encoding_chars": "#?",
+ "auto_detect_content_type": False,
}