Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Revert "feat(s3): add aliyun oss vendor support and config (#4564)" (#4603)
baojun-zhang35d0b69636ad
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rsindex fc37e83b8..b8f4a5d91 100644--- a/crates/ragfs/src/plugins/s3fs/client.rs+++ b/crates/ragfs/src/plugins/s3fs/client.rs@@ -8,8 +8,6 @@ use aws_sdk_s3::config::http::HttpResponse;use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};use aws_sdk_s3::error::ProvideErrorMetadata;use aws_sdk_s3::error::SdkError;-use aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder;-use aws_sdk_s3::operation::put_object::{PutObjectError, PutObjectOutput};use aws_sdk_s3::operation::{RequestId, RequestIdExt};use aws_sdk_s3::primitives::ByteStream;use aws_sdk_s3::Client;@@ -20,34 +18,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};const ENCODED_SEGMENT_PREFIX: char = '!';const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF";-/// S3-compatible vendor behavior selected by configuration.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub(super) enum S3Vendor {- /// Standard S3-compatible behavior.- Standard,- /// Alibaba Cloud OSS S3-compatible behavior.- AliyunOss,-}--impl S3Vendor {- /// Parse the S3 vendor option from plugin configuration.- ///- /// # Arguments- /// * `config` - S3FS plugin configuration.- ///- /// # Returns- /// The configured S3 vendor, or `Standard` when omitted.- pub(super) fn from_config(config: &HashMap<String, ConfigValue>) -> Result<Self> {- match config.get("s3_vendor").and_then(|v| v.as_string()) {- None | Some("") | Some("standard") => Ok(Self::Standard),- Some("aliyun_oss") => Ok(Self::AliyunOss),- Some(v) => Err(Error::config(format!(- "invalid s3_vendor: {v}; expected one of: standard, aliyun_oss"- ))),- }- }-}-fn partial_delete_error(bucket: &str, errors: &[aws_sdk_s3::types::Error]) -> Option<Error> {if errors.is_empty() {return None;@@ -399,7 +369,6 @@ pub struct S3Client {marker_mode: DirectoryMarkerMode,disable_batch_delete: bool,auto_detect_content_type: bool,- s3_vendor: S3Vendor,}impl S3Client {@@ -428,8 +397,6 @@ impl S3Client {.unwrap_or("us-east-1").to_string();- let s3_vendor = S3Vendor::from_config(config)?;-let raw_endpoint = config.get("endpoint").and_then(|v| v.as_string());let use_ssl = if let Some(v) = config.get("use_ssl").and_then(|v| v.as_bool()) {v@@ -519,7 +486,6 @@ impl S3Client {marker_mode,disable_batch_delete,auto_detect_content_type,- s3_vendor,})}@@ -689,6 +655,7 @@ impl S3Client {.put_object().bucket(&self.bucket).key(key)+ .if_none_match("*").body(ByteStream::from(data));if self.auto_detect_content_type {@@ -697,19 +664,17 @@ impl S3Client {}}- self.if_none_match_adapter(request, "*")- .await- .map_err(|e| {- if is_s3_conditional_failure(&e) {- Error::already_exists(key)- } else {- format_sdk_s3_error(- "PutObjectCreateNew",- &format!("bucket={} key={key}", self.bucket),- &e,- )- }- })?;+ request.send().await.map_err(|e| {+ if is_s3_conditional_failure(&e) {+ Error::already_exists(key)+ } else {+ format_sdk_s3_error(+ "PutObjectCreateNew",+ &format!("bucket={} key={key}", self.bucket),+ &e,+ )+ }+ })?;Ok(())}@@ -721,6 +686,7 @@ impl S3Client {.put_object().bucket(&self.bucket).key(key)+ .if_match(etag).body(ByteStream::from(data));if self.auto_detect_content_type {@@ -729,7 +695,7 @@ impl S3Client {}}- self.if_match_adapter(request, etag, key).await.map_err(|e| {+ request.send().await.map_err(|e| {if is_s3_conditional_failure(&e) {Error::AlreadyExists(key.to_string())} else {@@ -748,46 +714,6 @@ impl S3Client {})}- /// Apply create-if-absent behavior to a PutObject request.- async fn if_none_match_adapter(- &self,- request: PutObjectFluentBuilder,- value: &str,- ) -> std::result::Result<PutObjectOutput, SdkError<PutObjectError, HttpResponse>> {- match self.s3_vendor {- S3Vendor::Standard => request.if_none_match(value).send().await,- S3Vendor::AliyunOss => {- request- .customize()- .mutate_request(|req| {- req.headers_mut().insert("x-oss-forbid-overwrite", "true");- })- .send()- .await- }- }- }-- /// Apply match-current-ETag behavior to a PutObject request when supported.- async fn if_match_adapter(- &self,- request: PutObjectFluentBuilder,- etag: &str,- key: &str,- ) -> std::result::Result<PutObjectOutput, SdkError<PutObjectError, HttpResponse>> {- match self.s3_vendor {- S3Vendor::Standard => request.if_match(etag).send().await,- S3Vendor::AliyunOss => {- tracing::info!(- bucket = %self.bucket,- key = %key,- "aliyun_oss does not support PutObject If-Match; proceeding without CAS guarantee"- );- request.send().await- }- }- }-/// Delete a single objectpub async fn delete_object(&self, key: &str) -> Result<()> {self.client@@ -1301,7 +1227,6 @@ mod tests {}}- /// Build a test S3 client without performing network setup.fn test_client(prefix: &str, normalize_encoding_chars: &str) -> S3Client {S3Client {client: Client::from_conf(@@ -1316,7 +1241,6 @@ mod tests {marker_mode: DirectoryMarkerMode::Empty,disable_batch_delete: false,auto_detect_content_type: false,- s3_vendor: S3Vendor::Standard,}}diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rsindex acf5b4da2..eeec24015 100644--- a/crates/ragfs/src/plugins/s3fs/mod.rs+++ b/crates/ragfs/src/plugins/s3fs/mod.rs@@ -23,7 +23,7 @@ use std::sync::{Arc, Mutex};use std::time::SystemTime;use cache::{S3ListDirCache, S3StatCache};-use client::{ListTreePage, S3Client, S3Vendor};+use client::{ListTreePage, S3Client};use futures::stream::{self, StreamExt};use regex::Regex;use std::sync::atomic::{AtomicUsize, Ordering};@@ -1175,12 +1175,6 @@ impl S3FSPlugin {"true","Use path-style addressing (bucket/key vs bucket.host/key)",),- ConfigParameter::optional(- "s3_vendor",- "string",- "standard",- "S3 vendor behavior: standard, aliyun_oss",- ),ConfigParameter::optional("prefix","string",@@ -1326,9 +1320,7 @@ plugins:config:bucket: my-oss-bucketregion: cn-beijing- endpoint: https://s3.oss-cn-beijing.aliyuncs.com- s3_vendor: aliyun_oss- use_path_style: false+ endpoint: http://s3.oss-cn-beijing.aliyuncs.comdisable_batch_delete: true```@@ -1385,8 +1377,6 @@ plugins:}}- S3Vendor::from_config(&config.params)?;-if let Some(value) = config.params.get("auto_detect_content_type") {if value.as_bool().is_none() {return Err(Error::config(diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 633359bb3..4ebcb5d22 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -1320,7 +1320,6 @@ Code entry: `openviking/session/auto_commit_policy.py:AutoCommitPolicy`.| `prefix` | str | Optional key prefix for namespace isolation | "" || `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections. Also controls the scheme auto-prefixed onto bare-hostname `endpoint` values | true || `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true |-| `s3_vendor` | str | S3 vendor behavior: `standard` or `aliyun_oss` | `"standard"` || `auto_detect_content_type` | bool | Automatically infer MIME type from the object key / filename extension and set the S3 object `Content-Type` header during upload | false || `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` || `normalize_encoding_chars` | str | Characters to escape in S3 object keys as `!HH` hexadecimal bytes; empty string disables normalization | `"?#%+@"` |@@ -1337,18 +1336,6 @@ Typical choices:- For TOS or other VirtualHostStyle backends that reject zero-byte directory markers, use `nonempty`.- If you want pure prefix-style behavior and do not need persisted empty directories, use `none`.-`s3_vendor` controls vendor-specific request behavior:--- `standard` is the default. It uses standard S3 conditional headers.-- `aliyun_oss` is for Alibaba Cloud OSS.-- For create-if-absent writes, OSS uses `x-oss-forbid-overwrite: true`.-- For overwrite writes, OSS does not send `If-Match`.-- The write still proceeds and emits an `INFO` log.-- CAS is not guaranteed in this case. CAS means write-after-match.-- `s3_vendor` does not change other options.-- Set `use_path_style` explicitly.-- Set `disable_batch_delete` explicitly.-`normalize_encoding_chars` controls which characters RAGFS rewrites before issuing S3 requests:- The default value is `"?#%+@"`, so only `?`, `#`, `%`, `+`, and `@` are escaped.diff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.mdindex b7623c963..c16385161 100644--- a/docs/en/guides/13-multi-write-storage.md+++ b/docs/en/guides/13-multi-write-storage.md@@ -99,7 +99,6 @@ When using S3-compatible services (MinIO, RustFS, Ceph, etc.), the `s3` section| Field | Required? | Description || --- | --- | --- || `use_path_style` | Yes for most S3-compatible | Set to `true` for path-style URLs (`http://host/bucket/key`). Most S3-compatible services require this. |-| `s3_vendor` | Required for Alibaba Cloud OSS | Set to `aliyun_oss`. Create-if-absent writes use `x-oss-forbid-overwrite: true`. || `directory_marker_mode` | Yes for S3-compatible | **Must be explicitly set to `"none"`**. Without this, the RAGFS Rust binding panics with `AGFSConfigError: invalid directory_marker_mode: null` during startup, causing a silent crash loop. || `use_ssl` | Optional | Set to `false` for HTTP endpoints (e.g. `http://localhost:9000`). |@@ -122,32 +121,6 @@ When using S3-compatible services (MinIO, RustFS, Ceph, etc.), the `s3` section}```-**Alibaba Cloud OSS example:**--```json-{- "name": "oss-backup",- "backend": "s3",- "s3": {- "bucket": "my-oss-bucket",- "endpoint": "https://s3.oss-cn-beijing.aliyuncs.com",- "region": "cn-beijing",- "access_key": "your-access-key",- "secret_key": "your-secret-key",- "prefix": "openviking",- "use_path_style": false,- "s3_vendor": "aliyun_oss",- "disable_batch_delete": true- }-}-```--> `aliyun_oss` does not change other options.-> Set `use_path_style` and `disable_batch_delete` explicitly.-> OSS does not support `PutObject If-Match`.-> OpenViking continues the write and emits an `INFO` log.-> CAS is not guaranteed on that path.-> **Why is `directory_marker_mode` required?**>> S3-compatible storage services handle "directories" differently from AWS S3. The RAGFS Rust binding must know whether to write directory marker objects when creating directories. Valid values are `"none"`, `"empty"`, and `"nonempty"`. For S3-compatible services that don't use directory markers (RustFS, MinIO, Ceph, etc.), set this to `"none"`. If omitted, the Rust binding defaults to `null` which is rejected, causing the server to crash silently during startup with `AGFSConfigError: invalid directory_marker_mode: null`.diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex ce39eb3d2..c239d0a6b 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -1295,7 +1295,6 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系| `prefix` | str | 用于命名空间隔离的可选键前缀 | "" || `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS)。也用于决定 `endpoint` 仅填主机名时自动补的协议前缀 | true || `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true |-| `s3_vendor` | str | S3 厂商行为,可选 `standard`、`aliyun_oss` | `"standard"` || `auto_detect_content_type` | bool | 上传时根据 object key / 文件名后缀自动推断 MIME 类型,并写入 S3 对象的 `Content-Type` | false || `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` || `normalize_encoding_chars` | str | 需要在 S3 object key 中转义为 `!HH` 十六进制字节的字符集合;空字符串表示关闭编码 | `"?#%+@"` |@@ -1312,18 +1311,6 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系- 对 TOS 或其他拒绝 0 字节目录 marker 的 VirtualHostStyle 后端,使用 `nonempty`。- 如果你想完全使用 prefix 风格行为,并且不需要持久化空目录,可以使用 `none`。-`s3_vendor` 控制厂商请求差异:--- `standard` 是默认值。使用标准 S3 条件头。-- `aliyun_oss` 用于 Alibaba Cloud OSS。-- OSS 创建对象时,使用 `x-oss-forbid-overwrite: true`。-- OSS 覆盖写时,不发送 `If-Match`。-- 覆盖写会继续执行,并打印 `INFO` 日志。-- 此时无法保证 CAS。CAS 指按旧值匹配后再写。-- `s3_vendor` 不会修改其他配置。-- `use_path_style` 仍需显式设置。-- `disable_batch_delete` 仍需显式设置。-`normalize_encoding_chars` 用来控制 RAGFS 在发起 S3 请求前需要重写哪些字符:- 默认值是 `"?#%+@"`,所以只会转义 `?`、`#`、`%`、`+`、`@`。diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.mdindex 21dbd011d..1549592a0 100644--- a/docs/zh/guides/13-multi-write-storage.md+++ b/docs/zh/guides/13-multi-write-storage.md@@ -99,7 +99,6 @@| 字段 | 是否必填 | 说明 || --- | --- | --- || `use_path_style` | 大多数 S3 兼容服务必填 | 设置为 `true` 使用路径风格 URL(`http://host/bucket/key`)。大多数 S3 兼容服务需要此配置。 |-| `s3_vendor` | Alibaba Cloud OSS 必填 | 设置为 `aliyun_oss`。创建对象会使用 `x-oss-forbid-overwrite: true`。 || `directory_marker_mode` | S3 兼容服务必填 | **必须显式设置为 `"none"`**。如果不配置,RAGFS Rust binding 启动时会报 `AGFSConfigError: invalid directory_marker_mode: null` 并静默崩溃。 || `use_ssl` | 可选 | HTTP 端点(如 `http://localhost:9000`)需要设置为 `false`。 |@@ -122,32 +121,6 @@}```-**Alibaba Cloud OSS 示例:**--```json-{- "name": "oss-backup",- "backend": "s3",- "s3": {- "bucket": "my-oss-bucket",- "endpoint": "https://s3.oss-cn-beijing.aliyuncs.com",- "region": "cn-beijing",- "access_key": "your-access-key",- "secret_key": "your-secret-key",- "prefix": "openviking",- "use_path_style": false,- "s3_vendor": "aliyun_oss",- "disable_batch_delete": true- }-}-```--> `aliyun_oss` 不会联动其他配置。-> `use_path_style` 和 `disable_batch_delete` 仍需显式设置。-> OSS 不支持 `PutObject If-Match`。-> OpenViking 会继续写入,并打印 `INFO` 日志。-> 该路径无法保证 CAS。-> **为什么需要 `directory_marker_mode`?**>> S3 兼容存储服务对"目录"的处理方式与 AWS S3 不同。RAGFS Rust binding 必须知道创建目录时是否需要写入目录标记对象。合法取值为 `"none"`、`"empty"` 和 `"nonempty"`。对于不使用目录标记的 S3 兼容服务(RustFS、MinIO、Ceph 等),设置为 `"none"`。如果省略,Rust binding 默认值为 `null`(不合法),导致服务端在启动时静默崩溃,报错 `AGFSConfigError: invalid directory_marker_mode: null`。diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.pyindex 5898051e3..06b677e69 100644--- a/openviking/utils/agfs_utils.py+++ b/openviking/utils/agfs_utils.py@@ -383,7 +383,6 @@ def _serialize_s3_plugin_params(s3_config: Any) -> Dict[str, Any]:"prefix": _get_config_value(s3_config, "prefix", ""),"disable_ssl": not _get_config_value(s3_config, "use_ssl", True),"use_path_style": _get_config_value(s3_config, "use_path_style", True),- "s3_vendor": _get_config_value(s3_config, "s3_vendor", "standard"),"directory_marker_mode": directory_marker_mode.valueif hasattr(directory_marker_mode, "value")else directory_marker_mode,diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex 9b1c31702..4d40dc075 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -62,11 +62,6 @@ class S3Config(BaseModel):description="true represent UsePathStyle for MinIO and some S3-compatible services; false represent VirtualHostStyle for TOS and some S3-compatible services.",)- s3_vendor: Literal["standard", "aliyun_oss"] = Field(- default="standard",- description="S3 vendor behavior. Use 'aliyun_oss' for Alibaba Cloud OSS.",- )-directory_marker_mode: DirectoryMarkerMode = Field(default=DirectoryMarkerMode.EMPTY,description="How to persist S3 directory markers: 'none' skips marker creation, 'empty' writes a zero-byte marker, and 'nonempty' writes a non-empty marker payload. Defaults to 'empty'.",diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex 6369c6cfa..7a24ec966 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -390,7 +390,9 @@ def test_rediss_scheme_enables_tls_without_boolean_flag():binding = RagfsBindingConfig(config.storage.agfs, cache=config.cache).to_binding_dict()- assert binding["cache"]["redis"]["endpoints"] == ["rediss://redis.example.com:6380"]+ assert binding["cache"]["redis"]["endpoints"] == [+ "rediss://redis.example.com:6380"+ ]assert "tls_enabled" not in binding["cache"]["redis"]@@ -601,7 +603,6 @@ def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):"prefix": "backup-prefix","disable_ssl": True,"use_path_style": False,- "s3_vendor": "standard","directory_marker_mode": None,"disable_batch_delete": False,"normalize_encoding_chars": "#?",