Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Add non-empty S3 directory marker compatibility (#997)
Jiahui Zhou2ec855815570
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 337cbad4b..bd6fe9d55 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -619,6 +619,19 @@ Directly uses the AGFS Go implementation through a shared library.| `prefix` | str | Optional key prefix for namespace isolation | "" || `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections | true || `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true |+| `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` |++`directory_marker_mode` controls how AGFS materializes directory objects in S3:++- `empty` is the default. AGFS writes a zero-byte directory marker and preserves empty-directory semantics.+- `nonempty` writes a non-empty marker payload. Use this for S3-compatible services such as TOS that reject zero-byte directory markers.+- `none` switches AGFS to prefix-style S3 semantics. AGFS does not create directory marker objects, so empty directories are not persisted and may not be discoverable until they contain at least one child object.++Typical choices:++- For MinIO, SeaweedFS, and most PathStyle backends, keep the default `empty`.+- For TOS or other VirtualHostStyle backends that reject zero-byte directory markers, use `nonempty`.+- If you want pure prefix-style behavior and do not need persisted empty directories, use `none`.</details>@@ -660,7 +673,8 @@ Supports S3 storage in VirtualHostStyle mode, such as TOS."region": "us-east-1","access_key": "your-ak","secret_key": "your-sk",- "use_path_style": false+ "use_path_style": false,+ "directory_marker_mode": "nonempty"}}}diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex 2b2ead6d6..675c34234 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -595,6 +595,19 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。| `prefix` | str | 用于命名空间隔离的可选键前缀 | "" || `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS) | true || `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true |+| `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` |++`directory_marker_mode` 用来控制 AGFS 在 S3 中如何落目录对象:++- `empty` 是默认值。AGFS 会写入 0 字节目录 marker,并保留空目录语义。+- `nonempty` 会写入非空目录 marker。对于 TOS 这类拒绝 0 字节目录 marker 的 S3 兼容后端,应使用这个模式。+- `none` 会让 AGFS 采用更接近原生 S3 prefix 的目录语义,不再创建目录 marker 对象。此时空目录不会被持久化,只有目录下至少存在一个子对象后,相关目录才可能被发现。++典型选择:++- 对 MinIO、SeaweedFS 以及大多数 PathStyle 后端,保持默认 `empty` 即可。+- 对 TOS 或其他拒绝 0 字节目录 marker 的 VirtualHostStyle 后端,使用 `nonempty`。+- 如果你想完全使用 prefix 风格行为,并且不需要持久化空目录,可以使用 `none`。</details>@@ -636,7 +649,8 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。"region": "us-east-1","access_key": "your-ak","secret_key": "your-sk",- "use_path_style": false+ "use_path_style": false,+ "directory_marker_mode": "nonempty"}}}diff --git a/openviking/agfs_manager.py b/openviking/agfs_manager.pyindex 9ae796f24..0b4375cbf 100644--- a/openviking/agfs_manager.py+++ b/openviking/agfs_manager.py@@ -166,19 +166,22 @@ class AGFSManager:# AGFS S3 backend configuration (s3fs plugin)# This enables AGFS to mount an S3 bucket as a local filesystem.# Implementation details: third_party/agfs/agfs-server/pkg/plugins/s3fs/s3fs.go+ s3_plugin_config = {+ "bucket": self.s3_config.bucket,+ "region": self.s3_config.region,+ "access_key_id": self.s3_config.access_key,+ "secret_access_key": self.s3_config.secret_key,+ "endpoint": self.s3_config.endpoint,+ "prefix": self.s3_config.prefix,+ "disable_ssl": not self.s3_config.use_ssl,+ "use_path_style": self.s3_config.use_path_style,+ "directory_marker_mode": self.s3_config.directory_marker_mode.value,+ }+config["plugins"]["s3fs"] = {"enabled": True,"path": "/local",- "config": {- "bucket": self.s3_config.bucket,- "region": self.s3_config.region,- "access_key_id": self.s3_config.access_key,- "secret_access_key": self.s3_config.secret_key,- "endpoint": self.s3_config.endpoint,- "prefix": self.s3_config.prefix,- "disable_ssl": not self.s3_config.use_ssl,- "use_path_style": self.s3_config.use_path_style,- },+ "config": s3_plugin_config,}elif self.backend == "memory":config["plugins"]["memfs"] = {diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex 8e3061fef..96c6bee23 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -1,10 +1,19 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0+from enum import Enumfrom typing import Optionalfrom pydantic import BaseModel, Field, model_validator+class DirectoryMarkerMode(str, Enum):+ """How S3 directory markers should be persisted."""++ NONE = "none"+ EMPTY = "empty"+ NONEMPTY = "nonempty"++class S3Config(BaseModel):"""Configuration for S3 backend."""@@ -46,6 +55,11 @@ class S3Config(BaseModel):description="true represent UsePathStyle for MinIO and some S3-compatible services; false represent VirtualHostStyle for TOS and some S3-compatible services.",)+ directory_marker_mode: DirectoryMarkerMode = Field(+ default=DirectoryMarkerMode.EMPTY,+ description="How to persist S3 directory markers: 'none' skips marker creation, 'empty' writes a zero-byte marker, and 'nonempty' writes a non-empty marker payload. Defaults to 'empty'.",+ )+model_config = {"extra": "forbid"}def validate_config(self):diff --git a/tests/misc/test_agfs_s3_config.py b/tests/misc/test_agfs_s3_config.pynew file mode 100644index 000000000..0d85f0b73--- /dev/null+++ b/tests/misc/test_agfs_s3_config.py@@ -0,0 +1,52 @@+#!/usr/bin/env python3+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++import pytest++from openviking.agfs_manager import AGFSManager+from openviking_cli.utils.config.agfs_config import AGFSConfig, DirectoryMarkerMode, S3Config+++def _build_s3_config(**overrides) -> S3Config:+ return S3Config(+ bucket="my-bucket",+ region="us-west-1",+ access_key="fake-access-key-for-testing",+ secret_key="fake-secret-key-for-testing-12345",+ endpoint="https://tos-cn-beijing.volces.com",+ **overrides,+ )+++def test_s3_directory_marker_mode_defaults_to_empty():+ default_s3 = S3Config()++ assert default_s3.directory_marker_mode is DirectoryMarkerMode.EMPTY+++def test_s3_rejects_removed_legacy_nonempty_directory_marker_alias():+ with pytest.raises(ValueError, match="Extra inputs are not permitted"):+ _build_s3_config(nonempty_directory_marker=True)+++ ("mode", "expected"),+ [+ (DirectoryMarkerMode.EMPTY, "empty"),+ (DirectoryMarkerMode.NONEMPTY, "nonempty"),+ (DirectoryMarkerMode.NONE, "none"),+ ],+)+def test_agfs_manager_emits_directory_marker_mode_only(tmp_path, mode, expected):+ config = AGFSConfig(+ path=str(tmp_path),+ backend="s3",+ s3=_build_s3_config(directory_marker_mode=mode),+ )++ manager = AGFSManager(config=config)+ agfs_config = manager._generate_config()+ s3_plugin_config = agfs_config["plugins"]["s3fs"]["config"]++ assert s3_plugin_config["directory_marker_mode"] == expecteddiff --git a/third_party/agfs/agfs-server/pkg/plugins/s3fs/client.go b/third_party/agfs/agfs-server/pkg/plugins/s3fs/client.goindex c7ba54bbc..3579d9f4c 100644--- a/third_party/agfs/agfs-server/pkg/plugins/s3fs/client.go+++ b/third_party/agfs/agfs-server/pkg/plugins/s3fs/client.go@@ -17,24 +17,63 @@ import (log "github.com/sirupsen/logrus")+type DirectoryMarkerMode string++const (+ DirectoryMarkerModeNone DirectoryMarkerMode = "none"+ DirectoryMarkerModeEmpty DirectoryMarkerMode = "empty"+ DirectoryMarkerModeNonEmpty DirectoryMarkerMode = "nonempty"+)+// S3Client wraps AWS S3 client with helper methodstype S3Client struct {- client *s3.Client- bucket string- region string // AWS region- prefix string // Optional prefix for all keys+ client *s3.Client+ bucket string+ region string // AWS region+ prefix string // Optional prefix for all keys+ directoryMarkerMode DirectoryMarkerMode}// S3Config holds S3 client configurationtype S3Config struct {- Region string- Bucket string- AccessKeyID string- SecretAccessKey string- Endpoint string // Optional custom endpoint (for S3-compatible services)- Prefix string // Optional prefix for all keys- DisableSSL bool // For testing with local S3- UsePathStyle bool // Whether to use path-style addressing (true) or virtual-host-style (false)+ Region string+ Bucket string+ AccessKeyID string+ SecretAccessKey string+ Endpoint string // Optional custom endpoint (for S3-compatible services)+ Prefix string // Optional prefix for all keys+ DisableSSL bool // For testing with local S3+ UsePathStyle bool // Whether to use path-style addressing (true) or virtual-host-style (false)+ DirectoryMarkerMode DirectoryMarkerMode+}++var nonEmptyDirectoryMarkerPayload = []byte{'\n'}++func normalizeDirectoryMarkerMode(mode DirectoryMarkerMode) DirectoryMarkerMode {+ if mode == "" {+ return DirectoryMarkerModeEmpty+ }+ return mode+}++func isValidDirectoryMarkerMode(mode DirectoryMarkerMode) bool {+ switch mode {+ case DirectoryMarkerModeNone, DirectoryMarkerModeEmpty, DirectoryMarkerModeNonEmpty:+ return true+ default:+ return false+ }+}++func directoryMarkerPayload(mode DirectoryMarkerMode) ([]byte, bool) {+ switch normalizeDirectoryMarkerMode(mode) {+ case DirectoryMarkerModeNone:+ return nil, false+ case DirectoryMarkerModeNonEmpty:+ return nonEmptyDirectoryMarkerPayload, true+ default:+ return []byte{}, true+ }}// NewS3Client creates a new S3 client@@ -70,7 +109,7 @@ func NewS3Client(cfg S3Config) (*S3Client, error) {o.BaseEndpoint = aws.String(cfg.Endpoint)// true represent UsePathStyle for MinIO and some S3-compatible services// false represent VirtualHostStyle for TOS and some S3-compatible services- o.UsePathStyle = cfg.UsePathStyle+ o.UsePathStyle = cfg.UsePathStyle})}@@ -90,13 +129,18 @@ func NewS3Client(cfg S3Config) (*S3Client, error) {prefix := strings.Trim(cfg.Prefix, "/")return &S3Client{- client: client,- bucket: cfg.Bucket,- region: cfg.Region,- prefix: prefix,+ client: client,+ bucket: cfg.Bucket,+ region: cfg.Region,+ prefix: prefix,+ directoryMarkerMode: normalizeDirectoryMarkerMode(cfg.DirectoryMarkerMode),}, nil}+func (c *S3Client) shouldEnforceParentDirectoryExistence() bool {+ return normalizeDirectoryMarkerMode(c.directoryMarkerMode) != DirectoryMarkerModeNone+}+// buildKey builds the full S3 key with prefixfunc (c *S3Client) buildKey(path string) string {// Normalize path@@ -308,9 +352,14 @@ func (c *S3Client) ListObjects(ctx context.Context, path string) ([]S3Object, erreturn objects, nil}-// CreateDirectory creates a directory marker in S3-// S3 doesn't have real directories, but we create empty objects ending with "/"+// CreateDirectory creates a directory marker in S3.+// S3 doesn't have real directories; they are represented by object keys ending with "/".func (c *S3Client) CreateDirectory(ctx context.Context, path string) error {+ payload, shouldCreate := directoryMarkerPayload(c.directoryMarkerMode)+ if !shouldCreate {+ return nil+ }+key := c.buildKey(path)if !strings.HasSuffix(key, "/") {key += "/"@@ -319,7 +368,7 @@ func (c *S3Client) CreateDirectory(ctx context.Context, path string) error {_, err := c.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(c.bucket),Key: aws.String(key),- Body: bytes.NewReader([]byte{}),+ Body: bytes.NewReader(payload),})if err != nil {return fmt.Errorf("failed to create directory %s: %w", key, err)diff --git a/third_party/agfs/agfs-server/pkg/plugins/s3fs/marker_mode_test.go b/third_party/agfs/agfs-server/pkg/plugins/s3fs/marker_mode_test.gonew file mode 100644index 000000000..b9061a175--- /dev/null+++ b/third_party/agfs/agfs-server/pkg/plugins/s3fs/marker_mode_test.go@@ -0,0 +1,65 @@+package s3fs++import "testing"++func TestNormalizeDirectoryMarkerModeConfigDefaultsToEmpty(t *testing.T) {+ mode, err := normalizeDirectoryMarkerModeConfig(map[string]interface{}{})+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if mode != DirectoryMarkerModeEmpty {+ t.Fatalf("expected default mode %q, got %q", DirectoryMarkerModeEmpty, mode)+ }+}++func TestNormalizeDirectoryMarkerModeConfigRejectsUnknownMode(t *testing.T) {+ _, err := normalizeDirectoryMarkerModeConfig(map[string]interface{}{+ "directory_marker_mode": "mystery",+ })+ if err == nil {+ t.Fatal("expected unknown mode error, got nil")+ }+}++func TestDirectoryMarkerPayload(t *testing.T) {+ payload, shouldCreate := directoryMarkerPayload(DirectoryMarkerModeNone)+ if shouldCreate {+ t.Fatal("expected none mode to skip marker creation")+ }+ if payload != nil {+ t.Fatalf("expected nil payload for none mode, got %v", payload)+ }++ payload, shouldCreate = directoryMarkerPayload(DirectoryMarkerModeEmpty)+ if !shouldCreate {+ t.Fatal("expected empty mode to create marker")+ }+ if len(payload) != 0 {+ t.Fatalf("expected empty marker payload, got %d bytes", len(payload))+ }++ payload, shouldCreate = directoryMarkerPayload(DirectoryMarkerModeNonEmpty)+ if !shouldCreate {+ t.Fatal("expected nonempty mode to create marker")+ }+ if len(payload) != 1 || payload[0] != '\n' {+ t.Fatalf("expected newline marker payload, got %v", payload)+ }+}++func TestShouldEnforceParentDirectoryExistence(t *testing.T) {+ client := &S3Client{directoryMarkerMode: DirectoryMarkerModeNone}+ if client.shouldEnforceParentDirectoryExistence() {+ t.Fatal("expected none mode to skip parent directory enforcement")+ }++ client.directoryMarkerMode = DirectoryMarkerModeEmpty+ if !client.shouldEnforceParentDirectoryExistence() {+ t.Fatal("expected empty mode to enforce parent directories")+ }++ client.directoryMarkerMode = DirectoryMarkerModeNonEmpty+ if !client.shouldEnforceParentDirectoryExistence() {+ t.Fatal("expected nonempty mode to enforce parent directories")+ }+}diff --git a/third_party/agfs/agfs-server/pkg/plugins/s3fs/s3fs.go b/third_party/agfs/agfs-server/pkg/plugins/s3fs/s3fs.goindex befe75995..16699168f 100644--- a/third_party/agfs/agfs-server/pkg/plugins/s3fs/s3fs.go+++ b/third_party/agfs/agfs-server/pkg/plugins/s3fs/s3fs.go@@ -87,7 +87,7 @@ func (fs *S3FS) Create(path string) error {// Check if parent directory existsparent := getParentPath(path)- if parent != "" {+ if parent != "" && fs.client.shouldEnforceParentDirectoryExistence() {dirExists, err := fs.client.DirectoryExists(ctx, parent)if err != nil {return fmt.Errorf("failed to check parent directory: %w", err)@@ -125,7 +125,7 @@ func (fs *S3FS) Mkdir(path string, perm uint32) error {// Check if parent directory existsparent := getParentPath(path)- if parent != "" {+ if parent != "" && fs.client.shouldEnforceParentDirectoryExistence() {dirExists, err := fs.client.DirectoryExists(ctx, parent)if err != nil {return fmt.Errorf("failed to check parent directory: %w", err)@@ -560,11 +560,36 @@ func (p *S3FSPlugin) Name() string {return PluginName}+func normalizeDirectoryMarkerModeConfig(cfg map[string]interface{}) (DirectoryMarkerMode, error) {+ rawMode, exists := cfg["directory_marker_mode"]+ if !exists {+ return DirectoryMarkerModeEmpty, nil+ }++ modeString, ok := rawMode.(string)+ if !ok {+ return "", fmt.Errorf("directory_marker_mode must be a string")+ }+ modeValue := strings.ToLower(strings.TrimSpace(modeString))+ mode := DirectoryMarkerMode(modeValue)+ if !isValidDirectoryMarkerMode(mode) {+ return "", fmt.Errorf(+ "directory_marker_mode must be one of: %s, %s, %s",+ DirectoryMarkerModeNone,+ DirectoryMarkerModeEmpty,+ DirectoryMarkerModeNonEmpty,+ )+ }++ return mode, nil+}+func (p *S3FSPlugin) Validate(cfg map[string]interface{}) error {// Check for unknown parametersallowedKeys := []string{"bucket", "region", "access_key_id", "secret_access_key", "endpoint", "prefix", "disable_ssl", "mount_path","cache_enabled", "cache_ttl", "stat_cache_ttl", "cache_max_size", "use_path_style",+ "directory_marker_mode",}if err := config.ValidateOnlyKnownKeys(cfg, allowedKeys); err != nil {return err@@ -581,6 +606,9 @@ func (p *S3FSPlugin) Validate(cfg map[string]interface{}) error {return err}}+ if err := config.ValidateStringType(cfg, "directory_marker_mode"); err != nil {+ return err+ }// Validate disable_ssl (optional boolean)if err := config.ValidateBoolType(cfg, "disable_ssl"); err != nil {@@ -597,22 +625,32 @@ func (p *S3FSPlugin) Validate(cfg map[string]interface{}) error {return err}+ if _, err := normalizeDirectoryMarkerModeConfig(cfg); err != nil {+ return err+ }+return nil}func (p *S3FSPlugin) Initialize(config map[string]interface{}) error {p.config = config+ directoryMarkerMode, err := normalizeDirectoryMarkerModeConfig(config)+ if err != nil {+ return err+ }+// Parse S3 configurationcfg := S3Config{- Region: getStringConfig(config, "region", "us-east-1"),- Bucket: getStringConfig(config, "bucket", ""),- AccessKeyID: getStringConfig(config, "access_key_id", ""),- SecretAccessKey: getStringConfig(config, "secret_access_key", ""),- Endpoint: getStringConfig(config, "endpoint", ""),- Prefix: getStringConfig(config, "prefix", ""),- DisableSSL: getBoolConfig(config, "disable_ssl", false),- UsePathStyle: getBoolConfig(config, "use_path_style", true),+ Region: getStringConfig(config, "region", "us-east-1"),+ Bucket: getStringConfig(config, "bucket", ""),+ AccessKeyID: getStringConfig(config, "access_key_id", ""),+ SecretAccessKey: getStringConfig(config, "secret_access_key", ""),+ Endpoint: getStringConfig(config, "endpoint", ""),+ Prefix: getStringConfig(config, "prefix", ""),+ DisableSSL: getBoolConfig(config, "disable_ssl", false),+ UsePathStyle: getBoolConfig(config, "use_path_style", true),+ DirectoryMarkerMode: directoryMarkerMode,}if cfg.Bucket == "" {@@ -634,7 +672,13 @@ func (p *S3FSPlugin) Initialize(config map[string]interface{}) error {}p.fs = fs- log.Infof("[s3fs] Initialized with bucket: %s, region: %s, cache: %v", cfg.Bucket, cfg.Region, cacheCfg.Enabled)+ log.Infof(+ "[s3fs] Initialized with bucket: %s, region: %s, cache: %v, directory_marker_mode: %s",+ cfg.Bucket,+ cfg.Region,+ cacheCfg.Enabled,+ cfg.DirectoryMarkerMode,+ )return nil}@@ -702,7 +746,14 @@ func (p *S3FSPlugin) GetConfigParams() []plugin.ConfigParameter {Type: "bool",Required: false,Default: "true",- Description: "Whether to use path-style addressing (true) or virtual-host-style (false). Defaults to false for TOS, true for other services.",+ Description: "Whether to use path-style addressing (true) or virtual-host-style (false). Set false for TOS and other VirtualHostStyle backends.",+ },+ {+ Name: "directory_marker_mode",+ Type: "string",+ Required: false,+ Default: "empty",+ Description: "How to persist directory markers: 'none' skips marker creation, 'empty' writes a zero-byte marker, and 'nonempty' writes a non-empty payload.",},{Name: "cache_enabled",@@ -764,6 +815,7 @@ CONFIGURATION:bucket = "my-bucket"access_key_id = "AKIAIOSFODNN7EXAMPLE"secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"+ directory_marker_mode = "empty"prefix = "agfs/" # Optional: all keys will be prefixed with thisS3-Compatible Service (MinIO, LocalStack):@@ -847,6 +899,9 @@ EXAMPLES:NOTES:- S3 doesn't have real directories; they are simulated with "/" in object keys+ - directory_marker_mode = "empty" is the default and preserves empty-directory semantics with zero-byte markers+ - Use directory_marker_mode = "nonempty" for backends such as TOS that reject zero-byte directory markers+ - Use directory_marker_mode = "none" for pure prefix-style behavior when you do not need persisted empty directories- Use --stream flag for large files to minimize memory usage (256KB chunks)- Permissions (chmod) are not supported by S3- Atomic operations are limited by S3's eventual consistency model