Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor: refactor agfs s3 backend config (#113)

chuanbao6665395854385b8
diff --git a/openviking/agfs_manager.py b/openviking/agfs_manager.py
index 316bab430..4bd92e832 100644
--- a/openviking/agfs_manager.py
+++ b/openviking/agfs_manager.py
@@ -78,13 +78,7 @@ class AGFSManager:
self.port = config.port
self.log_level = config.log_level
self.backend = config.backend
- self.s3_bucket = config.s3.bucket
- self.s3_region = config.s3.region
- self.s3_access_key = config.s3.access_key
- self.s3_secret_key = config.s3.secret_key
- self.s3_endpoint = config.s3.endpoint
- self.s3_prefix = config.s3.prefix
- self.s3_use_ssl = config.s3.use_ssl
+ self.s3_config = config.s3
self.process: Optional[subprocess.Popen] = None
self.config_file: Optional[Path] = None
@@ -161,13 +155,13 @@ class AGFSManager:
"enabled": True,
"path": "/local",
"config": {
- "bucket": self.s3_bucket,
- "region": self.s3_region,
- "access_key_id": self.s3_access_key,
- "secret_access_key": self.s3_secret_key,
- "endpoint": self.s3_endpoint,
- "prefix": self.s3_prefix,
- "disable_ssl": not self.s3_use_ssl,
+ "bucket": self.s3_config.bucket,
+ "region": self.s3_config.region,
+ "access_key_id": self.s3_config.access_key,
+ "secret_access_key": self.s3_config.secret_key,
+ "endpoint": self.s3_config.endpoint,
+ "prefix": self.s3_config.prefix,
+ "disable_ssl": not self.s3_config.use_ssl,
},
}
elif self.backend == "memory":
diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py
index 00b5afc1c..d7f0ebddd 100644
--- a/openviking/service/fs_service.py
+++ b/openviking/service/fs_service.py
@@ -8,7 +8,7 @@ Provides file system operations: ls, mkdir, rm, mv, tree, stat, read, abstract,
from typing import Any, Dict, List, Optional
-from openviking.exceptions import NotFoundError, NotInitializedError
+from openviking.exceptions import NotInitializedError
from openviking.storage.viking_fs import VikingFS
from openviking.utils import get_logger
diff --git a/openviking/service/session_service.py b/openviking/service/session_service.py
index f3738ff6d..8dedd73cf 100644
--- a/openviking/service/session_service.py
+++ b/openviking/service/session_service.py
@@ -9,7 +9,6 @@ Provides session management operations: session, sessions, add_message, compress
from typing import Any, Dict, List, Optional
from openviking.exceptions import NotFoundError, NotInitializedError
-from openviking.message import Part
from openviking.session import Session
from openviking.session.compressor import SessionCompressor
from openviking.storage import VikingDBManager
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 5c7a1b45a..db493abff 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -6,13 +6,6 @@ import asyncio
from typing import Any, Dict, List, Optional, Tuple
from openviking.core.context import Context, ResourceContentType, Vectorize
-from openviking.prompts import render_prompt
-from openviking.storage.queuefs.named_queue import DequeueHandlerBase
-from openviking.storage.queuefs.semantic_msg import SemanticMsg
-from openviking.storage.viking_fs import get_viking_fs
-from openviking.utils import VikingURI
-from openviking.utils.config import get_openviking_config
-from openviking.utils.logger import get_logger
from openviking.parse.parsers.constants import (
CODE_EXTENSIONS,
DOCUMENTATION_EXTENSIONS,
@@ -20,6 +13,13 @@ from openviking.parse.parsers.constants import (
FILE_TYPE_DOCUMENTATION,
FILE_TYPE_OTHER,
)
+from openviking.prompts import render_prompt
+from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from openviking.storage.queuefs.semantic_msg import SemanticMsg
+from openviking.storage.viking_fs import get_viking_fs
+from openviking.utils import VikingURI
+from openviking.utils.config import get_openviking_config
+from openviking.utils.logger import get_logger
logger = get_logger(__name__)
diff --git a/openviking/utils/config/agfs_config.py b/openviking/utils/config/agfs_config.py
index f7b963eb5..7845941ff 100644
--- a/openviking/utils/config/agfs_config.py
+++ b/openviking/utils/config/agfs_config.py
@@ -82,6 +82,11 @@ class AGFSConfig(BaseModel):
retry_times: int = Field(default=3, description="AGFS retry times on failure")
+ use_ssl: bool = Field(
+ default=True,
+ description="Enable/Disable SSL (HTTPS) for AGFS service. Set to False for local testing without HTTPS.",
+ )
+
# S3 backend configuration
# These settings are used when backend is set to 's3'.
# AGFS will act as a gateway to the specified S3 bucket.
diff --git a/tests/agfs/__init__.py b/tests/agfs/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/agfs/conftest.py b/tests/agfs/conftest.py
new file mode 100644
index 000000000..09467c001
--- /dev/null
+++ b/tests/agfs/conftest.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+
[email protected](scope="session")
+def agfs_test_root():
+ """Root directory for AGFS tests."""
+ path = Path("/tmp/openviking_agfs_test")
+ path.mkdir(parents=True, exist_ok=True)
+ yield path
+ shutil.rmtree(path, ignore_errors=True)
diff --git a/tests/agfs/test_fs_local.py b/tests/agfs/test_fs_local.py
new file mode 100644
index 000000000..ea728da35
--- /dev/null
+++ b/tests/agfs/test_fs_local.py
@@ -0,0 +1,144 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+"""AGFS Local Backend Tests for VikingFS interface"""
+
+import json
+import os
+import uuid
+from pathlib import Path
+
+import pytest
+
+from openviking.agfs_manager import AGFSManager
+from openviking.storage.viking_fs import init_viking_fs
+from openviking.utils.config.agfs_config import AGFSConfig
+
+# 1. Config loading logic
+# Try to load from environment variable or default ov.conf
+CONFIG_FILE = os.getenv("OPENVIKING_CONFIG_FILE")
+if not CONFIG_FILE:
+ # Try default ov.conf in tests/agfs
+ default_conf = Path(__file__).parent / "ov.conf"
+ if default_conf.exists():
+ CONFIG_FILE = str(default_conf)
+
+def load_agfs_config() -> AGFSConfig:
+ """Load only AGFS configuration from the config file."""
+ if not CONFIG_FILE or not Path(CONFIG_FILE).exists():
+ return None
+
+ try:
+ with open(CONFIG_FILE, "r") as f:
+ full_config = json.load(f)
+
+ # Support both 'storage.agfs' and top-level 'agfs' structures
+ agfs_data = full_config.get("storage", {}).get("agfs") or full_config.get("agfs")
+ if not agfs_data:
+ return None
+
+ return AGFSConfig(**agfs_data)
+ except Exception:
+ return None
+
+AGFS_CONF = load_agfs_config()
+
+# 2. Skip tests if no local config found or backend is not local
+pytestmark = pytest.mark.skipif(
+ AGFS_CONF is None or AGFS_CONF.backend != "local",
+ reason="AGFS local configuration not found in ov.conf or backend is not local"
+)
+
[email protected](scope="module")
+async def viking_fs_instance():
+ """Initialize AGFS Manager and VikingFS singleton."""
+ manager = AGFSManager(config=AGFS_CONF)
+ manager.start()
+
+ # Initialize VikingFS with agfs_url
+ vfs = init_viking_fs(agfs_url=AGFS_CONF.url, timeout=AGFS_CONF.timeout)
+
+ yield vfs
+
+ # AGFSManager.stop is synchronous
+ manager.stop()
+
+class TestVikingFSLocal:
+ """Test VikingFS operations with local backend."""
+
+ async def test_file_operations(self, viking_fs_instance):
+ """Test VikingFS file operations: read, write, ls, stat."""
+ vfs = viking_fs_instance
+ test_filename = f"local_file_{uuid.uuid4().hex}.txt"
+ test_content = "Hello VikingFS Local! " + uuid.uuid4().hex
+ test_uri = f"viking://{test_filename}"
+
+ # 1. Write file
+ await vfs.write(test_uri, test_content)
+
+ # 2. Stat file
+ stat_info = await vfs.stat(test_uri)
+ assert stat_info["name"] == test_filename
+ assert not stat_info["isDir"]
+
+ # 3. List directory
+ entries = await vfs.ls("viking://")
+ assert any(e["name"] == test_filename for e in entries)
+
+ # 4. Read file
+ read_data = await vfs.read(test_uri)
+ assert read_data.decode("utf-8") == test_content
+
+ # Cleanup
+ await vfs.rm(test_uri)
+
+ async def test_directory_operations(self, viking_fs_instance):
+ """Test VikingFS directory operations: mkdir, rm, ls, stat."""
+ vfs = viking_fs_instance
+ test_dir = f"local_dir_{uuid.uuid4().hex}"
+ test_dir_uri = f"viking://{test_dir}/"
+
+ # 1. Create directory
+ await vfs.mkdir(test_dir_uri)
+
+ # 2. Stat directory
+ stat_info = await vfs.stat(test_dir_uri)
+ assert stat_info["name"] == test_dir
+ assert stat_info["isDir"]
+
+ # 3. List root to see directory
+ root_entries = await vfs.ls("viking://")
+ assert any(e["name"] == test_dir and e["isDir"] for e in root_entries)
+
+ # 4. Write a file inside
+ file_uri = f"{test_dir_uri}inner.txt"
+ await vfs.write(file_uri, "inner content")
+
+ # 5. List subdirectory
+ sub_entries = await vfs.ls(test_dir_uri)
+ assert any(e["name"] == "inner.txt" for e in sub_entries)
+
+ # 6. Delete directory (recursive)
+ await vfs.rm(test_dir_uri, recursive=True)
+
+ # 7. Verify deletion
+ root_entries = await vfs.ls("viking://")
+ assert not any(e["name"] == test_dir for e in root_entries)
+
+ async def test_ensure_dirs(self, viking_fs_instance):
+ """Test VikingFS ensure_dirs."""
+ vfs = viking_fs_instance
+ base_dir = f"local_tree_test_{uuid.uuid4().hex}"
+ sub_dir = f"viking://{base_dir}/a/b/"
+ file_uri = f"{sub_dir}leaf.txt"
+
+ await vfs.mkdir(sub_dir)
+ await vfs.write(file_uri, "leaf content")
+
+ # VikingFS.tree provides recursive listing
+ entries = await vfs.tree(f"viking://{base_dir}/")
+ assert any("leaf.txt" in e["uri"] for e in entries)
+
+ # Cleanup
+ await vfs.rm(f"viking://{base_dir}/", recursive=True)
diff --git a/tests/agfs/test_fs_s3.py b/tests/agfs/test_fs_s3.py
new file mode 100644
index 000000000..e0884abb3
--- /dev/null
+++ b/tests/agfs/test_fs_s3.py
@@ -0,0 +1,175 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+"""AGFS S3 Backend Tests for VikingFS interface with S3 client verification"""
+
+import json
+import os
+import uuid
+from pathlib import Path
+
+import botocore
+import pytest
+
+from openviking.agfs_manager import AGFSManager
+from openviking.storage.viking_fs import VikingFS, init_viking_fs
+from openviking.utils.config.agfs_config import AGFSConfig
+
+# 1. Simplified Config loading logic
+# Only extract the AGFS part for focused testing
+CONFIG_FILE = os.getenv("OPENVIKING_CONFIG_FILE")
+if not CONFIG_FILE:
+ # Try default ov.conf in tests/agfs
+ default_conf = Path(__file__).parent / "ov.conf"
+ if default_conf.exists():
+ CONFIG_FILE = str(default_conf)
+
+def load_agfs_config() -> AGFSConfig:
+ """Load only AGFS configuration from the config file."""
+ if not CONFIG_FILE or not Path(CONFIG_FILE).exists():
+ return None
+
+ try:
+ with open(CONFIG_FILE, "r") as f:
+ full_config = json.load(f)
+
+ # Support both 'storage.agfs' and top-level 'agfs' structures
+ agfs_data = full_config.get("storage", {}).get("agfs") or full_config.get("agfs")
+ if not agfs_data:
+ return None
+
+ return AGFSConfig(**agfs_data)
+ except Exception:
+ return None
+
+AGFS_CONF = load_agfs_config()
+
+# 2. Skip tests if no S3 config found or backend is not S3
+pytestmark = pytest.mark.skipif(
+ AGFS_CONF is None or AGFS_CONF.backend != "s3",
+ reason="AGFS S3 configuration not found in ov.conf"
+)
+
[email protected](scope="module")
+def s3_client():
+ """Boto3 client for S3 verification."""
+ import boto3
+
+ s3_conf = AGFS_CONF.s3
+ return boto3.client(
+ 's3',
+ aws_access_key_id=s3_conf.access_key,
+ aws_secret_access_key=s3_conf.secret_key,
+ region_name=s3_conf.region,
+ endpoint_url=s3_conf.endpoint,
+ use_ssl=s3_conf.use_ssl
+ )
+
[email protected](scope="module")
+async def viking_fs_instance():
+ """Initialize AGFS Manager and VikingFS singleton."""
+ manager = AGFSManager(config=AGFS_CONF)
+ manager.start()
+
+ # Initialize VikingFS with agfs_url (only basic IO needed)
+ vfs = init_viking_fs(agfs_url=AGFS_CONF.url, timeout=AGFS_CONF.timeout)
+
+ yield vfs
+
+ # AGFSManager.stop is synchronous
+ manager.stop()
+
+class TestVikingFSS3:
+ """Test VikingFS operations with S3 backend and verify via S3 client."""
+
+ async def test_file_operations(self, viking_fs_instance: "VikingFS", s3_client):
+ """Test VikingFS file operations and verify with S3 client."""
+ vfs = viking_fs_instance
+ s3_conf = AGFS_CONF.s3
+ bucket = s3_conf.bucket
+ prefix = s3_conf.prefix or ""
+
+ test_filename = f"verify_{uuid.uuid4().hex}.txt"
+ test_content = "Hello VikingFS S3! " + uuid.uuid4().hex
+ test_uri = f"viking://{test_filename}"
+
+ # 1. Write via VikingFS
+ await vfs.write(test_uri, test_content)
+
+ # 2. Verify existence and content via S3 client
+ s3_key = f"{prefix}{test_filename}"
+ response = s3_client.get_object(Bucket=bucket, Key=s3_key)
+ s3_content = response['Body'].read().decode('utf-8')
+ assert s3_content == test_content
+
+ # 3. Stat via VikingFS
+ stat_info = await vfs.stat(test_uri)
+ assert stat_info["name"] == test_filename
+ assert not stat_info["isDir"]
+
+ # 4. List via VikingFS
+ entries = await vfs.ls("viking://")
+ assert any(e["name"] == test_filename for e in entries)
+
+ # 5. Read back via VikingFS
+ read_data = await vfs.read(test_uri)
+ assert read_data.decode("utf-8") == test_content
+
+ # 6. Cleanup via VikingFS
+ await vfs.rm(test_uri)
+
+ # 7. Verify deletion via S3 client
+ with pytest.raises(botocore.exceptions.ClientError) as excinfo:
+ s3_client.get_object(Bucket=bucket, Key=s3_key)
+ assert excinfo.value.response['Error']['Code'] in ['NoSuchKey', '404']
+
+ async def test_directory_operations(self, viking_fs_instance, s3_client):
+ """Test VikingFS directory operations and verify with S3 client."""
+ vfs = viking_fs_instance
+ s3_conf = AGFS_CONF.s3
+ bucket = s3_conf.bucket
+ prefix = s3_conf.prefix or ""
+
+ test_dir = f"test_dir_{uuid.uuid4().hex}"
+ test_dir_uri = f"viking://{test_dir}/"
+
+ # 1. Create directory via VikingFS
+ await vfs.mkdir(test_dir_uri)
+
+ # 2. Verify via S3 client by writing a file inside
+ file_uri = f"{test_dir_uri}inner.txt"
+ file_content = "inner content"
+ await vfs.write(file_uri, file_content)
+
+ s3_key = f"{prefix}{test_dir}/inner.txt"
+ response = s3_client.get_object(Bucket=bucket, Key=s3_key)
+ assert response['Body'].read().decode('utf-8') == file_content
+
+ # 3. List via VikingFS
+ root_entries = await vfs.ls("viking://")
+ assert any(e["name"] == test_dir and e["isDir"] for e in root_entries)
+
+ # 4. Delete directory recursively via VikingFS
+ await vfs.rm(test_dir_uri, recursive=True)
+
+ # 5. Verify deletion via S3 client
+ with pytest.raises(botocore.exceptions.ClientError):
+ s3_client.get_object(Bucket=bucket, Key=s3_key)
+
+ async def test_ensure_dirs(self, viking_fs_instance: "VikingFS"):
+ """Test VikingFS ensure_dirs."""
+ vfs = viking_fs_instance
+ base_dir = f"tree_test_{uuid.uuid4().hex}"
+ sub_dir = f"viking://{base_dir}/a/b/"
+ file_uri = f"{sub_dir}leaf.txt"
+
+ await vfs.mkdir(sub_dir)
+ await vfs.write(file_uri, "leaf content")
+
+ # VikingFS.tree provides recursive listing
+ entries = await vfs.tree(f"viking://{base_dir}/")
+ assert any("leaf.txt" in e["uri"] for e in entries)
+
+ # Cleanup
+ await vfs.rm(f"viking://{base_dir}/", recursive=True)