Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor: refactor agfs s3 backend config (#113)
chuanbao6665395854385b8
diff --git a/openviking/agfs_manager.py b/openviking/agfs_manager.pyindex 316bab430..4bd92e832 100644--- a/openviking/agfs_manager.py+++ b/openviking/agfs_manager.py@@ -78,13 +78,7 @@ class AGFSManager:self.port = config.portself.log_level = config.log_levelself.backend = config.backend- self.s3_bucket = config.s3.bucket- self.s3_region = config.s3.region- self.s3_access_key = config.s3.access_key- self.s3_secret_key = config.s3.secret_key- self.s3_endpoint = config.s3.endpoint- self.s3_prefix = config.s3.prefix- self.s3_use_ssl = config.s3.use_ssl+ self.s3_config = config.s3self.process: Optional[subprocess.Popen] = Noneself.config_file: Optional[Path] = None@@ -161,13 +155,13 @@ class AGFSManager:"enabled": True,"path": "/local","config": {- "bucket": self.s3_bucket,- "region": self.s3_region,- "access_key_id": self.s3_access_key,- "secret_access_key": self.s3_secret_key,- "endpoint": self.s3_endpoint,- "prefix": self.s3_prefix,- "disable_ssl": not self.s3_use_ssl,+ "bucket": self.s3_config.bucket,+ "region": self.s3_config.region,+ "access_key_id": self.s3_config.access_key,+ "secret_access_key": self.s3_config.secret_key,+ "endpoint": self.s3_config.endpoint,+ "prefix": self.s3_config.prefix,+ "disable_ssl": not self.s3_config.use_ssl,},}elif self.backend == "memory":diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.pyindex 00b5afc1c..d7f0ebddd 100644--- a/openviking/service/fs_service.py+++ b/openviking/service/fs_service.py@@ -8,7 +8,7 @@ Provides file system operations: ls, mkdir, rm, mv, tree, stat, read, abstract,from typing import Any, Dict, List, Optional-from openviking.exceptions import NotFoundError, NotInitializedError+from openviking.exceptions import NotInitializedErrorfrom openviking.storage.viking_fs import VikingFSfrom openviking.utils import get_loggerdiff --git a/openviking/service/session_service.py b/openviking/service/session_service.pyindex f3738ff6d..8dedd73cf 100644--- a/openviking/service/session_service.py+++ b/openviking/service/session_service.py@@ -9,7 +9,6 @@ Provides session management operations: session, sessions, add_message, compressfrom typing import Any, Dict, List, Optionalfrom openviking.exceptions import NotFoundError, NotInitializedError-from openviking.message import Partfrom openviking.session import Sessionfrom openviking.session.compressor import SessionCompressorfrom openviking.storage import VikingDBManagerdiff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 5c7a1b45a..db493abff 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -6,13 +6,6 @@ import asynciofrom typing import Any, Dict, List, Optional, Tuplefrom openviking.core.context import Context, ResourceContentType, Vectorize-from openviking.prompts import render_prompt-from openviking.storage.queuefs.named_queue import DequeueHandlerBase-from openviking.storage.queuefs.semantic_msg import SemanticMsg-from openviking.storage.viking_fs import get_viking_fs-from openviking.utils import VikingURI-from openviking.utils.config import get_openviking_config-from openviking.utils.logger import get_loggerfrom openviking.parse.parsers.constants import (CODE_EXTENSIONS,DOCUMENTATION_EXTENSIONS,@@ -20,6 +13,13 @@ from openviking.parse.parsers.constants import (FILE_TYPE_DOCUMENTATION,FILE_TYPE_OTHER,)+from openviking.prompts import render_prompt+from openviking.storage.queuefs.named_queue import DequeueHandlerBase+from openviking.storage.queuefs.semantic_msg import SemanticMsg+from openviking.storage.viking_fs import get_viking_fs+from openviking.utils import VikingURI+from openviking.utils.config import get_openviking_config+from openviking.utils.logger import get_loggerlogger = get_logger(__name__)diff --git a/openviking/utils/config/agfs_config.py b/openviking/utils/config/agfs_config.pyindex f7b963eb5..7845941ff 100644--- a/openviking/utils/config/agfs_config.py+++ b/openviking/utils/config/agfs_config.py@@ -82,6 +82,11 @@ class AGFSConfig(BaseModel):retry_times: int = Field(default=3, description="AGFS retry times on failure")+ use_ssl: bool = Field(+ default=True,+ description="Enable/Disable SSL (HTTPS) for AGFS service. Set to False for local testing without HTTPS.",+ )+# S3 backend configuration# These settings are used when backend is set to 's3'.# AGFS will act as a gateway to the specified S3 bucket.diff --git a/tests/agfs/__init__.py b/tests/agfs/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/agfs/conftest.py b/tests/agfs/conftest.pynew file mode 100644index 000000000..09467c001--- /dev/null+++ b/tests/agfs/conftest.py@@ -0,0 +1,16 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++import shutil+from pathlib import Path++import pytest++[email protected](scope="session")+def agfs_test_root():+ """Root directory for AGFS tests."""+ path = Path("/tmp/openviking_agfs_test")+ path.mkdir(parents=True, exist_ok=True)+ yield path+ shutil.rmtree(path, ignore_errors=True)diff --git a/tests/agfs/test_fs_local.py b/tests/agfs/test_fs_local.pynew file mode 100644index 000000000..ea728da35--- /dev/null+++ b/tests/agfs/test_fs_local.py@@ -0,0 +1,144 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++"""AGFS Local Backend Tests for VikingFS interface"""++import json+import os+import uuid+from pathlib import Path++import pytest++from openviking.agfs_manager import AGFSManager+from openviking.storage.viking_fs import init_viking_fs+from openviking.utils.config.agfs_config import AGFSConfig++# 1. Config loading logic+# Try to load from environment variable or default ov.conf+CONFIG_FILE = os.getenv("OPENVIKING_CONFIG_FILE")+if not CONFIG_FILE:+ # Try default ov.conf in tests/agfs+ default_conf = Path(__file__).parent / "ov.conf"+ if default_conf.exists():+ CONFIG_FILE = str(default_conf)++def load_agfs_config() -> AGFSConfig:+ """Load only AGFS configuration from the config file."""+ if not CONFIG_FILE or not Path(CONFIG_FILE).exists():+ return None++ try:+ with open(CONFIG_FILE, "r") as f:+ full_config = json.load(f)++ # Support both 'storage.agfs' and top-level 'agfs' structures+ agfs_data = full_config.get("storage", {}).get("agfs") or full_config.get("agfs")+ if not agfs_data:+ return None++ return AGFSConfig(**agfs_data)+ except Exception:+ return None++AGFS_CONF = load_agfs_config()++# 2. Skip tests if no local config found or backend is not local+pytestmark = pytest.mark.skipif(+ AGFS_CONF is None or AGFS_CONF.backend != "local",+ reason="AGFS local configuration not found in ov.conf or backend is not local"+)+[email protected](scope="module")+async def viking_fs_instance():+ """Initialize AGFS Manager and VikingFS singleton."""+ manager = AGFSManager(config=AGFS_CONF)+ manager.start()++ # Initialize VikingFS with agfs_url+ vfs = init_viking_fs(agfs_url=AGFS_CONF.url, timeout=AGFS_CONF.timeout)++ yield vfs++ # AGFSManager.stop is synchronous+ manager.stop()++class TestVikingFSLocal:+ """Test VikingFS operations with local backend."""++ async def test_file_operations(self, viking_fs_instance):+ """Test VikingFS file operations: read, write, ls, stat."""+ vfs = viking_fs_instance+ test_filename = f"local_file_{uuid.uuid4().hex}.txt"+ test_content = "Hello VikingFS Local! " + uuid.uuid4().hex+ test_uri = f"viking://{test_filename}"++ # 1. Write file+ await vfs.write(test_uri, test_content)++ # 2. Stat file+ stat_info = await vfs.stat(test_uri)+ assert stat_info["name"] == test_filename+ assert not stat_info["isDir"]++ # 3. List directory+ entries = await vfs.ls("viking://")+ assert any(e["name"] == test_filename for e in entries)++ # 4. Read file+ read_data = await vfs.read(test_uri)+ assert read_data.decode("utf-8") == test_content++ # Cleanup+ await vfs.rm(test_uri)++ async def test_directory_operations(self, viking_fs_instance):+ """Test VikingFS directory operations: mkdir, rm, ls, stat."""+ vfs = viking_fs_instance+ test_dir = f"local_dir_{uuid.uuid4().hex}"+ test_dir_uri = f"viking://{test_dir}/"++ # 1. Create directory+ await vfs.mkdir(test_dir_uri)++ # 2. Stat directory+ stat_info = await vfs.stat(test_dir_uri)+ assert stat_info["name"] == test_dir+ assert stat_info["isDir"]++ # 3. List root to see directory+ root_entries = await vfs.ls("viking://")+ assert any(e["name"] == test_dir and e["isDir"] for e in root_entries)++ # 4. Write a file inside+ file_uri = f"{test_dir_uri}inner.txt"+ await vfs.write(file_uri, "inner content")++ # 5. List subdirectory+ sub_entries = await vfs.ls(test_dir_uri)+ assert any(e["name"] == "inner.txt" for e in sub_entries)++ # 6. Delete directory (recursive)+ await vfs.rm(test_dir_uri, recursive=True)++ # 7. Verify deletion+ root_entries = await vfs.ls("viking://")+ assert not any(e["name"] == test_dir for e in root_entries)++ async def test_ensure_dirs(self, viking_fs_instance):+ """Test VikingFS ensure_dirs."""+ vfs = viking_fs_instance+ base_dir = f"local_tree_test_{uuid.uuid4().hex}"+ sub_dir = f"viking://{base_dir}/a/b/"+ file_uri = f"{sub_dir}leaf.txt"++ await vfs.mkdir(sub_dir)+ await vfs.write(file_uri, "leaf content")++ # VikingFS.tree provides recursive listing+ entries = await vfs.tree(f"viking://{base_dir}/")+ assert any("leaf.txt" in e["uri"] for e in entries)++ # Cleanup+ await vfs.rm(f"viking://{base_dir}/", recursive=True)diff --git a/tests/agfs/test_fs_s3.py b/tests/agfs/test_fs_s3.pynew file mode 100644index 000000000..e0884abb3--- /dev/null+++ b/tests/agfs/test_fs_s3.py@@ -0,0 +1,175 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++"""AGFS S3 Backend Tests for VikingFS interface with S3 client verification"""++import json+import os+import uuid+from pathlib import Path++import botocore+import pytest++from openviking.agfs_manager import AGFSManager+from openviking.storage.viking_fs import VikingFS, init_viking_fs+from openviking.utils.config.agfs_config import AGFSConfig++# 1. Simplified Config loading logic+# Only extract the AGFS part for focused testing+CONFIG_FILE = os.getenv("OPENVIKING_CONFIG_FILE")+if not CONFIG_FILE:+ # Try default ov.conf in tests/agfs+ default_conf = Path(__file__).parent / "ov.conf"+ if default_conf.exists():+ CONFIG_FILE = str(default_conf)++def load_agfs_config() -> AGFSConfig:+ """Load only AGFS configuration from the config file."""+ if not CONFIG_FILE or not Path(CONFIG_FILE).exists():+ return None++ try:+ with open(CONFIG_FILE, "r") as f:+ full_config = json.load(f)++ # Support both 'storage.agfs' and top-level 'agfs' structures+ agfs_data = full_config.get("storage", {}).get("agfs") or full_config.get("agfs")+ if not agfs_data:+ return None++ return AGFSConfig(**agfs_data)+ except Exception:+ return None++AGFS_CONF = load_agfs_config()++# 2. Skip tests if no S3 config found or backend is not S3+pytestmark = pytest.mark.skipif(+ AGFS_CONF is None or AGFS_CONF.backend != "s3",+ reason="AGFS S3 configuration not found in ov.conf"+)+[email protected](scope="module")+def s3_client():+ """Boto3 client for S3 verification."""+ import boto3++ s3_conf = AGFS_CONF.s3+ return boto3.client(+ 's3',+ aws_access_key_id=s3_conf.access_key,+ aws_secret_access_key=s3_conf.secret_key,+ region_name=s3_conf.region,+ endpoint_url=s3_conf.endpoint,+ use_ssl=s3_conf.use_ssl+ )+[email protected](scope="module")+async def viking_fs_instance():+ """Initialize AGFS Manager and VikingFS singleton."""+ manager = AGFSManager(config=AGFS_CONF)+ manager.start()++ # Initialize VikingFS with agfs_url (only basic IO needed)+ vfs = init_viking_fs(agfs_url=AGFS_CONF.url, timeout=AGFS_CONF.timeout)++ yield vfs++ # AGFSManager.stop is synchronous+ manager.stop()++class TestVikingFSS3:+ """Test VikingFS operations with S3 backend and verify via S3 client."""++ async def test_file_operations(self, viking_fs_instance: "VikingFS", s3_client):+ """Test VikingFS file operations and verify with S3 client."""+ vfs = viking_fs_instance+ s3_conf = AGFS_CONF.s3+ bucket = s3_conf.bucket+ prefix = s3_conf.prefix or ""++ test_filename = f"verify_{uuid.uuid4().hex}.txt"+ test_content = "Hello VikingFS S3! " + uuid.uuid4().hex+ test_uri = f"viking://{test_filename}"++ # 1. Write via VikingFS+ await vfs.write(test_uri, test_content)++ # 2. Verify existence and content via S3 client+ s3_key = f"{prefix}{test_filename}"+ response = s3_client.get_object(Bucket=bucket, Key=s3_key)+ s3_content = response['Body'].read().decode('utf-8')+ assert s3_content == test_content++ # 3. Stat via VikingFS+ stat_info = await vfs.stat(test_uri)+ assert stat_info["name"] == test_filename+ assert not stat_info["isDir"]++ # 4. List via VikingFS+ entries = await vfs.ls("viking://")+ assert any(e["name"] == test_filename for e in entries)++ # 5. Read back via VikingFS+ read_data = await vfs.read(test_uri)+ assert read_data.decode("utf-8") == test_content++ # 6. Cleanup via VikingFS+ await vfs.rm(test_uri)++ # 7. Verify deletion via S3 client+ with pytest.raises(botocore.exceptions.ClientError) as excinfo:+ s3_client.get_object(Bucket=bucket, Key=s3_key)+ assert excinfo.value.response['Error']['Code'] in ['NoSuchKey', '404']++ async def test_directory_operations(self, viking_fs_instance, s3_client):+ """Test VikingFS directory operations and verify with S3 client."""+ vfs = viking_fs_instance+ s3_conf = AGFS_CONF.s3+ bucket = s3_conf.bucket+ prefix = s3_conf.prefix or ""++ test_dir = f"test_dir_{uuid.uuid4().hex}"+ test_dir_uri = f"viking://{test_dir}/"++ # 1. Create directory via VikingFS+ await vfs.mkdir(test_dir_uri)++ # 2. Verify via S3 client by writing a file inside+ file_uri = f"{test_dir_uri}inner.txt"+ file_content = "inner content"+ await vfs.write(file_uri, file_content)++ s3_key = f"{prefix}{test_dir}/inner.txt"+ response = s3_client.get_object(Bucket=bucket, Key=s3_key)+ assert response['Body'].read().decode('utf-8') == file_content++ # 3. List via VikingFS+ root_entries = await vfs.ls("viking://")+ assert any(e["name"] == test_dir and e["isDir"] for e in root_entries)++ # 4. Delete directory recursively via VikingFS+ await vfs.rm(test_dir_uri, recursive=True)++ # 5. Verify deletion via S3 client+ with pytest.raises(botocore.exceptions.ClientError):+ s3_client.get_object(Bucket=bucket, Key=s3_key)++ async def test_ensure_dirs(self, viking_fs_instance: "VikingFS"):+ """Test VikingFS ensure_dirs."""+ vfs = viking_fs_instance+ base_dir = f"tree_test_{uuid.uuid4().hex}"+ sub_dir = f"viking://{base_dir}/a/b/"+ file_uri = f"{sub_dir}leaf.txt"++ await vfs.mkdir(sub_dir)+ await vfs.write(file_uri, "leaf content")++ # VikingFS.tree provides recursive listing+ entries = await vfs.tree(f"viking://{base_dir}/")+ assert any("leaf.txt" in e["uri"] for e in entries)++ # Cleanup+ await vfs.rm(f"viking://{base_dir}/", recursive=True)