Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
from __future__ import annotations
import math
import os
from enum import Enum
from typing import Any, List, Literal, Optional
from urllib.parse import urlparse
from pydantic import BaseModel, Field, model_validator
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
class DirectoryMarkerMode(str, Enum):
"""How S3 directory markers should be persisted."""
NONE = "none"
EMPTY = "empty"
NONEMPTY = "nonempty"
class S3Config(BaseModel):
"""Configuration for S3 backend."""
bucket: Optional[str] = Field(default=None, description="S3 bucket name")
region: Optional[str] = Field(
default=None,
description="AWS region where the bucket is located (e.g., us-east-1, cn-beijing)",
)
access_key: Optional[str] = Field(
default=None,
description="S3 access key ID. If not provided, RAGFS may attempt to use environment variables or IAM roles.",
)
secret_key: Optional[str] = Field(
default=None,
description="S3 secret access key corresponding to the access key ID.",
)
endpoint: Optional[str] = Field(
default=None,
description="Custom S3 endpoint URL. Required for S3-compatible services like MinIO or LocalStack. "
"Leave empty for standard AWS S3.",
)
prefix: Optional[str] = Field(
default="",
description="Optional key prefix for namespace isolation. All objects will be stored under this prefix.",
)
use_ssl: bool = Field(
default=True,
description="Enable/Disable SSL (HTTPS) for S3 connections. Set to False for local testing without HTTPS.",
)
use_path_style: bool = Field(
default=True,
description="true represent UsePathStyle for MinIO and some S3-compatible services; false represent VirtualHostStyle for TOS and some S3-compatible services.",
)
directory_marker_mode: DirectoryMarkerMode = Field(
default=DirectoryMarkerMode.EMPTY,
description="How to persist S3 directory markers: 'none' skips marker creation, 'empty' writes a zero-byte marker, and 'nonempty' writes a non-empty marker payload. Defaults to 'empty'.",
)
disable_batch_delete: bool = Field(
default=False,
description="Disable batch delete (DeleteObjects) and use sequential single-object deletes instead. "
"Required for S3-compatible services like Alibaba Cloud OSS that require a Content-MD5 header "
"for DeleteObjects but AWS SDK v2 does not send it by default. Defaults to False.",
)
normalize_encoding_chars: str = Field(
default="?#%+@",
description="Characters to escape in S3 object keys as !HH hexadecimal bytes. "
"Set to an empty string to disable key normalization. Defaults to ?#%+@.",
)
auto_detect_content_type: bool = Field(
default=False,
description="Automatically infer S3 object Content-Type from the object key filename extension "
"during uploads. Disabled by default for backward compatibility.",
)
def validate_config(self):
"""Validate S3 configuration completeness"""
missing = []
if not self.bucket:
missing.append("bucket")
if not self.endpoint:
missing.append("endpoint")
if not self.region:
missing.append("region")
if not self.access_key:
missing.append("access_key")
if not self.secret_key:
missing.append("secret_key")
if missing:
raise ValueError(f"S3 backend requires the following fields: {', '.join(missing)}")
return self
class QueueFSConfig(BaseModel):
"""Configuration for QueueFS backend."""
mode: str = Field(
default="shared",
description="QueueFS namespace mode: 'shared' | 'worker'",
)
backend: str = Field(
default="sqlite",
description="QueueFS backend: 'memory' | 'sqlite' | 'sqlite3' | 'cache'",
)
db_path: Optional[str] = Field(
default=None,
description="SQLite database path for QueueFS when backend is 'sqlite' or 'sqlite3'.",
)
recover_stale_sec: int = Field(
default=0,
description="Recover processing messages older than this many seconds on startup (0 = recover all).",
)
busy_timeout_ms: int = Field(
default=5000,
description="SQLite busy timeout for QueueFS in milliseconds.",
)
@model_validator(mode="after")
def validate_config(self):
"""Validate QueueFS mode and backend choices."""
valid_modes = {"shared", "worker"}
if self.mode not in valid_modes:
raise ValueError("queuefs mode must be one of: 'shared', 'worker'")
valid_backends = {"memory", "sqlite", "sqlite3", "cache"}
if self.backend not in valid_backends:
raise ValueError(
"queuefs backend must be one of: 'memory', 'sqlite', 'sqlite3', 'cache'; "
"backend='redis' was removed, use backend='cache' with top-level cache.provider/cache.params"
)
if self.recover_stale_sec < 0:
raise ValueError("queuefs recover_stale_sec must be >= 0")
if self.busy_timeout_ms < 0:
raise ValueError("queuefs busy_timeout_ms must be >= 0")
return self
class AGFSCacheTraversalMode(str, Enum):
"""Traversal strategy for cache-aware recursive RAGFS APIs."""
BACKEND = "backend"
CACHED_TRAVERSAL = "cached_traversal"
class AGFSCacheFSConfig(BaseModel):
"""CacheFS behavior independent from the selected global Provider."""
backend: Literal["local", "cache"] = Field(
default="local",
description="CacheFS backend: 'local' | 'cache'",
)
max_file_size_bytes: int = Field(
default=1024 * 1024,
description="Maximum full-file object size admitted to cache",
)
traversal_mode: AGFSCacheTraversalMode = Field(
default=AGFSCacheTraversalMode.BACKEND,
description="Traversal strategy for tree, glob, and grep",
)
bypass_prefixes: list[str] = Field(
default_factory=list,
description="Path prefixes that bypass cache",
)
@model_validator(mode="after")
def validate_config(self):
"""Validate CacheFS limits."""
if self.max_file_size_bytes <= 0:
raise ValueError("cachefs max_file_size_bytes must be > 0")
return self
class RedisCacheConfig(BaseModel):
"""Configuration for Redis cache provider."""
mode: str = Field(default="standalone", description="Redis deployment mode")
endpoints: list[str] = Field(
default_factory=lambda: ["redis://127.0.0.1:6379"],
description="Redis endpoint URLs",
)
username: str = Field(default="", description="Redis ACL username")
password_env: str = Field(default="", description="Environment variable containing password")
password: str = Field(
default="",
description="Legacy plaintext Redis password; prefer password_env",
repr=False,
)
master_name: Optional[str] = Field(default=None, description="Sentinel master name")
sentinel_username: str = Field(default="", description="Sentinel ACL username")
sentinel_password_env: str = Field(
default="", description="Environment variable containing Sentinel password"
)
sentinel_password: str = Field(
default="",
description="Legacy plaintext Sentinel password; prefer sentinel_password_env",
repr=False,
)
db: int = Field(default=0, description="Redis database number")
pool_size: int = Field(default=32, description="Redis command concurrency")
connect_timeout_ms: int = Field(default=1000, description="Redis connect timeout")
command_timeout_ms: int = Field(default=20, description="Redis command timeout")
default_ttl_seconds: int = Field(default=3600, description="Redis default cache TTL")
tls_insecure_skip_verify: bool = Field(
default=False, description="Skip Redis TLS certificate verification"
)
@model_validator(mode="after")
def validate_config(self):
if self.mode not in {"standalone", "cluster", "sentinel"}:
raise ValueError("redis mode must be standalone, cluster, or sentinel")
if not self.endpoints:
raise ValueError("redis endpoints must not be empty")
schemes = set()
for endpoint in self.endpoints:
parsed = urlparse(endpoint)
if parsed.scheme not in {"redis", "rediss"} or not parsed.hostname:
raise ValueError("redis endpoints must use redis:// or rediss:// URLs")
schemes.add(parsed.scheme)
try:
port = parsed.port
except ValueError as error:
raise ValueError("redis endpoint port is invalid") from error
if port == 0:
raise ValueError("redis endpoint port is invalid")
if (
parsed.username is not None
or parsed.password is not None
or parsed.path not in {"", "/"}
or parsed.query
or parsed.fragment
):
raise ValueError(
"redis endpoints must not include credentials, database paths, "
"query parameters, or fragments; use dedicated redis fields"
)
if self.mode == "standalone" and len(self.endpoints) != 1:
raise ValueError("redis standalone mode requires exactly one endpoint")
if self.mode == "cluster" and self.db != 0:
raise ValueError("redis cluster mode requires db=0")
if self.mode == "sentinel" and not (self.master_name or "").strip():
raise ValueError("redis sentinel mode requires master_name")
if self.db < 0 or self.db > 255:
raise ValueError("redis db must be between 0 and 255")
if self.pool_size <= 0:
raise ValueError("redis pool_size must be > 0")
if self.connect_timeout_ms <= 0:
raise ValueError("redis connect_timeout_ms must be > 0")
if self.command_timeout_ms <= 0:
raise ValueError("redis command_timeout_ms must be > 0")
if self.default_ttl_seconds < 0:
raise ValueError("redis default_ttl_seconds must be >= 0")
if len(schemes) != 1:
raise ValueError("redis endpoints must use the same URL scheme")
if self.password_env and self.password:
raise ValueError("redis password and password_env cannot both be configured")
if self.sentinel_password_env and self.sentinel_password:
raise ValueError(
"redis sentinel_password and sentinel_password_env cannot both be configured"
)
if self.tls_insecure_skip_verify and schemes != {"rediss"}:
raise ValueError("redis tls_insecure_skip_verify requires rediss:// endpoints")
return self
class AGFSPathLockConfig(BaseModel):
"""Configuration for the native RAGFS path lock service."""
@model_validator(mode="after")
def ignore_deprecated_lock_timeout(self):
"""Warn and force the deprecated timeout back to the fixed runtime value."""
if "lock_timeout_secs" in self.model_fields_set:
logger.warning(
"AGFSPathLockConfig: 'storage.agfs.pathlock.lock_timeout_secs' is deprecated "
"and ignored; the runtime wait timeout is fixed at 0.0 seconds."
)
self.lock_timeout_secs = 0.0
return self
provider: str = Field(
default="filesystem",
description="PathLock provider: 'filesystem' | 'memory' | 'sqlite' | 'cache'",
)
sqlite_path: Optional[str] = Field(
default=None,
description="Absolute local SQLite lock database file shared by all participating processes. "
"Defaults to '{storage.workspace}/_system/pathlock/pathlock.db' when not set.",
)
lock_timeout_secs: float = Field(
default=0.0,
description="Deprecated internal field. Runtime auto-pathlock wait timeout is fixed at 0.0 seconds.",
)
lock_expire_secs: float = Field(
default=30.0,
description="Seconds before an unrefreshed lock token becomes stale.",
)
@model_validator(mode="after")
def validate_config(self):
"""Validate provider and timeout/expiry ranges."""
if self.provider not in {"filesystem", "memory", "sqlite", "cache"}:
raise ValueError(
"pathlock provider must be one of: 'filesystem', 'memory', 'sqlite', 'cache'"
)
if (
self.provider == "sqlite"
and self.sqlite_path is not None
and not os.path.isabs(self.sqlite_path)
):
raise ValueError("pathlock sqlite_path must be an absolute local file path")
if not math.isfinite(self.lock_expire_secs) or self.lock_expire_secs < 1.0:
raise ValueError("pathlock lock_expire_secs must be finite and >= 1.0")
return self
class AGFSConfig(BaseModel):
"""Configuration for RAGFS (Rust-based AGFS)."""
name: str = Field(
default="primary",
description="Logical backend name, globally unique across primary and all backups",
)
path: Optional[str] = Field(
default=None,
description="[Deprecated in favor of `storage.workspace`] RAGFS data storage path. This will be ignored if `storage.workspace` is set.",
)
port: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS service port. Ignored by RAGFS.",
)
log_level: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS log level. Ignored by RAGFS.",
)
url: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS service URL. Ignored by RAGFS.",
)
mode: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS client mode. Ignored by RAGFS.",
)
impl: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS binding implementation selector. Ignored by RAGFS.",
)
backend: str = Field(
default="local", description="RAGFS storage backend: 'local' | 's3' | 'memory'"
)
timeout: int = Field(default=10, description="RAGFS request timeout (seconds)")
queue_db_path: Optional[str] = Field(
default=None,
description="Override path of the queuefs sqlite database file. "
"Defaults to '{storage.workspace}/_system/queue/queue.db' when not set. "
"Useful when the workspace volume does not support sqlite (e.g. some network filesystems).",
)
queuefs: QueueFSConfig = Field(
default_factory=QueueFSConfig,
description="QueueFS configuration.",
)
cachefs: AGFSCacheFSConfig = Field(
default_factory=AGFSCacheFSConfig,
description="CacheFS configuration.",
)
pathlock: AGFSPathLockConfig = Field(
default_factory=AGFSPathLockConfig,
description="Native RAGFS path lock configuration.",
)
retry_times: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS retry count. Ignored by RAGFS.",
)
use_ssl: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS SSL switch. Ignored by RAGFS.",
)
lib_path: Any = Field(
default=None,
exclude=True,
description="[Deprecated] Legacy AGFS binding library path. Ignored by RAGFS.",
)
# S3 backend configuration
# These settings are used when backend is set to 's3'.
# RAGFS will act as a gateway to the specified S3 bucket.
s3: S3Config = Field(default_factory=lambda: S3Config(), description="S3 backend configuration")
# Multi-write configuration
backups: Optional[dict[str, Any]] = Field(
default=None, description="Multi-write backups configuration. None = single backend mode."
)
redirects: Optional[List[dict[str, Any]]] = Field(
default=None, description="[Deprecated] Primary redirect policies. Ignored by V2."
)
@model_validator(mode="after")
def validate_config(self):
"""Validate configuration completeness and consistency"""
deprecated_fields = (
"port",
"log_level",
"url",
"mode",
"impl",
"retry_times",
"use_ssl",
"lib_path",
)
for field_name in deprecated_fields:
if field_name in self.model_fields_set:
logger.warning(
"AGFSConfig: 'storage.agfs.%s' is deprecated and ignored after the RAGFS migration.",
field_name,
)
if self.backend not in ["local", "s3", "memory"]:
raise ValueError(
f"Invalid RAGFS backend: '{self.backend}'. Must be one of: 'local', 's3', 'memory'"
)
if self.backend == "local":
pass
elif self.backend == "s3":
# Validate S3 configuration
self.s3.validate_config()
if self.queue_db_path is not None and self.queuefs.db_path is None:
logger.warning(
"AGFSConfig: 'storage.agfs.queue_db_path' is deprecated; "
"prefer 'storage.agfs.queuefs.db_path'."
)
if self.queuefs.backend == "memory":
if self.queuefs.db_path is not None or self.queue_db_path is not None:
logger.warning(
"AGFSConfig: QueueFS backend is 'memory'; "
"db_path/queue_db_path will be ignored."
)
if self.backups is not None:
initial_partitions = self.backups.get("initial_partitions", 16)
if (
not isinstance(initial_partitions, int)
or isinstance(initial_partitions, bool)
or not 1 <= initial_partitions <= 1024
):
raise ValueError("backups initial_partitions must be between 1 and 1024")
checkpoint_interval_secs = self.backups.get("checkpoint_interval_secs", 86400)
if (
not isinstance(checkpoint_interval_secs, int)
or isinstance(checkpoint_interval_secs, bool)
or checkpoint_interval_secs < 60
):
raise ValueError("backups checkpoint_interval_secs must be at least 60")
if self.backups.get("provider", "filesystem") not in {"filesystem", "cache"}:
raise ValueError("backups provider must be 'filesystem' or 'cache'")
names = set()
for item in self.backups.get("items", []):
if not isinstance(item, dict):
raise ValueError("backups items must be objects")
name = item.get("name")
if not isinstance(name, str) or not name.strip():
raise ValueError("backup name must not be empty")
name = name.strip()
if name == "primary":
raise ValueError("backup name 'primary' is reserved")
if name in names:
raise ValueError("backup names must be unique")
names.add(name)
return self