Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: add opt-in Redis directory TTL (#6392)

Lusen Dong8113272edf83
diff --git a/openviking/config/account_config.py b/openviking/config/account_config.py
index 8a172927a..72cb76223 100644
--- a/openviking/config/account_config.py
+++ b/openviking/config/account_config.py
@@ -28,6 +28,7 @@ from openviking.models.vlm.registry import is_valid_provider
from openviking_cli.utils.config.agent_evolution_config import AgentEvolutionConfig
from openviking_cli.utils.config.github_config import GitHubConfig
from openviking_cli.utils.config.runtime_field import RuntimeField
+from openviking_cli.utils.config.ttl_config import TTLConfig
from openviking_cli.utils.config.vlm_config import VLMConfig, VLMCredential
@@ -147,6 +148,8 @@ class AccountConfig(BaseModel):
)
acl: Optional[AccountAclSettings] = RuntimeField(default=None)
+ ttl: Optional[TTLConfig] = RuntimeField(default=None)
+
embedding: Optional[AccountEmbeddingConfig] = RuntimeField(default=None)
vectordb: Optional[AccountVectorDBConfig] = RuntimeField(
default=None,
diff --git a/openviking/config/binding.py b/openviking/config/binding.py
index ce88e4662..700252f30 100644
--- a/openviking/config/binding.py
+++ b/openviking/config/binding.py
@@ -79,6 +79,13 @@ def _validate_request(patch: dict, is_account: bool, creating: bool) -> None:
patch = {**patch, "vectordb": dict(patch["vectordb"])}
if "project" in patch["vectordb"]:
patch["vectordb"]["project_name"] = patch["vectordb"].pop("project")
+ if isinstance(patch, dict) and isinstance(patch.get("ttl"), dict):
+ # ``global`` is a reserved word, so the model declares it as ``global_``
+ # with a ``global`` alias; translate the external name for validation
+ # only, mirroring the vectordb ``project`` alias above.
+ patch = {**patch, "ttl": dict(patch["ttl"])}
+ if "global" in patch["ttl"]:
+ patch["ttl"]["global_"] = patch["ttl"].pop("global")
validate_patch(model, patch, creating=creating)
diff --git a/openviking/config/ttl.py b/openviking/config/ttl.py
new file mode 100644
index 000000000..c94ec411f
--- /dev/null
+++ b/openviking/config/ttl.py
@@ -0,0 +1,103 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Resolve TTL per field: User > Account > global, then fall back to global TTL.
+
+All of a user's Peers share peer_events. Unset fields inherit; disabled
+stops fallback. Configuration sources are already loaded, so resolution has no I/O.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+from openviking.core.namespace import uri_parts
+from openviking_cli.utils.config.ttl_config import TTLConfig
+
+
+class TTLIndexType(str, Enum):
+ USER_EVENTS = "user_events"
+ PEER_EVENTS = "peer_events"
+ SESSIONS = "sessions"
+
+
+TTL_ROOT_DEPTH = {
+ TTLIndexType.USER_EVENTS: 4,
+ TTLIndexType.PEER_EVENTS: 6,
+ TTLIndexType.SESSIONS: 3,
+}
+
+
+@dataclass(frozen=True)
+class EffectiveTTL:
+ index_type: Optional[TTLIndexType]
+ enabled: bool
+ days: Optional[int]
+
+ @classmethod
+ def off(cls, index_type: Optional[TTLIndexType]) -> "EffectiveTTL":
+ return cls(index_type=index_type, enabled=False, days=None)
+
+
+def classify_ttl_uri(uri: str) -> Optional[TTLIndexType]:
+ """Identify supported user/peer Events and Sessions, including descendants."""
+ try:
+ parts = uri_parts(uri)
+ except ValueError:
+ return None
+ if len(parts) < 3 or parts[0] != "user":
+ return None
+ if len(parts) == 3 and parts[2] == "peers":
+ return TTLIndexType.PEER_EVENTS # Shared user configuration scope.
+ # user/{uid}/peers/{peer_id}/memories/events[/...]
+ if len(parts) >= 6 and parts[2] == "peers" and parts[4] == "memories" and parts[5] == "events":
+ return TTLIndexType.PEER_EVENTS
+ # user/{uid}/memories/events[/...]
+ if len(parts) >= 4 and parts[2] == "memories" and parts[3] == "events":
+ return TTLIndexType.USER_EVENTS
+ # user/{uid}/sessions[/...]
+ if parts[2] == "sessions":
+ return TTLIndexType.SESSIONS
+ return None
+
+
+def resolve_effective_ttl(
+ cluster_ttl: Optional[TTLConfig],
+ account_ttl: Optional[TTLConfig],
+ uri: str,
+ user_ttl: Optional[TTLConfig] = None,
+) -> EffectiveTTL:
+ """Resolve one URI using the already-loaded configuration pair; no I/O."""
+ kind = classify_ttl_uri(uri)
+ if kind is None:
+ return EffectiveTTL.off(None)
+ for node in (kind.value, "global_"):
+ for config in (user_ttl, account_ttl, cluster_ttl):
+ value = getattr(config, node, None)
+ if value is not None:
+ return (
+ EffectiveTTL.off(kind)
+ if value == "disabled"
+ else EffectiveTTL(kind, True, value)
+ )
+ return EffectiveTTL.off(kind)
+
+
+def validate_ttl_root(uri: str, *, allow_peer_scope: bool = False) -> None:
+ """Validate a concrete scan root, or the shared Peer scope for policy reads."""
+ from openviking.core.identifiers import validate_identifier_part
+
+ kind = classify_ttl_uri(uri)
+ parts = uri_parts(uri)
+ peer_scope = allow_peer_scope and kind == TTLIndexType.PEER_EVENTS and len(parts) == 3
+ if (
+ kind is None
+ or (not peer_scope and len(parts) != TTL_ROOT_DEPTH[kind])
+ or uri != "viking://" + "/".join(parts)
+ or validate_identifier_part(parts[1], "user_id")
+ or (
+ kind == TTLIndexType.PEER_EVENTS
+ and not peer_scope
+ and validate_identifier_part(parts[3], "peer_id")
+ )
+ ):
+ raise ValueError("TTL target must be a canonical Events or Sessions root")
diff --git a/openviking/scheduling/expiry/api.py b/openviking/scheduling/expiry/api.py
index 9c1fd5045..6cf21013d 100644
--- a/openviking/scheduling/expiry/api.py
+++ b/openviking/scheduling/expiry/api.py
@@ -10,7 +10,10 @@ from typing import Protocol
SESSION_ITEM_TYPE = "session"
COMPILE_TARGET_ITEM_TYPE = "compile_target"
-SUPPORTED_ITEM_TYPES: frozenset[str] = frozenset({SESSION_ITEM_TYPE, COMPILE_TARGET_ITEM_TYPE})
+TTL_ITEM_TYPE = "ttl"
+SUPPORTED_ITEM_TYPES: frozenset[str] = frozenset(
+ {SESSION_ITEM_TYPE, COMPILE_TARGET_ITEM_TYPE, TTL_ITEM_TYPE}
+)
class ExpiryMode(str, Enum):
@@ -65,8 +68,10 @@ class CallbackResult:
@dataclass(frozen=True)
class HandlerOptions:
concurrency: int = 4
- max_attempts: int = 3
+ # None retains a failed claim for retry after the processing timeout.
+ max_attempts: int | None = 3
metadata_enabled: bool = True
+ lazy_start: bool = False
class ExpiryHandler(Protocol):
diff --git a/openviking/scheduling/expiry/consumer.py b/openviking/scheduling/expiry/consumer.py
index bdb4485ca..d2c235551 100644
--- a/openviking/scheduling/expiry/consumer.py
+++ b/openviking/scheduling/expiry/consumer.py
@@ -195,7 +195,7 @@ class ExpiryConsumer:
) -> _BucketScan:
next_poll_at_ms: int | None = None
started_at_seconds = redis_now_ms // 1000
- for attempt in range(self._binding.options.max_attempts):
+ for attempt in range(self._binding.options.max_attempts or 1):
while available > 0:
limit = min(self._config.scan_batch_size, available)
page = await self._store.read_bucket_page(
@@ -322,6 +322,10 @@ class ExpiryConsumer:
async def _finish_failed_attempt(
self, lease: ShardExecution, claim: Claim, reason: str
) -> None:
+ if self._binding.options.max_attempts is None:
+ # Leave the durable claim in place. Its timestamp bounds retries,
+ # including after consumer replacement, without an in-memory timer.
+ return
attempt, _ = decode_claim_score(claim.claim_score)
if attempt + 1 >= self._binding.options.max_attempts:
await self._log_exhausted_and_ack(lease, claim, reason)
diff --git a/openviking/scheduling/expiry/redis_store.py b/openviking/scheduling/expiry/redis_store.py
index cfade5513..22ed7f34c 100644
--- a/openviking/scheduling/expiry/redis_store.py
+++ b/openviking/scheduling/expiry/redis_store.py
@@ -46,6 +46,52 @@ from .models import (
RetryPolicy = Literal["read", "idempotent", "resolve_by_caller"]
+def create_redis_client(config: ExpiryRedisConfig, *, retry=None):
+ """Build without connecting; callers retain their own startup/retry lifecycle."""
+ import redis.asyncio as redis_async
+
+ endpoint = config.endpoints[0]
+ if "://" not in endpoint:
+ endpoint = ("rediss" if config.tls else "redis") + "://" + endpoint
+ password = config.resolve_password()
+ kwargs: dict[str, Any] = {
+ "username": config.username or None,
+ "password": password,
+ "decode_responses": True,
+ "socket_connect_timeout": config.connect_timeout_seconds,
+ "socket_timeout": config.socket_timeout_seconds,
+ "max_connections": config.max_connections,
+ }
+ if retry is not None:
+ kwargs["retry"] = retry
+ if config.mode == "standalone":
+ return redis_async.Redis.from_url(endpoint, **kwargs)
+ else:
+ from redis.asyncio.cluster import ClusterNode, RedisCluster
+ from redis.backoff import NoBackoff
+ from redis.retry import Retry
+
+ nodes = []
+ use_tls = config.tls
+ for raw in config.endpoints:
+ normalized = (
+ raw if "://" in raw else (("rediss" if config.tls else "redis") + f"://{raw}")
+ )
+ parsed = urlsplit(normalized)
+ use_tls = parsed.scheme == "rediss"
+ nodes.append(ClusterNode(parsed.hostname or "", parsed.port or 6379))
+ # Store retry policies must be the only authority for replaying
+ # commands: a lost CAS/SET NX response cannot safely be replayed.
+ # These instance-local flags leave MOVED/ASK routing intact.
+ kwargs.setdefault("retry", Retry(NoBackoff(), 0))
+ return RedisCluster(
+ startup_nodes=nodes,
+ ssl=use_tls,
+ cluster_error_retry_attempts=0,
+ **kwargs,
+ )
+
+
class RedisExpiryStore:
def __init__(
self,
@@ -62,6 +108,7 @@ class RedisExpiryStore:
self._scripts: dict[str, str] = {}
self._script_shas: dict[str, str] = {}
self._ready = False
+ self._enabled_consumers: set[str] = set()
async def open(self) -> None:
await self._io.run(self._open_on_loop)
@@ -80,6 +127,11 @@ class RedisExpiryStore:
"identity_encoding": spec.identity_encoding,
}
current = await self._command("hgetall", key)
+ self._enabled_consumers = {
+ field.removeprefix("consumer_enabled:")
+ for field, value in current.items()
+ if field.startswith("consumer_enabled:") and value == "1"
+ }
initializing = True
if current:
normalized = {str(k): str(v) for k, v in current.items()}
@@ -108,6 +160,20 @@ class RedisExpiryStore:
await self._command("hset", key, "state", "READY", retry="idempotent")
self._ready = True
+ def consumer_enabled(self, item_type: str) -> bool:
+ return item_type in self._enabled_consumers
+
+ async def enable_consumer(self, item_type: str) -> None:
+ if not self.consumer_enabled(item_type):
+ await self._command(
+ "hset",
+ self._keys.namespace(),
+ f"consumer_enabled:{item_type}",
+ "1",
+ retry="idempotent",
+ )
+ self._enabled_consumers.add(item_type)
+
async def ensure_metadata(self, item_id: str, task: ExpiryRegistration) -> None:
identity = task.identity
await self._eval(
@@ -598,52 +664,13 @@ class RedisExpiryStore:
async def _open_on_loop(self) -> None:
try:
- import redis.asyncio as redis_async
+ import redis.asyncio # noqa: F401 - validate the optional dependency
except ImportError as exc:
raise StoreOperationError(
"dependency_missing",
"Redis expiry scheduling requires the 'expiry' optional dependency",
) from exc
- endpoint = self._config.endpoints[0]
- if "://" not in endpoint:
- endpoint = ("rediss" if self._config.tls else "redis") + "://" + endpoint
- password = self._config.resolve_password()
- kwargs: dict[str, Any] = {
- "username": self._config.username or None,
- "password": password,
- "decode_responses": True,
- "socket_connect_timeout": self._config.connect_timeout_seconds,
- "socket_timeout": self._config.socket_timeout_seconds,
- "max_connections": self._config.max_connections,
- }
- if self._config.mode == "standalone":
- self._client = redis_async.Redis.from_url(endpoint, **kwargs)
- else:
- from redis.asyncio.cluster import ClusterNode, RedisCluster
- from redis.backoff import NoBackoff
- from redis.retry import Retry
-
- nodes = []
- use_tls = self._config.tls
- for raw in self._config.endpoints:
- normalized = (
- raw
- if "://" in raw
- else (("rediss" if self._config.tls else "redis") + f"://{raw}")
- )
- parsed = urlsplit(normalized)
- use_tls = parsed.scheme == "rediss"
- nodes.append(ClusterNode(parsed.hostname or "", parsed.port or 6379))
- # Store retry policies must be the only authority for replaying
- # commands: a lost CAS/SET NX response cannot safely be replayed.
- # These instance-local flags leave MOVED/ASK routing intact.
- self._client = RedisCluster(
- startup_nodes=nodes,
- ssl=use_tls,
- cluster_error_retry_attempts=0,
- retry=Retry(NoBackoff(), 0),
- **kwargs,
- )
+ self._client = create_redis_client(self._config)
try:
await self._client.ping()
except Exception as exc:
diff --git a/openviking/scheduling/expiry/registry.py b/openviking/scheduling/expiry/registry.py
index 2b70d7151..6192b1fa2 100644
--- a/openviking/scheduling/expiry/registry.py
+++ b/openviking/scheduling/expiry/registry.py
@@ -37,10 +37,14 @@ class HandlerRegistry:
raise ExpiryError("invalid_handler", "Invalid timer handler options")
if type(options.concurrency) is not int or options.concurrency <= 0:
raise ExpiryError("invalid_handler", "Handler concurrency must be positive")
- if type(options.max_attempts) is not int or not 1 <= options.max_attempts <= 256:
- raise ExpiryError("invalid_handler", "Handler max_attempts must be in [1, 256]")
+ if options.max_attempts is not None and (
+ type(options.max_attempts) is not int or not 1 <= options.max_attempts <= 256
+ ):
+ raise ExpiryError("invalid_handler", "Handler max_attempts must be None or in [1, 256]")
if type(options.metadata_enabled) is not bool:
raise ExpiryError("invalid_handler", "Handler metadata_enabled must be a boolean")
+ if type(options.lazy_start) is not bool:
+ raise ExpiryError("invalid_handler", "Handler lazy_start must be a boolean")
if item_type in self._bindings:
raise ExpiryError("duplicate_handler", f"Handler already registered: {item_type}")
self._bindings[item_type] = HandlerBinding(handler, options)
diff --git a/openviking/scheduling/expiry/runtime.py b/openviking/scheduling/expiry/runtime.py
index 6e4e9a81c..a272dce39 100644
--- a/openviking/scheduling/expiry/runtime.py
+++ b/openviking/scheduling/expiry/runtime.py
@@ -299,12 +299,14 @@ class ExpiryRuntime:
shard_count=config.shard_count,
reporting_enabled=roles.report,
metrics=self._metrics,
+ on_report=self._start_consumer,
)
self._leases = ShardLeaseCoordinator(self._store, config, self._metrics)
self._shard_runner = ShardRunner(self._leases, config, self._metrics)
self._publisher: ExpiryPublisher | None = None
self._consumers: dict[str, ExpiryConsumer] = {}
self._tasks: dict[str, asyncio.Task[None]] = {}
+ self._role_loop: asyncio.AbstractEventLoop | None = None
self._stop = asyncio.Event()
self._storage_started = False
self._storage_closed = False
@@ -382,6 +384,7 @@ class ExpiryRuntime:
if self._state.accepting_calls:
return
bindings = self._registry.freeze()
+ self._role_loop = asyncio.get_running_loop()
self._stop.clear()
if self._roles.publish:
self._publisher = ExpiryPublisher(
@@ -390,21 +393,27 @@ class ExpiryRuntime:
self._spawn_role("publisher", self._publisher.run(self._stop))
if self._roles.consume:
for item_type, binding in bindings.items():
- consumer = ExpiryConsumer(
- self._store,
- self._shard_runner,
- item_type,
- binding,
- self._config,
- self._metrics,
- )
- self._consumers[item_type] = consumer
- self._spawn_role(
- f"callback:{item_type}",
- consumer.run(self._stop),
- )
+ if not binding.options.lazy_start or self._store.consumer_enabled(item_type):
+ self._start_consumer(item_type)
self._state.accepting_calls = True
+ def _start_consumer(self, item_type: str) -> None:
+ # Reports can arrive from QueueManager loops; roles belong to startup's loop.
+ if self._role_loop is not None and asyncio.get_running_loop() is not self._role_loop:
+ self._role_loop.call_soon_threadsafe(self._start_consumer, item_type)
+ return
+ name = f"callback:{item_type}"
+ if not self._roles.consume or self._stop.is_set() or name in self._tasks:
+ return
+ binding = self._registry.get(item_type)
+ if binding is None:
+ return
+ consumer = ExpiryConsumer(
+ self._store, self._shard_runner, item_type, binding, self._config, self._metrics
+ )
+ self._consumers[item_type] = consumer
+ self._spawn_role(name, consumer.run(self._stop))
+
async def wait(self) -> None:
if not self._tasks:
await self._stop.wait()
@@ -413,8 +422,10 @@ class ExpiryRuntime:
for task in done:
await task
- async def stop_roles(self) -> None:
- self._state.accepting_calls = False
+ async def stop_roles(self, *, keep_reporting: bool = False) -> None:
+ # A producer drain may still need durable reports after callbacks stop.
+ if not keep_reporting:
+ self._state.accepting_calls = False
self._stop.set()
tasks = set(self._tasks.values())
if tasks:
diff --git a/openviking/scheduling/expiry/service.py b/openviking/scheduling/expiry/service.py
index 4bb264c5b..9008e287c 100644
--- a/openviking/scheduling/expiry/service.py
+++ b/openviking/scheduling/expiry/service.py
@@ -5,6 +5,7 @@
from __future__ import annotations
from dataclasses import dataclass
+from typing import Callable
from .api import (
SUPPORTED_ITEM_TYPES,
@@ -38,6 +39,7 @@ class _ExpirySchedulerFacade:
shard_count: int,
reporting_enabled: bool,
metrics: ExpiryMetrics,
+ on_report: Callable[[str], None] | None = None,
) -> None:
self._store = store
self._registry = registry
@@ -45,6 +47,7 @@ class _ExpirySchedulerFacade:
self._metrics = metrics
self._shard_count = shard_count
self._reporting_enabled = reporting_enabled
+ self._on_report = on_report
async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult:
self._require_ready()
@@ -53,14 +56,19 @@ class _ExpirySchedulerFacade:
validate_report(task)
self._ensure_supported_type(task.identity.item_type)
try:
+ binding = self._registry.get(task.identity.item_type)
+ if binding is not None and binding.options.lazy_start:
+ # Persist activation before the source write can become durable.
+ await self._store.enable_consumer(task.identity.item_type)
item_id = encode_item_id(task.identity)
shard_id = shard_for(item_id, self._shard_count)
result = await self._store.upsert_resource(shard_id, item_id, task)
- binding = self._registry.get(task.identity.item_type)
if binding is None or binding.options.metadata_enabled:
await self._store.ensure_metadata(item_id, task)
except StoreOperationError as exc:
raise self._business_error(exc) from exc
+ if binding is not None and binding.options.lazy_start and self._on_report is not None:
+ self._on_report(task.identity.item_type)
self._metrics.count("timer_report", item_type=task.identity.item_type, outcome="written")
return result
diff --git a/openviking/server/app.py b/openviking/server/app.py
index a4458f72b..91f1d5a42 100644
--- a/openviking/server/app.py
+++ b/openviking/server/app.py
@@ -162,15 +162,55 @@ async def _initialize_auth_plugin(
logger.info("Auth plugin initialized: %s", effective_auth_mode)
+async def _initialize_ttl_adapter(
+ app: FastAPI,
+ service: OpenVikingService,
+ config: ServerConfig,
+) -> None:
+ """Wire built-in Redis TTL when explicitly enabled.
+
+ The default configuration installs no hook, starts no TTL worker and leaves
+ the internal TTL routes unregistered.
+ """
+ from openviking.ttl.wiring import build_ttl_adapter, install_ttl_hook, set_ttl_adapter
+
+ adapter = build_ttl_adapter(config.ttl_adapter)
+ set_ttl_adapter(adapter)
+ app.state.ttl_adapter = adapter
+ if adapter is None:
+ return
+ adapter.bind_runtime(service, app.state.api_key_manager)
+ service.configure_ttl(adapter)
+ viking_fs = service.viking_fs
+ if viking_fs is not None:
+ install_ttl_hook(viking_fs, adapter)
+ if not getattr(app.state, "ttl_routes_registered", False):
+ from openviking.ttl.internal import router as ttl_internal_router
+
+ app.include_router(ttl_internal_router)
+ app.state.ttl_routes_registered = True
+ logger.info("TTL adapter initialized")
+
+
async def _initialize_runtime_state(
app: FastAPI,
service: OpenVikingService,
config: ServerConfig,
) -> None:
"""Initialize service and auth dependencies before traffic is accepted."""
- await service.initialize()
- await service.apply_agent_evolution_config()
- await _initialize_auth_plugin(app, service, config)
+ if config.ttl_adapter is not None and config.ttl_adapter.enabled:
+
+ async def install_ttl_hooks():
+ await _initialize_auth_plugin(app, service, config)
+ await _initialize_ttl_adapter(app, service, config)
+
+ await service.initialize(before_background_start=install_ttl_hooks)
+ await service.apply_agent_evolution_config()
+ else:
+ await service.initialize()
+ await service.apply_agent_evolution_config()
+ await _initialize_auth_plugin(app, service, config)
+ await _initialize_ttl_adapter(app, service, config)
manager = getattr(app.state, "api_key_manager", None)
if manager is not None and get_openviking_config().reconcile_system_assets_on_startup:
account_count = await service.reconcile_system_assets(manager)
diff --git a/openviking/server/bootstrap.py b/openviking/server/bootstrap.py
index 5576b3082..94af8cf29 100644
--- a/openviking/server/bootstrap.py
+++ b/openviking/server/bootstrap.py
@@ -461,8 +461,10 @@ def _main(recovery):
idle = getattr(getattr(ov_config, "memory", None), "session_auto_commit", None)
compile_api = getattr(ov_config, "compile_api", None)
expiry = getattr(ov_config, "expiry", None)
- uses_redis = (idle is not None and idle.enabled and idle.idle_backend == "redis") or (
- compile_api is not None and compile_api.scheduler_enabled
+ uses_redis = (
+ (idle is not None and idle.enabled and idle.idle_backend == "redis")
+ or (compile_api is not None and compile_api.scheduler_enabled)
+ or (config.ttl_adapter is not None and config.ttl_adapter.writer_enabled)
)
if uses_redis and expiry is not None and expiry.publisher_separate_process:
if resolved_config_path is None:
diff --git a/openviking/server/config.py b/openviking/server/config.py
index 3f8161b1d..b0d465772 100644
--- a/openviking/server/config.py
+++ b/openviking/server/config.py
@@ -30,6 +30,7 @@ from openviking_cli.utils.config.consts import (
OPENVIKING_CONFIG_ENV,
SYSTEM_CONFIG_DIR,
)
+from openviking_cli.utils.config.ttl_config import TTLConfig
logger = get_logger(__name__)
@@ -141,6 +142,7 @@ class UserConfig(BaseModel):
add_targets: AddTargetsConfig = Field(default_factory=AddTargetsConfig)
memory_policy: Optional[Dict[str, Any]] = None
+ ttl: Optional[TTLConfig] = None
agent_evolution: DeprecatedUserAgentEvolutionConfig = Field(
default_factory=DeprecatedUserAgentEvolutionConfig,
exclude=True,
@@ -262,6 +264,22 @@ class UsageReporterConfig(BaseModel):
sinks: List[UsageReporterSinkConfig] = Field(default_factory=list)
+class TTLAdapterConfig(BaseModel):
+ """Opt-in Redis-backed directory TTL configuration.
+
+ Uses the built-in implementation, supplied with its ResourceID, Redis
+ connection and writer/reader role through ``config``. No external Python
+ package is required. Unset or disabled leaves all TTL hooks inactive.
+ """
+
+ enabled: bool = False
+ config: Dict[str, object] = Field(default_factory=dict)
+
+ @property
+ def writer_enabled(self) -> bool:
+ return self.enabled and bool(self.config.get("writer", True))
+
+
class TraceDumpBodyConfig(BaseModel):
"""HTTP body dump configuration.
@@ -371,6 +389,8 @@ class ServerConfig(BaseModel):
trusted_identity_pending_max_size: int = Field(10_000, gt=0)
observability: ObservabilityConfig = Field(default_factory=ObservabilityConfig)
usage_reporter: UsageReporterConfig = Field(default_factory=UsageReporterConfig)
+ # Built-in Redis TTL; disabled unless explicitly configured.
+ ttl_adapter: Optional[TTLAdapterConfig] = None
# Public-facing base URL emitted in MCP-issued upload instructions. See
# ``openviking.server.mcp_endpoint._resolve_public_base_url`` for the full
# resolution chain: env var > this field > X-Forwarded-Host/Proto > Host header
diff --git a/openviking/server/routers/admin.py b/openviking/server/routers/admin.py
index 9ed391617..0c967bedf 100644
--- a/openviking/server/routers/admin.py
+++ b/openviking/server/routers/admin.py
@@ -3,6 +3,7 @@
"""Admin endpoints for OpenViking multi-tenant HTTP Server."""
import asyncio
+import json
from collections.abc import Mapping
from typing import Any, Literal, Optional
@@ -31,7 +32,6 @@ from openviking.server.user_config import (
validate_add_targets,
validate_user_memory_policy,
write_user_config,
- write_user_memory_policy,
)
from openviking.session.memory.account_templates import (
EDITABLE_MEMORY_TEMPLATE_FIELDS,
@@ -47,6 +47,7 @@ from openviking_cli.exceptions import (
InvalidArgumentError,
NotFoundError,
PermissionDeniedError,
+ UnimplementedError,
)
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config import get_openviking_config
@@ -56,6 +57,8 @@ logger = get_logger(__name__)
router = APIRouter(prefix="/api/v1/admin", tags=["admin"])
+_TTL_UNSUPPORTED_MESSAGE = "Redis-backed TTL is not enabled in this deployment."
+
class CreateAccountRequest(BaseModel):
account_id: str
@@ -132,6 +135,28 @@ class ConfigPatchRequest(BaseModel):
content: str | None = None
+def _reject_unsupported_ttl_config(
+ settings: Mapping[str, object] | None = None,
+ content: str | None = None,
+) -> None:
+ """Reject explicit TTL settings unless the built-in Redis runtime is enabled."""
+
+ from openviking.ttl.wiring import ttl_config_supported
+
+ if ttl_config_supported():
+ return
+ if settings is not None and "ttl" in settings:
+ raise UnimplementedError(_TTL_UNSUPPORTED_MESSAGE)
+ if content is None:
+ return
+ try:
+ parsed = json.loads(content.lstrip("\ufeff"))
+ except (TypeError, ValueError):
+ return
+ if isinstance(parsed, dict) and "ttl" in parsed:
+ raise UnimplementedError(_TTL_UNSUPPORTED_MESSAGE)
+
+
_ROOT_ONLY_ACCOUNT_CONFIG_SECTIONS = frozenset({"vlm", "query_planner", "embedding", "vectordb"})
@@ -161,7 +186,8 @@ def _visible_account_config(
class UserSettingsPatch(BaseModel):
- memory_policy: Optional[dict]
+ memory_policy: Optional[dict] = None
+ ttl: Optional[dict] = None
model_config = {"extra": "forbid"}
@@ -287,7 +313,10 @@ def _has_add_targets(user_config: UserConfig | None) -> bool:
def _has_initial_user_config(user_config: UserConfig | None) -> bool:
- return bool(_has_add_targets(user_config) or (user_config and user_config.memory_policy))
+ return bool(
+ _has_add_targets(user_config)
+ or (user_config and (user_config.memory_policy or user_config.ttl is not None))
+ )
async def _validate_initial_user_config(
@@ -306,6 +335,8 @@ async def _validate_initial_user_config(
viking_fs=service.viking_fs,
)
if user_config is not None:
+ if user_config.ttl is not None:
+ _reject_unsupported_ttl_config({"ttl": {}})
validate_user_memory_policy(user_config.memory_policy)
@@ -409,6 +440,9 @@ def _user_settings_result(
"account_id": account_id,
"user_id": user_id,
"memory_policy": memory_policy,
+ "ttl": user_config.ttl.model_dump(by_alias=True, exclude_none=True)
+ if user_config.ttl is not None
+ else None,
}
@@ -423,6 +457,7 @@ async def create_account(
ctx: RequestContext = Depends(get_request_context),
):
"""Create a new account (workspace) with its first admin user."""
+ _reject_unsupported_ttl_config(body.settings)
service = get_service()
account_ctx = RequestContext(
user=UserIdentifier(body.account_id, body.admin_user_id),
@@ -588,20 +623,41 @@ async def patch_account_settings(
async def get_account_configuration(
request: Request,
account_id: str = Path(..., description="Account ID"),
+ path: Optional[str] = Query(None),
ctx: RequestContext = Depends(get_request_context),
):
"""Return this account layer's explicit runtime configuration."""
_check_account_access(ctx, account_id)
- await _check_account_exists(request, account_id)
- settings = await _get_runtime_config_manager().get_settings(
- ConfigScope.account(account_id)
- )
+ manager = await _check_account_exists(request, account_id)
+ settings = await _get_runtime_config_manager().get_settings(ConfigScope.account(account_id))
+ result = {
+ "account_id": account_id,
+ "settings": _visible_account_config(ctx, settings),
+ }
+ if path is not None:
+ from openviking.config.ttl import resolve_effective_ttl, validate_ttl_root
+ from openviking.core.namespace import uri_parts
+
+ try:
+ validate_ttl_root(path, allow_peer_scope=True)
+ except ValueError as exc:
+ raise InvalidArgumentError(str(exc)) from exc
+ user_id = uri_parts(path)[1]
+ await _check_user_exists(request, account_id, user_id, manager)
+ user_ctx = RequestContext(user=UserIdentifier(account_id, user_id), role=Role.USER)
+ config = await read_user_config(get_service().viking_fs, user_ctx)
+ policy = await _get_runtime_config_manager().resolve_account(
+ account_id,
+ lambda v: resolve_effective_ttl(v.cluster.ttl, v.account.ttl, path, config.ttl),
+ )
+ result["user_ttl"] = (
+ config.ttl.model_dump(by_alias=True, exclude_none=True) if config.ttl else None
+ )
+ result["path"] = path
+ result["effective_policy"] = policy.days if policy.enabled else "disabled"
return Response(
status="ok",
- result={
- "account_id": account_id,
- "settings": _visible_account_config(ctx, settings),
- },
+ result=result,
)
@@ -724,13 +780,17 @@ async def patch_account_configuration(
_check_account_access(ctx, account_id)
await _check_account_exists(request, account_id)
_authorize_account_config_patch(ctx, body.settings)
+ _reject_unsupported_ttl_config(body.settings)
try:
- await _get_runtime_config_manager().patch_account(account_id, body.settings)
+ if "ttl" in body.settings:
+ from openviking.ttl.wiring import get_ttl_adapter
+
+ await get_ttl_adapter().patch_configuration(body.settings, account_id)
+ else:
+ await _get_runtime_config_manager().patch_account(account_id, body.settings)
except (ConfigPatchError, ValueError) as exc:
raise InvalidArgumentError(str(exc)) from exc
- settings = await _get_runtime_config_manager().get_settings(
- ConfigScope.account(account_id)
- )
+ settings = await _get_runtime_config_manager().get_settings(ConfigScope.account(account_id))
return Response(
status="ok",
result={
@@ -856,6 +916,7 @@ async def patch_cluster_configuration(
ctx: RequestContext = Depends(get_request_context),
):
"""Apply a three-state PATCH to the cluster configuration layer."""
+ _reject_unsupported_ttl_config(body.settings, body.content)
if dry_run and (source != "file" or body.content is None):
raise InvalidArgumentError("dry_run requires source=file and full file content")
if source == "file":
@@ -888,7 +949,12 @@ async def patch_cluster_configuration(
raise InvalidArgumentError("Full file content requires source=file")
runtime_config = _get_runtime_config_manager()
try:
- await runtime_config.patch_cluster(body.settings)
+ if "ttl" in body.settings:
+ from openviking.ttl.wiring import get_ttl_adapter
+
+ await get_ttl_adapter().patch_configuration(body.settings)
+ else:
+ await runtime_config.patch_cluster(body.settings)
except (ConfigPatchError, ValueError) as exc:
raise InvalidArgumentError(str(exc)) from exc
settings = await runtime_config.get_settings(ConfigScope.cluster())
@@ -1055,7 +1121,7 @@ async def patch_user_settings(
user_id: str = Path(..., description="User ID"),
ctx: RequestContext = Depends(get_request_context),
):
- """Update or clear the allowlisted User memory policy without restarting."""
+ """Patch the allowlisted User memory and TTL policies without restarting."""
_check_account_access(ctx, account_id)
manager = await _check_account_exists(request, account_id)
await _check_user_exists(request, account_id, user_id, manager)
@@ -1066,8 +1132,11 @@ async def patch_user_settings(
user=UserIdentifier(account_id, user_id),
role=Role.USER,
)
- await write_user_memory_policy(service.viking_fs, user_ctx, body.memory_policy)
- user_config = await read_user_config(service.viking_fs, user_ctx)
+ from openviking.server.user_config import patch_user_config
+
+ patch = body.model_dump(exclude_unset=True)
+ _reject_unsupported_ttl_config(patch)
+ user_config = await patch_user_config(service.viking_fs, user_ctx, patch)
return Response(
status="ok",
result=_user_settings_result(
diff --git a/openviking/server/routers/filesystem.py b/openviking/server/routers/filesystem.py
index df3e50e60..02a96be72 100644
--- a/openviking/server/routers/filesystem.py
+++ b/openviking/server/routers/filesystem.py
@@ -23,6 +23,7 @@ from openviking.storage.acl import AclSpec
from openviking.storage.expr import And, Eq, In
from openviking.storage.vector_ids import is_vector_record_id
from openviking.storage.vikingdb_manager import VikingDBManagerProxy
+from openviking.ttl.wiring import attach_expires_at
from openviking.utils.tags import normalize_search_tags
from openviking_cli.exceptions import NotFoundError
@@ -127,6 +128,7 @@ async def ls(
if mapped is not None:
raise mapped from e
raise
+ await attach_expires_at(page.entries, _ctx)
return ListingResponse(status="ok", result=page.entries, has_more=page.has_more)
@@ -183,6 +185,7 @@ async def tree(
if mapped is not None:
raise mapped from e
raise
+ await attach_expires_at(page.entries, _ctx)
return ListingResponse(status="ok", result=page.entries, has_more=page.has_more)
@@ -204,7 +207,9 @@ async def stat(
# URI requests use the canonical validated URI. ID requests are resolved
# inside VikingFS, which returns the corresponding canonical URI.
response_uri = result.get("uri", resolved)
- return Response(status="ok", result={**result, "uri": response_uri})
+ payload = {**result, "uri": response_uri}
+ await attach_expires_at([payload], _ctx)
+ return Response(status="ok", result=payload)
except AGFSNotFoundError:
raise NotFoundError(uri, "file")
except AGFSClientError as e:
diff --git a/openviking/server/user_config.py b/openviking/server/user_config.py
index 1f9e78da7..38bd5182a 100644
--- a/openviking/server/user_config.py
+++ b/openviking/server/user_config.py
@@ -161,13 +161,14 @@ async def update_user_config(
uri = user_config_uri(ctx)
async with _user_config_lock(viking_fs, uri, ctx) as handle:
current = await read_user_config(viking_fs, ctx)
- before = current.model_dump()
+ before = current.model_dump(by_alias=True)
result = updater(current)
runtime = await validate_add_targets(current.add_targets, ctx=ctx, viking_fs=viking_fs)
current.add_targets.resource_uri = runtime.resource_uri
current.add_targets.skill_uri = runtime.skill_uri
validate_user_memory_policy(current.memory_policy)
- if current.model_dump() != before:
+ if current.model_dump(by_alias=True) != before:
+ await _prepare_ttl_change(ctx, UserConfig.model_validate(before), current)
before_content = json.dumps(before, ensure_ascii=False, sort_keys=True)
await viking_fs.write_file(
user_config_backup_uri(ctx),
@@ -178,7 +179,7 @@ async def update_user_config(
await viking_fs.write_file(
uri,
json.dumps(
- current.model_dump(exclude_none=True),
+ current.model_dump(by_alias=True, exclude_none=True),
ensure_ascii=False,
sort_keys=True,
),
@@ -193,9 +194,49 @@ async def update_user_config(
lease_ref=handle,
)
raise
+ if before.get("ttl") != current.model_dump(by_alias=True).get("ttl"):
+ await _publish_ttl_change(ctx, current)
return result
+async def _prepare_ttl_change(ctx: RequestContext, before: UserConfig, after: UserConfig) -> None:
+ if before.ttl == after.ttl:
+ return
+ from openviking.ttl.wiring import get_ttl_adapter, ttl_config_supported
+ from openviking_cli.exceptions import UnimplementedError
+
+ if not ttl_config_supported():
+ raise UnimplementedError("Redis-backed TTL is not enabled in this deployment.")
+ await get_ttl_adapter().prepare_user_configuration(ctx, before.ttl, after.ttl)
+
+
+async def _publish_ttl_change(ctx: RequestContext, config: UserConfig) -> None:
+ from openviking.ttl.wiring import get_ttl_adapter
+
+ adapter = get_ttl_adapter()
+ if adapter is not None:
+ await adapter.publish_user_configuration(ctx, config.ttl)
+
+
+async def patch_user_config(viking_fs: VikingFS, ctx: RequestContext, patch: dict) -> UserConfig:
+ """Apply a sparse settings patch through the existing locked write lifecycle."""
+ from openviking.config.merge import apply_three_state_patch
+
+ def update(current: UserConfig) -> UserConfig:
+ payload = current.model_dump(by_alias=True, exclude_none=True)
+ if "ttl" in patch:
+ payload = apply_three_state_patch(payload, {"ttl": patch["ttl"]})
+ if "memory_policy" in patch:
+ # Keep the existing whole-policy replacement semantics.
+ payload["memory_policy"] = patch["memory_policy"]
+ candidate = _user_config_from_payload(payload)
+ current.memory_policy = candidate.memory_policy
+ current.ttl = candidate.ttl
+ return current
+
+ return await update_user_config(viking_fs, ctx, update)
+
+
async def write_user_config(
viking_fs: VikingFS,
ctx: RequestContext,
@@ -207,16 +248,18 @@ async def write_user_config(
validate_user_memory_policy(user_config.memory_policy)
uri = user_config_uri(ctx)
async with _user_config_lock(viking_fs, uri, ctx) as handle:
+ await _prepare_ttl_change(ctx, UserConfig(), user_config)
await viking_fs.write_file(
uri,
json.dumps(
- user_config.model_dump(exclude_none=True),
+ user_config.model_dump(by_alias=True, exclude_none=True),
ensure_ascii=False,
sort_keys=True,
),
ctx=ctx,
lease_ref=handle,
)
+ await _publish_ttl_change(ctx, user_config)
return runtime
@@ -224,7 +267,8 @@ async def delete_user_config(viking_fs: VikingFS, ctx: RequestContext) -> None:
try:
await viking_fs.rm(user_config_uri(ctx), ctx=ctx)
except (NotFoundError, FileNotFoundError):
- return
+ pass
+ await _publish_ttl_change(ctx, UserConfig())
async def read_user_add_targets(
diff --git a/openviking/service/core.py b/openviking/service/core.py
index 993c805bc..d4894f4ed 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -9,7 +9,7 @@ Main service class that composes all sub-services and manages infrastructure lif
import asyncio
import inspect
import os
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional
from openviking.core.directories import DirectoryInitializer
from openviking.privacy.sanitizer import PrivacySanitizer
@@ -71,6 +71,7 @@ if TYPE_CHECKING:
from openviking.config.vlm import VLMResolver
from openviking.scheduling.expiry.runtime import ExpiryRuntime
from openviking.session.compressor_v3 import SessionCompressorV3
+ from openviking.ttl.adapter import TTLAdapter
class OpenVikingService:
@@ -116,6 +117,7 @@ class OpenVikingService:
self._session_auto_commit_scheduler: Optional[SessionAutoCommitScheduler] = None
self._rule_cleanup_scheduler: Optional[RuleCleanupScheduler] = None
self._expiry_runtime: Optional["ExpiryRuntime"] = None
+ self._ttl_adapter: Optional["TTLAdapter"] = None
self._compile_scheduler: Optional[CompileScheduler] = None
self._encryptor: Optional[Any] = None
self._privacy_config_service: Optional[UserPrivacyConfigService] = None
@@ -461,7 +463,9 @@ class OpenVikingService:
"""Get the Compile task service."""
return self._compile_service
- async def initialize(self) -> None:
+ async def initialize(
+ self, *, before_background_start: Optional[Callable[[], Awaitable[None]]] = None
+ ) -> None:
"""Initialize OpenViking storage and indexes."""
if self._initialized:
logger.debug("Already initialized")
@@ -619,6 +623,9 @@ class OpenVikingService:
viking_fs=self._viking_fs,
session_service=self._session_service,
)
+ # Install optional server write hooks before any producer or consumer starts.
+ if before_background_start is not None:
+ await before_background_start()
try:
session_auto_commit_config = get_openviking_config().memory.session_auto_commit
except Exception:
@@ -754,9 +761,11 @@ class OpenVikingService:
await system_asset_synchronizer.sync(default_ctx)
compile_expiry_enabled = self._compile_scheduler is not None
- if session_expiry_enabled or compile_expiry_enabled:
+ ttl_expiry_enabled = self._ttl_adapter is not None and self._ttl_adapter.supports_config()
+ if session_expiry_enabled or compile_expiry_enabled or ttl_expiry_enabled:
from openviking.scheduling.expiry.runtime import ExpiryRuntime, ExpiryRuntimeRoles
+ expiry_config.validate_for_runtime()
self._expiry_runtime = ExpiryRuntime(
expiry_config,
roles=ExpiryRuntimeRoles(
@@ -773,8 +782,12 @@ class OpenVikingService:
)
if compile_expiry_enabled:
self._compile_scheduler.install_expiry(self._expiry_runtime.scheduler)
+ if ttl_expiry_enabled:
+ self._ttl_adapter.install_expiry(self._expiry_runtime.scheduler)
await self._expiry_runtime.start_storage()
await self._expiry_runtime.start_roles()
+ if ttl_expiry_enabled:
+ await self._ttl_adapter.start()
except Exception:
await self._stop_expiry()
raise
@@ -809,8 +822,15 @@ class OpenVikingService:
self._initialized = True
logger.info("OpenVikingService initialized")
+ def configure_ttl(self, adapter: "TTLAdapter") -> None:
+ """Install optional TTL before background producers and expiry roles start."""
+ self._ttl_adapter = adapter
+
async def _stop_expiry(self) -> None:
"""Stop expiry work before releasing dependencies used by callbacks."""
+ ttl_adapter = getattr(self, "_ttl_adapter", None)
+ if ttl_adapter is not None:
+ await ttl_adapter.aclose()
runtime = getattr(self, "_expiry_runtime", None)
if runtime is None:
session_service = getattr(self, "_session_service", None)
@@ -840,7 +860,11 @@ class OpenVikingService:
self._compile_scheduler = None
logger.info("CompileScheduler stopped")
- await self._stop_expiry()
+ if self._ttl_adapter is None:
+ await self._stop_expiry()
+ elif self._expiry_runtime is not None:
+ # Stop new callbacks, but keep reports available while writers drain.
+ await self._expiry_runtime.stop_roles(keep_reporting=True)
if self._runtime_config_manager:
await self._runtime_config_manager.stop_refresh_loop()
@@ -868,6 +892,19 @@ class OpenVikingService:
self._queue_manager = None
logger.info("Queue manager stopped")
+ # Draining writers still need post-write registration.
+ # Stop TTL consumers only after those writers settle, while their storage
+ # and vector clients remain available for in-flight recycling.
+ from openviking.ttl.wiring import get_ttl_adapter, set_ttl_adapter
+
+ ttl_adapter = get_ttl_adapter()
+ if ttl_adapter is not None:
+ await ttl_adapter.aclose()
+ set_ttl_adapter(None)
+ self._ttl_adapter = None
+ if self._expiry_runtime is not None:
+ await self._stop_expiry()
+
# Queue workers and scheduled producers persist their final Task/Work
# transitions while stopping, so keep their store open until they drain.
await get_task_tracker().close()
diff --git a/openviking/service/deletion.py b/openviking/service/deletion.py
index ad6d2d696..07c64cf8a 100644
--- a/openviking/service/deletion.py
+++ b/openviking/service/deletion.py
@@ -338,6 +338,12 @@ class DeletionService:
if scheduler is not None:
await run_to_completion(lambda: scheduler.delete_tasks(account_id, user_id))
await run_to_completion(lambda: self._cancel_tasks(account_id, user_id))
+ if user_id is None:
+ from openviking.ttl.wiring import get_ttl_adapter
+
+ ttl = get_ttl_adapter()
+ if ttl is not None:
+ await run_to_completion(lambda: ttl.delete_account(account_id))
await run_to_completion(
lambda: tracker.delete_tasks(
account_id,
diff --git a/openviking/session/session.py b/openviking/session/session.py
index 0e83e1909..c5554f2fe 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -751,6 +751,10 @@ class Session:
session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS
)
try:
+ if not filenames:
+ # Phase 1 writes different child files, each with its own lock.
+ yield session_lease
+ return
file_lease = await agfs.pathlock_acquire_exact_batch(
[f"{session_path}/{name}" for name in filenames],
timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS,
@@ -1252,11 +1256,7 @@ class Session:
if await self._archives.file_exists(archive_uri, ".failed.json"):
return False
- session_path = self._viking_fs._uri_to_path(self._session_uri, ctx=self.ctx)
- lease = await self._viking_fs._async_agfs.pathlock_acquire_exact(
- session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS
- )
- try:
+ async with self._state_lock():
marker = await self._read_phase1_meta(archive_uri)
if marker.get("status") == "ready":
return True
@@ -1351,8 +1351,6 @@ class Session:
await self._write_phase1_ready_marker(archive_uri)
logger.warning("Recovered interrupted Session Phase 1: %s", archive_uri)
return True
- finally:
- await self._viking_fs._async_agfs.pathlock_release(lease)
def commit(
self,
@@ -1502,12 +1500,8 @@ class Session:
# The exact lock on the Session directory is the mutex for root Session
# state and archive-number allocation. Child files use their own exact
# locks, so Phase 2 can keep writing an earlier archive concurrently.
- session_path = self._viking_fs._uri_to_path(self._session_uri, ctx=self.ctx)
- lease = await self._viking_fs._async_agfs.pathlock_acquire_exact(
- session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS
- )
publication_lease: Optional[Any] = None
- try:
+ async with self._state_lock():
self._messages = await self._read_live_messages_strict()
await self._load_meta()
if (
@@ -1792,8 +1786,6 @@ class Session:
)
publication_lease = None
raise
- finally:
- await self._viking_fs._async_agfs.pathlock_release(lease)
# Lock released; Phase 1 intent, retained root, metadata and ready
# metadata are durable before Task-store or QueueFS latency.
@@ -3055,11 +3047,7 @@ class Session:
record_auto_commit_success: bool = False,
) -> Optional[SessionMeta]:
"""Return saved metadata, or None when its read fails and statistics are skipped."""
- session_path = self._viking_fs._uri_to_path(self._session_uri, ctx=self.ctx)
- lease = await self._viking_fs._async_agfs.pathlock_acquire_exact(
- session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS
- )
- try:
+ async with self._state_lock(".meta.json") as lease:
try:
latest_meta = await self._read_commit_meta()
except Exception as exc:
@@ -3097,10 +3085,8 @@ class Session:
# a clean auto-commit even after Phase 2 reloads the latest meta.
latest_meta.last_auto_commit_at = get_current_timestamp()
self._meta = latest_meta
- await self._save_meta()
+ await self._save_meta(lease_ref=lease)
return latest_meta
- finally:
- await self._viking_fs._async_agfs.pathlock_release(lease)
async def _read_live_messages_strict(self) -> List[Message]:
"""Read the authoritative root JSONL without silently dropping corrupt rows."""
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index dd95d6a19..698c4a845 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -680,7 +680,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):
execute_started_at = time.perf_counter()
if embedding_msg.queue_enqueued_at > 0:
queue_wait_ms = max((time.time() - embedding_msg.queue_enqueued_at) * 1000.0, 0.0)
- inserted_data = embedding_msg.context_data
+ inserted_data = dict(embedding_msg.context_data)
+ inserted_data.pop("_write_source", None) # Ignore sources in already queued messages.
+ inserted_data.pop("_write_lease", None) # Ignore leases in already queued messages.
logger.debug(
"Processing embedding message: %s action=%s",
self._embedding_delivery_log_context(embedding_msg),
@@ -923,9 +925,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
)
else:
result = await self._vikingdb.upsert(
- inserted_data,
- ctx=ctx,
- options=upsert_options,
+ inserted_data, ctx=ctx, options=upsert_options
)
record_id = result
if record_id:
diff --git a/openviking/storage/viking_fs/__init__.py b/openviking/storage/viking_fs/__init__.py
index 83944e387..25fcb8bd2 100644
--- a/openviking/storage/viking_fs/__init__.py
+++ b/openviking/storage/viking_fs/__init__.py
@@ -163,6 +163,7 @@ class VikingFS(
self._background_tasks: set = set()
self._deletion_guard: Optional[Callable[[str, str], bool]] = None
self._privacy_sanitizer: Optional[Any] = None
+ self._post_write_hook: Optional[Callable[[str, Optional[RequestContext], int], Any]] = None
def set_vlm_resolver(self, resolver: "VLMResolver") -> None:
"""Set the VLM resolver supplied by the owning service."""
@@ -172,6 +173,13 @@ class VikingFS(
"""Install the account service's text sanitizer for direct writes."""
self._privacy_sanitizer = sanitizer
+ def set_post_write_hook(
+ self,
+ hook: Optional[Callable[[str, Optional[RequestContext], int], Any]],
+ ) -> None:
+ """Install an optional observer called after a file write succeeds."""
+ self._post_write_hook = hook
+
VikingFS.__module__ = __name__
diff --git a/openviking/storage/viking_fs/_ops.py b/openviking/storage/viking_fs/_ops.py
index 876ff6b7b..f19dd1a90 100644
--- a/openviking/storage/viking_fs/_ops.py
+++ b/openviking/storage/viking_fs/_ops.py
@@ -3,7 +3,9 @@
"""Core filesystem operations mixin for VikingFS."""
import asyncio
+import inspect
import math
+import time
import uuid
from dataclasses import replace
from datetime import datetime, timezone
@@ -181,7 +183,33 @@ class _OpsMixin:
data = data.encode("utf-8")
# Encryption (when configured) happens inside the ragfs layer keyed by account_id.
- return await self._async_agfs.write(path, data)
+ result = await self._async_agfs.write(path, data)
+ await self._notify_post_write(uri, ctx)
+ return result
+
+ async def _notify_post_write(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext],
+ written_at_ms: Optional[int] = None,
+ ) -> None:
+ """Best-effort notification for optional product adapters.
+
+ The storage write is already committed when this runs, so observer
+ failures are logged and must not turn a successful write into an API
+ failure. A disabled hook returns before reading the clock or doing I/O.
+ """
+ hook = getattr(self, "_post_write_hook", None)
+ if hook is None:
+ return
+ if written_at_ms is None:
+ written_at_ms = time.time_ns() // 1_000_000
+ try:
+ result = hook(uri, ctx, written_at_ms)
+ if inspect.isawaitable(result):
+ await result
+ except Exception:
+ logger.exception("[VikingFS] Post-write hook failed for %s", uri)
async def mkdir(
self,
@@ -268,7 +296,11 @@ class _OpsMixin:
uris_to_delete.append(target_uri)
real_ctx = self._ctx_or_default(ctx)
estimated_count = await _estimate_deleted_count(path, real_ctx)
- await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
+ vector_store = self._get_vector_store()
+ if recursive and vector_store is not None:
+ await vector_store.delete_uris(real_ctx, [target_uri], recursive=True)
+ else:
+ await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
logger.info(f"[VikingFS] rm target not found, cleaned orphan index: {uri}")
return {"estimated_deleted_count": estimated_count}
@@ -344,7 +376,11 @@ class _OpsMixin:
if not no_vector:
real_ctx = self._ctx_or_default(ctx)
estimated_count = await _estimate_deleted_count(path, real_ctx)
- await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
+ vector_store = self._get_vector_store()
+ if recursive and vector_store is not None:
+ await vector_store.delete_uris(real_ctx, [target_uri], recursive=True)
+ else:
+ await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
try:
result = await self._async_agfs.rm(
path,
@@ -1995,6 +2031,7 @@ class _OpsMixin:
fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
auto_pathlock=auto_pathlock,
)
+ await self._notify_post_write(uri, ctx)
async def read_file(
self,
@@ -2131,6 +2168,7 @@ class _OpsMixin:
fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
auto_pathlock=auto_pathlock,
)
+ await self._notify_post_write(uri, ctx)
async def append_file(
self,
@@ -2144,6 +2182,7 @@ class _OpsMixin:
path = self._uri_to_path(uri, ctx=ctx)
owned_lease = None
+ written_at_ms: Optional[int] = None
try:
await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
lease = lease_ref
@@ -2173,6 +2212,8 @@ class _OpsMixin:
final_content,
fs_ctx=fs_ctx,
)
+ if getattr(self, "_post_write_hook", None) is not None:
+ written_at_ms = time.time_ns() // 1_000_000
except Exception as e:
logger.error(f"[VikingFS] Failed to append to file {uri}: {e}")
@@ -2183,6 +2224,8 @@ class _OpsMixin:
finally:
if owned_lease is not None:
await self._async_agfs.pathlock_release(owned_lease)
+ if written_at_ms is not None:
+ await self._notify_post_write(uri, ctx, written_at_ms)
async def ls(
self,
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index a2323685b..a859188a2 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -2637,7 +2637,14 @@ class VikingVectorIndexBackend:
And([Eq("account_id", account_id), Eq("owner_user_id", user_id)])
)
- async def delete_uris(self, ctx: RequestContext, uris: List[str]) -> None:
+ async def delete_uris(
+ self, ctx: RequestContext, uris: List[str], *, recursive: bool = False
+ ) -> None:
+ if recursive:
+ backend = await self._get_backend_for_context(ctx)
+ for uri in uris:
+ await backend.delete_by_filter(self._uri_transfer_filter(ctx, uri, recursive=True))
+ return
for uri in uris:
conds: List[FilterExpr] = [
Eq("account_id", ctx.account_id),
diff --git a/openviking/ttl/README.md b/openviking/ttl/README.md
new file mode 100644
index 000000000..edce0b36c
--- /dev/null
+++ b/openviking/ttl/README.md
@@ -0,0 +1,139 @@
+# Redis directory TTL
+
+TTL is optional and disabled by default. The first implementation requires Redis;
+there is no Redis-free backend. The implementation and tests ship with OpenViking.
+It uses the existing expiry Redis client and storage/configuration lifecycle.
+
+Install OpenViking with the `expiry` extra, then merge into your server config:
+
+```json
+{
+ "server": {
+ "ttl_adapter": {
+ "enabled": true,
+ "config": {
+ "resource_id": "stable-library-id",
+ "redis": {"mode": "standalone", "endpoints": ["redis://127.0.0.1:6379"]}
+ }
+ }
+ },
+ "expiry": {
+ "service_name": "stable-library-id",
+ "publisher_separate_process": false,
+ "redis": {"mode": "standalone", "endpoints": ["redis://127.0.0.1:6379"]}
+ },
+ "ttl": {"sessions": 30, "user_events": 60, "peer_events": 60}
+}
+```
+
+Use a stable ResourceID across restarts; each Redis scope also includes the
+OpenViking Account ID. Production credentials and Redis durability are deployment
+configuration. The regular OV Docker image already includes the expiry extra.
+No external Python class path, wheel or overlay image is needed.
+`ttl_adapter.config.redis` stores directory indexes; `expiry.redis` stores the
+shared scheduler's tasks. The example uses the same Redis and runs the publisher
+in-process. If auto commit already configures expiry, retain that configuration
+and its publisher deployment. TTL also requires `expiry.service_name` and
+`expiry.redis.endpoints`.
+
+An omitted or disabled `server.ttl_adapter` installs no write hook, starts no TTL
+worker and loads no TTL Redis client. Explicit policy writes then return
+`501 UNIMPLEMENTED`: `Redis-backed TTL is not enabled in this deployment.`
+With the adapter enabled, writes only enqueue in-memory candidates. The
+registration worker loads policy and drops disabled candidates before Redis I/O.
+UserConfig cold reads stay off the business write path. A reader uses
+`config.writer=false`: expiry display is
+available, but it never scans or recycles. TTL is soft: searches and reads keep
+returning expired content until physical deletion. Historical Event writes are
+allowed; they do not renew the date directory's lifetime.
+
+The existing write path buffers successful writes and flushes them asynchronously
+in batches. Session directory scores use the write time (`ZADD GT`); Event date
+directories use the directory date (`ZADD NX`). Deletion rechecks current policy
+and Session `messages.jsonl` ModTime under the existing directory lock. Redis
+index removal compares the original score to preserve a concurrent renewal.
+
+TTL adds no directory locks to Session, Event or vector writes. Session mutations
+and memory updates keep their existing native file/transaction locks. Deletion
+uses the native Tree lock for the policy/ModTime recheck, recursive removal and
+conditional index cleanup; lock conflicts retain the registration for retry.
+
+TTL adds no Session existence preflight or missing-file rule. Session APIs keep
+their existing lookup behavior; TTL does not add guards to stale Session objects
+or delayed writes. Redis reporting remains asynchronous and acquires no path locks.
+
+User TTL lives in `settings/user_config.json` and uses the existing
+`GET/PATCH /api/v1/admin/accounts/{account_id}/users/{user_id}/settings` API:
+
+```json
+{"ttl": {"sessions": 30, "user_events": 60, "peer_events": 60}}
+```
+
+For each field, User overrides Account, which overrides global configuration.
+All Peers under one User share `peer_events`. `user_events` and `sessions`
+remain independent. Type defaults precede `global`.
+Missing fields inherit, `disabled` stops fallback, and PATCH `null` removes an
+override. `roots` is no longer part of the cluster/Account configuration schema.
+The existing UserConfig update lock, validation, backup and deletion apply.
+Account configuration path queries also require an existing User; a missing
+User returns 404, matching the User settings endpoint.
+TTL caches at most 1024 user snapshots, keyed by Account and User. A local save
+publishes immediately; other processes refresh on demand after 30 seconds.
+Concurrent misses share one load. This cache adds no periodic filesystem scan.
+
+The hidden ROOT-authenticated `/api/v1/admin/ttl/initial` endpoint registers or
+resets a fixed `initial` scan. Redis keeps its pending marker, checkpoint and done
+marker across OV restarts. Registration never lists business directories;
+`initial.discover=1` defers missing or reset scope discovery to maintenance.
+Reset retains the old roots for policy checks until discovery replaces them.
+The OV writer registers hourly maintenance with the existing expiry scheduler,
+using a stable ResourceID offset. Its TTL consumer starts on first registration;
+Redis retains that activation across restarts. An overdue callback after restart
+schedules the next window instead of starting a scan or recycling. Configuration
+saves, first access and startup cannot bypass this window. OV discovers initial
+roots there, resumes saved checkpoints and recycles in later hourly rounds.
+The callback only schedules the next window and wakes maintenance; it never
+loads UserConfig. Maintenance isolates failures per Account. Completed initial
+scans are not replayed unless reset intent exists. A disabled scope retains the
+hourly check, but its directories are neither backfilled nor deleted.
+Each round uses the existing Account user registry and UserConfig cache to
+compute its TTL cutoff, then queries the directory ZSETs by score in batches.
+It does not enumerate every directory index entry or rely on initial roots to
+find later users. Policy changes during a round may wait until the next round;
+deletion still rechecks current policy. No business directories are listed.
+Existing deletion throttling applies to eligible attempts, including failed or
+renewed Sessions.
+Background directory listings, policy batches and index pages wait
+`batch_interval` (50 ms by default), including Events and already-indexed entries.
+Incremental registration coalesces its first batch for at most 50 ms and paces
+further batches independently. Event summary writes do not enqueue registration;
+cross-batch Event writes may still repeat the idempotent `ZADD NX`.
+Failure backoff includes jitter and is capped at 60 seconds.
+These limits do not bound a single large `ls` or recursive `rm`, or total load
+across libraries sharing Redis. No new global coordinator is introduced.
+Local dev mode has no user registry, so it uses the minimum legal TTL as a
+conservative query bound and checks each candidate's policy before deletion.
+`/api/v1/admin/ttl/recycle` also supports explicit internal requests. The hidden
+GET `/api/v1/admin/ttl/initial` reads internal recovery state; no caller needs a
+Redis connection. TTL policy PATCHes persist reset intent before saving, so the
+next maintenance trigger can recover an interrupted handoff. Commercial Server
+configures the writer and forwards internal repair requests; it does not run the
+hourly scheduler. TTL adds no separate timer or callback queue.
+
+Account deletion uses the existing durable cleanup task to stop its TTL work,
+unregister maintenance, and remove the three directory indexes and backfill
+state. Redis failure keeps the deletion fence for the normal cleanup retry.
+Late callbacks for a deleted Account only repeat cleanup; they do not reschedule
+maintenance or scan the filesystem.
+
+TTL does not inspect archives before delayed writes: no `.meta.json` read,
+source stat or archive URI propagation. Existing Session and queue checks remain
+unchanged; in-flight extraction may finish after its source Session is deleted.
+Vector delivery adds no TTL policy or filesystem check. Deletion still rechecks
+current policy and Session ModTime. Rare late vectors may remain after directory
+recycling; once the directory index is removed, another TTL round is not
+guaranteed to find them. No automatic orphan-vector repair is added.
+
+Retention coverage lives in the existing admin API, storage deletion, Session
+commit and service lifecycle tests, using native local RAGFS and fakeredis with
+Lua support.
diff --git a/openviking/ttl/__init__.py b/openviking/ttl/__init__.py
new file mode 100644
index 000000000..96ced66f9
--- /dev/null
+++ b/openviking/ttl/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Default-off Redis directory TTL, interfaces and service wiring."""
diff --git a/openviking/ttl/adapter.py b/openviking/ttl/adapter.py
new file mode 100644
index 000000000..04b60ebf7
--- /dev/null
+++ b/openviking/ttl/adapter.py
@@ -0,0 +1,145 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Directory TTL contract shared by the runtime, hooks and internal transport."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Mapping,
+ Optional,
+ Protocol,
+ Sequence,
+ runtime_checkable,
+)
+
+if TYPE_CHECKING:
+ from openviking.scheduling.expiry.api import ExpiryScheduler
+ from openviking.server.identity import RequestContext
+
+
+@dataclass(frozen=True)
+class TTLBackfillRequest:
+ """Fixed initial scan: reset=False resumes the saved scope/checkpoint.
+
+ reset=True stops the scan, clears done, resets the checkpoint and registers
+ pending; indexes survive. Omitted roots are discovered at the next hourly
+ window, including after restart. Untagged resets require state/log
+ confirmation after a lost response. Tagged resets are adapter-deduplicated.
+ """
+
+ resource_id: str
+ account_id: str
+ roots: Optional[tuple[str, ...]] = None
+ #: True for explicit internal repair, (re)opening or widening scope; the adapter
+ #: must clear ``done`` / reset the checkpoint before re-running initial.
+ reset: bool = False
+ #: Optional explicit reset intents (scope -> token). The adapter consumes
+ #: matching tokens atomically with reset; replay must not reset twice.
+ reset_intents: Optional[dict[str, str]] = None
+
+
+@dataclass(frozen=True)
+class TTLRecycleRequest:
+ """Writer's hourly request to recycle one scope's due directories.
+
+ Normal triggers supply identity only. The writer resolves the conservative
+ candidate cutoff, initial state and per-directory policy. An explicit cutoff
+ remains available for isolated validation.
+ """
+
+ resource_id: str
+ account_id: str
+ #: Upper bound on the base-time score to scan, in epoch milliseconds.
+ max_base_time_ms: Optional[int] = None
+ limit: int = 256
+
+
+@dataclass(frozen=True)
+class TTLRecycleResult:
+ """Outcome of a single recycle sweep, for writer-side logging/metrics."""
+
+ scanned: int = 0
+ deleted: int = 0
+ skipped: int = 0
+ failed: int = 0
+
+
+@runtime_checkable
+class TTLAdapter(Protocol):
+ """Directory-TTL integration contract (default absent)."""
+
+ def bind_runtime(self, service: Any, account_manager: Any) -> None:
+ """Reuse the service's storage and published configuration lifecycle."""
+
+ def install_expiry(self, scheduler: "ExpiryScheduler") -> None:
+ """Register maintenance on the existing Redis scheduler before roles start."""
+
+ def supports_config(self) -> bool:
+ """Return True when explicit TTL configuration writes are allowed.
+
+ The admin configuration routes keep returning 501 until a loaded
+ adapter reports support; support depends on configuration, not edition.
+ """
+
+ async def patch_configuration(self, settings: dict, account_id: Optional[str] = None) -> None:
+ """Persist reset intent before publishing a TTL configuration change."""
+
+ async def prepare_user_configuration(self, ctx: Any, before: Any, after: Any) -> None:
+ """Confirm durable reset intent before a UserConfig TTL save."""
+
+ async def publish_user_configuration(self, ctx: Any, config: Any) -> None:
+ """Refresh the user's in-process policy after a confirmed save."""
+
+ async def read_initial(self, account_id: str) -> dict:
+ """Read the fixed initial state for internal diagnostics."""
+
+ async def delete_account(self, account_id: str) -> None:
+ """Remove Account indexes, backfill state and maintenance registration."""
+
+ async def on_post_write(
+ self,
+ uri: str,
+ ctx: Optional["RequestContext"],
+ written_at_ms: int,
+ ) -> None:
+ """Buffer a committed write for background directory registration.
+
+ Runs after the storage write is committed and must never turn a
+ successful write into a failure; the kernel already swallows and logs
+ exceptions raised here. The implementation only merges into a
+ bounded local buffer here; Redis registration belongs to its worker.
+ """
+
+ async def register_initial_backfill(self, request: TTLBackfillRequest) -> None:
+ """Register (or reset-and-resume) the fixed initial scan for a scope."""
+
+ async def recycle_due(self, request: TTLRecycleRequest) -> TTLRecycleResult:
+ """Recycle the due directories for one scope under the directory lock."""
+
+ async def resolve_expires_at(
+ self,
+ uris: Sequence[str],
+ ctx: Optional["RequestContext"],
+ ) -> Mapping[str, Optional[str]]:
+ """Batch-compute UTC expiry or None; never persist it or stat for display.
+
+ Missing scores/Redis failure yield None without failing the read.
+ None alone does not imply TTL disabled or permanent retention.
+ """
+
+ async def start(self) -> None:
+ """Start background consumers/connections after the service is ready."""
+
+ async def aclose(self) -> None:
+ """Stop in-flight work and release Redis connections on shutdown."""
+
+
+__all__ = [
+ "TTLAdapter",
+ "TTLBackfillRequest",
+ "TTLRecycleRequest",
+ "TTLRecycleResult",
+]
diff --git a/openviking/ttl/internal.py b/openviking/ttl/internal.py
new file mode 100644
index 000000000..95994ac21
--- /dev/null
+++ b/openviking/ttl/internal.py
@@ -0,0 +1,96 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Internal transport, registered only when Redis TTL is enabled.
+
+No product logic or Redis access belongs here. The Server supplies identity;
+the adapter owns scope discovery, registration, recovery and recycling.
+"""
+
+from dataclasses import asdict
+
+from fastapi import APIRouter, Depends, Request
+
+from openviking.server.auth import get_request_context, require_auth_root
+from openviking.server.identity import RequestContext
+from openviking.server.models import Response
+from openviking.server.routers.admin import _check_account_exists, _get_runtime_config_manager
+from openviking.ttl.adapter import TTLBackfillRequest, TTLRecycleRequest
+from openviking_cli.exceptions import InvalidArgumentError
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+router = APIRouter(prefix="/api/v1/admin/ttl", include_in_schema=False)
+
+
[email protected]("/configuration")
+@require_auth_root
+async def configuration(
+ request: Request,
+ account_id: str,
+ ctx: RequestContext = Depends(get_request_context),
+):
+ """Return one published policy pair, including startup/cluster merging."""
+ await _check_account_exists(request, account_id)
+ policies = await _get_runtime_config_manager().resolve_account(
+ account_id,
+ lambda view: {
+ name: config.ttl.model_dump(by_alias=True) if config.ttl else {}
+ for name, config in (("cluster", view.cluster), ("account", view.account))
+ },
+ )
+ cluster = {
+ "global": "disabled",
+ **{name: value for name, value in policies["cluster"].items() if value is not None},
+ }
+ return Response(status="ok", result={**policies, "settings": {"ttl": cluster}})
+
+
[email protected]("/initial")
+@require_auth_root
+async def initial_state(
+ request: Request,
+ account_id: str,
+ ctx: RequestContext = Depends(get_request_context),
+):
+ """Internal diagnostics use the writer; callers do not connect to Redis."""
+ await _check_account_exists(request, account_id)
+ state = await request.app.state.ttl_adapter.read_initial(account_id)
+ return Response(status="ok", result=state)
+
+
[email protected]("/initial")
+@require_auth_root
+async def initial(
+ body: TTLBackfillRequest,
+ request: Request,
+ ctx: RequestContext = Depends(get_request_context),
+):
+ if not body.resource_id.strip() or not body.account_id.strip():
+ raise InvalidArgumentError("resource_id and account_id are required")
+ await _check_account_exists(request, body.account_id)
+ # Adapter verifies target, current policy, roots and persisted state before
+ # accepting. Token deduplication belongs to the adapter; an untagged manual
+ # reset still requires confirmation before retry after a lost response.
+ await request.app.state.ttl_adapter.register_initial_backfill(body)
+ logger.info(
+ "TTL initial registered request_id=%s scope=%s/%s event=%s",
+ getattr(request.state, "request_id", ""),
+ body.resource_id,
+ body.account_id,
+ "reset" if body.reset else "resume",
+ )
+ return Response(status="ok")
+
+
[email protected]("/recycle")
+@require_auth_root
+async def recycle(
+ body: TTLRecycleRequest,
+ request: Request,
+ ctx: RequestContext = Depends(get_request_context),
+):
+ if not body.resource_id.strip() or not body.account_id.strip() or body.limit <= 0:
+ raise InvalidArgumentError("resource_id, account_id and a positive limit are required")
+ await _check_account_exists(request, body.account_id)
+ result = await request.app.state.ttl_adapter.recycle_due(body)
+ return Response(status="ok", result=asdict(result))
diff --git a/openviking/ttl/runtime.py b/openviking/ttl/runtime.py
new file mode 100644
index 000000000..11fe261c7
--- /dev/null
+++ b/openviking/ttl/runtime.py
@@ -0,0 +1,1101 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Redis directory indexes and expiry-driven maintenance for the OV writer."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import time
+from bisect import bisect_right
+from collections import OrderedDict
+from contextlib import nullcontext
+from datetime import datetime, timedelta, timezone
+from itertools import islice
+from random import uniform
+from typing import TYPE_CHECKING, Any, Sequence
+from urllib.parse import quote
+from uuid import uuid4
+from zoneinfo import ZoneInfo
+
+from openviking.config.ttl import (
+ TTL_ROOT_DEPTH,
+ TTLIndexType,
+ classify_ttl_uri,
+ resolve_effective_ttl,
+ validate_ttl_root,
+)
+from openviking.core.namespace import uri_parts
+from openviking.scheduling.expiry.api import (
+ TTL_ITEM_TYPE,
+ CallbackRequest,
+ CallbackResult,
+ CallbackStatus,
+ ExpiryRegistration,
+ ExpiryScheduler,
+ HandlerOptions,
+ ItemIdentity,
+)
+from openviking.scheduling.expiry.config import ExpiryRedisConfig
+from openviking.scheduling.expiry.redis_store import create_redis_client
+from openviking.server.error_mapping import is_not_found_error
+from openviking.server.identity import RequestContext, Role
+from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher
+from openviking.storage.abstract_overview import ABSTRACT_OVERVIEW_FILENAMES
+from openviking.ttl.adapter import TTLBackfillRequest, TTLRecycleRequest, TTLRecycleResult
+from openviking_cli.exceptions import (
+ ConflictError,
+ FailedPreconditionError,
+ InvalidArgumentError,
+ NotFoundError,
+ UnavailableError,
+)
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.config.ttl_config import TTLConfig
+
+if TYPE_CHECKING:
+ from openviking.server.api_keys import APIKeyManager
+ from openviking.service.core import OpenVikingService
+
+TTLDirectory = tuple[TTLIndexType, str, str, list[str]]
+
+logger = logging.getLogger(__name__)
+DAY_MS = 86_400_000
+# The schema uses Go url.PathEscape, not QueryEscape or Python's default quote.
+PATH_SAFE = "-._~$&+:=@"
+RESET = """
+local intents = cjson.decode(ARGV[3])
+local matched = {}
+for scope, token in pairs(intents) do
+ if redis.call('HGET', KEYS[4], scope) == token then
+ table.insert(matched, scope)
+ end
+end
+-- An acknowledged or superseded automatic reset must not erase new progress.
+if next(intents) and #matched == 0 then return 2 end
+redis.call('DEL', KEYS[2])
+redis.call('HSET', KEYS[1], 'roots', ARGV[1], 'checkpoint', ARGV[2], 'discover', ARGV[4])
+redis.call('SADD', KEYS[3], 'initial')
+if ARGV[4] == '0' then
+ for _, scope in ipairs(matched) do redis.call('HDEL', KEYS[4], scope) end
+end
+return 1
+"""
+INITIAL = """
+local roots = redis.call('HGET', KEYS[1], 'roots')
+local discover = redis.call('HGET', KEYS[1], 'discover') == '1'
+if roots and roots ~= ARGV[1] and not discover then return 0 end
+if not roots or discover then redis.call('HSET', KEYS[1], 'roots', ARGV[1]) end
+redis.call('HSETNX', KEYS[1], 'checkpoint', ARGV[2])
+redis.call('HSET', KEYS[1], 'discover', ARGV[4])
+if not redis.call('GET', KEYS[2]) then redis.call('SADD', KEYS[3], 'initial') end
+return 1
+"""
+REMOVE = """
+local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
+if score and tonumber(score) == tonumber(ARGV[2]) then
+ return redis.call('ZREM', KEYS[1], ARGV[1])
+end
+return 0
+"""
+READ_INITIAL = """
+local intents = {}
+local fields = redis.call('HGETALL', KEYS[4])
+for i = 1, #fields, 2 do intents[fields[i]] = fields[i + 1] end
+local encoded = '{}'
+if #fields > 0 then encoded = cjson.encode(intents) end
+return {redis.call('HGET', KEYS[1], 'roots') or '',
+ redis.call('HGET', KEYS[1], 'checkpoint') or '',
+ redis.call('GET', KEYS[2]) or '',
+ tostring(redis.call('SISMEMBER', KEYS[3], 'initial')), encoded,
+ redis.call('HGET', KEYS[1], 'discover') or '0'}
+"""
+
+
+def directory(uri: str) -> TTLDirectory | None:
+ """Return (index type, policy root, recycle unit, date components)."""
+ try:
+ parts = uri_parts(uri)
+ except (TypeError, ValueError):
+ return None
+ kind = classify_ttl_uri(uri)
+ if kind is None:
+ return None
+ root_depth = TTL_ROOT_DEPTH[kind]
+ root = "viking://" + "/".join(parts[:root_depth])
+ unit_depth = 1 if kind == "sessions" else 3
+ if len(parts) < root_depth + unit_depth:
+ return None
+ date_parts = parts[root_depth : root_depth + unit_depth]
+ if kind == "sessions" and date_parts[0] in ABSTRACT_OVERVIEW_FILENAMES:
+ return None
+ if kind != "sessions":
+ if not re.fullmatch(r"\d{4}/\d{2}/\d{2}", "/".join(date_parts)):
+ return None
+ try:
+ datetime(*map(int, date_parts))
+ except ValueError:
+ return None
+ return kind, root, root + "/" + "/".join(date_parts), date_parts
+
+
+def next_check_ms(resource_id: str, now_ms: int) -> int:
+ """Preserve the Server's FNV-1a32 hourly offset; skip missed windows."""
+ value = 2166136261
+ for byte in resource_id.encode():
+ value = ((value ^ byte) * 16777619) & 0xFFFFFFFF
+ hour = 3_600_000
+ due = now_ms // hour * hour + value % 3600 * 1000
+ return due if due > now_ms else due + hour
+
+
+class DirectoryTTL:
+ def __init__(
+ self,
+ *,
+ resource_id: str,
+ redis: dict[str, Any] | None = None,
+ writer: bool = True,
+ timezone_name: str | None = None,
+ batch_size: int = 256,
+ batch_interval: float = 0.05,
+ stat_rate: int = 200,
+ delete_interval: float = 0.05,
+ read_timeout: float = 0.2,
+ retry_limit: int = 10000,
+ ) -> None:
+ if not resource_id or not 1 <= batch_size <= 512 or stat_rate <= 0 or batch_interval < 0:
+ raise ValueError("invalid TTL resource or rate limits")
+ self.resource_id = resource_id
+ self.redis_config = ExpiryRedisConfig.model_validate(redis or {})
+ self.writer = writer
+ # With no override, naive date timestamps use the writer's existing
+ # local timezone, just like its date-directory generation fallback.
+ self.zone = ZoneInfo(timezone_name) if timezone_name else None
+ self.batch_size, self.stat_rate = batch_size, stat_rate
+ self.batch_interval = batch_interval
+ self.delete_interval, self.read_timeout = delete_interval, read_timeout
+ self.retry_limit = retry_limit
+ self.redis = None
+ self._task = None
+ self._registration_task = None
+ self._started = False
+ self._closed = False
+ self._wake = asyncio.Event()
+ self._registration_wake = asyncio.Event()
+ self._pending = set()
+ self._locks = {}
+ self._stop_scan = set()
+ self._retry = {}
+ self.registration_dropped_total = 0
+ self._recycle_lock = asyncio.Lock()
+ self._registration_lock = asyncio.Lock()
+ self._scheduler: ExpiryScheduler | None = None
+ self._scheduled: set[str] = set()
+ self._recycle: set[str] = set()
+ self._recycle_after_ms = 0
+ self._dispatcher = OwnerLoopDispatcher()
+ self._user_policies = OrderedDict()
+ self._user_loads = {}
+
+ def bind_runtime(self, service: OpenVikingService, account_manager: APIKeyManager) -> None:
+ # Bind before queue workers can invoke the write hooks.
+ self._dispatcher.bind_current_loop()
+ self.fs = service.viking_fs
+ self.manager = service.runtime_config_manager
+ self.account_manager = account_manager
+
+ def supports_config(self) -> bool:
+ return self.writer
+
+ def install_expiry(self, scheduler: ExpiryScheduler) -> None:
+ self._scheduler = scheduler
+ scheduler.register_handler(
+ TTL_ITEM_TYPE,
+ self,
+ options=HandlerOptions(
+ concurrency=1, max_attempts=None, metadata_enabled=False, lazy_start=True
+ ),
+ )
+
+ @staticmethod
+ def _identity(account: str) -> ItemIdentity:
+ return ItemIdentity(TTL_ITEM_TYPE, account, "_system", "maintenance")
+
+ async def _schedule(self, account: str, *, force: bool = False) -> bool:
+ if self._scheduler is None or (account in self._scheduled and not force):
+ return True
+ async with self._registration_lock:
+ manager = getattr(self, "account_manager", None)
+ info = await manager.get_account(account) if manager is not None else {}
+ if info is not None and info.get("status") != "deleting":
+ await self._scheduler.report(
+ ExpiryRegistration(
+ self._identity(account),
+ due_at_ms=next_check_ms(self.resource_id, time.time_ns() // 1_000_000),
+ timeout_seconds=3600,
+ )
+ )
+ self._scheduled.add(account)
+ return True
+ await self.delete_account(account)
+ return False
+
+ async def delete_account(self, account: str) -> None:
+ """Retire TTL state through the existing durable Account deletion task."""
+ self._stop_scan.add(account)
+ try:
+ async with (
+ self._locks.setdefault(account, asyncio.Lock()),
+ self._recycle_lock,
+ self._registration_lock,
+ ):
+ self._scheduled.discard(account)
+ self._pending.discard(account)
+ self._recycle.discard(account)
+ for key in list(self._retry):
+ if key[0] == account:
+ self._retry.pop(key, None)
+ for key in list(self._user_policies):
+ if key[0] == account:
+ self._user_policies.pop(key, None)
+ if self._scheduler is not None:
+ await self._scheduler.unregister(self._identity(account))
+ redis = await self.client()
+ prefix = self.prefix(account)
+ await redis.unlink(
+ *(prefix + kind for kind in TTLIndexType),
+ *(
+ prefix + "backfill:" + name
+ for name in ("initial", "initial:done", "pending", "reset_required")
+ ),
+ )
+ finally:
+ self._stop_scan.discard(account)
+ logger.info("TTL account state removed resource=%s account=%s", self.resource_id, account)
+
+ async def handle(self, request: CallbackRequest) -> CallbackResult:
+ if self._closed:
+ return CallbackResult(CallbackStatus.RETRY, "stopping")
+ account = request.identity.account_id
+ # Confirm the next durable wake-up before acknowledging this one.
+ # Long-running filesystem work stays outside the short expiry callback.
+ if not await self._schedule(account, force=True):
+ return CallbackResult(CallbackStatus.COMPLETED, "account_deleted")
+ if time.time_ns() // 1_000_000 >= self._recycle_after_ms:
+ self._recycle.add(account)
+ self._wake_worker()
+ return CallbackResult(CallbackStatus.COMPLETED, "maintenance_scheduled")
+
+ def _wake_worker(self) -> None:
+ self._wake.set()
+ if self.writer and self._started and not self._closed and self._recycle:
+ if self._task is None or self._task.done():
+ self._task = asyncio.create_task(self._run(), name="ttl-maintenance")
+
+ def _wake_registrations(self) -> None:
+ self._registration_wake.set()
+ if self.writer and self._started and not self._closed:
+ if self._registration_task is None or self._registration_task.done():
+ self._registration_task = asyncio.create_task(
+ self._flush_registrations(), name="ttl-registration"
+ )
+
+ async def _flush_registrations(self) -> None:
+ # Coalesce a commit's writes without delaying the business request.
+ await asyncio.sleep(min(self.batch_interval, 0.05))
+ delay = 1
+ while self._retry and not self._closed:
+ self._registration_wake.clear()
+ try:
+ await self._retry_writes()
+ except Exception:
+ logger.exception("TTL registration retry delay=%s", delay)
+ try:
+ await asyncio.wait_for(
+ self._registration_wake.wait(), uniform(delay / 2, delay)
+ )
+ except TimeoutError:
+ pass
+ delay = min(delay * 2, 60)
+ else:
+ delay = 1
+ if self._retry:
+ await asyncio.sleep(self.batch_interval)
+
+ async def patch_configuration(self, settings: dict, account_id: str | None = None) -> None:
+ patch = settings["ttl"]
+
+ def may_enable(value):
+ if value is None:
+ return True
+ if isinstance(value, dict):
+ return any(may_enable(node) for node in value.values())
+ return value != "disabled"
+
+ accounts = []
+ if may_enable(patch):
+ accounts = (
+ [{"account_id": account_id}]
+ if account_id is not None
+ else await self.account_manager.list_accounts()
+ )
+ for account in accounts:
+ account = account["account_id"]
+ pair = await self.policies(account)
+ roots = set(self.policy_roots())
+ scopes = [root for root in roots if not resolve_effective_ttl(*pair, root).enabled]
+ await self._register_reset_intent(account, scopes)
+ if account_id is None:
+ await self.manager.patch_cluster(settings)
+ else:
+ await self.manager.patch_account(account_id, settings)
+
+ async def _register_reset_intent(self, account: str, scopes: Sequence[str]) -> None:
+ try:
+ if scopes:
+ redis = await self.client()
+ pipe = redis.pipeline(transaction=False)
+ token = uuid4().hex
+ for scope in scopes:
+ pipe.hsetnx(self.prefix(account) + "backfill:reset_required", scope, token)
+ await pipe.execute()
+ if not await self._schedule(account):
+ raise NotFoundError(account, "account")
+ except Exception as exc:
+ raise UnavailableError("TTL reset intent registration unavailable") from exc
+
+ async def read_initial(self, account_id: str) -> dict:
+ prefix = self.prefix(account_id) + "backfill:"
+ try:
+ redis = await self.client()
+ values = await redis.eval(
+ READ_INITIAL,
+ 4,
+ prefix + "initial",
+ prefix + "initial:done",
+ prefix + "pending",
+ prefix + "reset_required",
+ )
+ return {
+ "roots": values[0],
+ "checkpoint": values[1],
+ "done": values[2] == "1",
+ "pending": values[3] == "1",
+ "reset_required": json.loads(values[4]),
+ "discover": values[5] == "1",
+ }
+ except Exception as exc:
+ raise UnavailableError("TTL initial state query unavailable") from exc
+
+ async def _initial_roots(self, account: str, pair) -> list[str]:
+ ctx = self.context(account)
+ roots = set()
+ for i, user in enumerate(await self._children("viking://user", ctx)):
+ if i % self.batch_size == 0:
+ await asyncio.sleep(self.batch_interval)
+ config = await self.user_policy(account, uri_parts(user)[1])
+ for root in (user + "/memories/events", user + "/sessions"):
+ if resolve_effective_ttl(*pair, root, config).enabled:
+ roots.add(root)
+ probe = user + "/peers"
+ if resolve_effective_ttl(*pair, probe, config).enabled:
+ for peer in await self._children(user + "/peers", ctx):
+ roots.add(peer + "/memories/events")
+ return sorted(roots)
+
+ def prefix(self, account_id: str) -> str:
+ scope = quote(self.resource_id, safe=PATH_SAFE) + "|" + quote(account_id, safe=PATH_SAFE)
+ return "ttl:{" + scope + "}:"
+
+ def context(self, account_id: str) -> RequestContext:
+ return RequestContext(user=UserIdentifier(account_id, "ttl-internal"), role=Role.ROOT)
+
+ @staticmethod
+ def user_roots(user: str) -> list[str]:
+ base = f"viking://user/{user}"
+ return [
+ base + "/sessions",
+ base + "/memories/events",
+ base + "/peers",
+ ]
+
+ def _cache_user_policy(self, key, config) -> None:
+ self._user_policies[key] = (time.monotonic(), config)
+ self._user_policies.move_to_end(key)
+ while len(self._user_policies) > 1024:
+ self._user_policies.popitem(last=False)
+
+ async def user_policy(self, account: str, user: str) -> TTLConfig | None:
+ """Share one snapshot per user; refresh on demand, not once per directory."""
+ key = (account, user)
+ cached = self._user_policies.get(key)
+ if cached is not None and time.monotonic() - cached[0] < 30:
+ return cached[1]
+
+ async def load():
+ cached = self._user_policies.get(key)
+ if cached is not None and time.monotonic() - cached[0] < 30:
+ return cached[1]
+ task = self._user_loads.get(key)
+ if task is None:
+ from openviking.server.user_config import read_user_config
+
+ async def read():
+ try:
+ ctx = RequestContext(user=UserIdentifier(account, user), role=Role.ROOT)
+ value = (await read_user_config(self.fs, ctx)).ttl
+ if self._user_loads.get(key) is not asyncio.current_task():
+ return await self.user_policy(account, user)
+ # A PATCH published while this read was in flight wins.
+ if self._user_policies.get(key) is cached:
+ self._cache_user_policy(key, value)
+ return self._user_policies.get(key, (0, value))[1]
+ finally:
+ if self._user_loads.get(key) is asyncio.current_task():
+ self._user_loads.pop(key, None)
+
+ task = asyncio.create_task(read())
+ self._user_loads[key] = task
+ return await asyncio.shield(task)
+
+ return await self._dispatcher.run(load)
+
+ async def policy(self, account: str, uri: str, pair=None):
+ pair = pair if pair is not None else await self.policies(account)
+ parts = uri_parts(uri)
+ user = (
+ await self.user_policy(account, parts[1]) if classify_ttl_uri(uri) is not None else None
+ )
+ return resolve_effective_ttl(*pair, uri, user)
+
+ async def _initial_enabled(self, account: str, pair, state) -> bool:
+ if self.enabled(pair):
+ return True
+ roots = set(json.loads(state["roots"]) if state["roots"] else [])
+ roots.update(state["reset_required"])
+ for (scope, user), (_, config) in list(self._user_policies.items()):
+ if scope == account and config is not None:
+ roots.update(self.user_roots(user))
+ for root in sorted(roots):
+ if (await self.policy(account, root, pair)).enabled:
+ return True
+ return False
+
+ async def prepare_user_configuration(self, ctx, before, after) -> None:
+ pair = await self.policies(ctx.account_id)
+ opening = [
+ root
+ for root in self.user_roots(ctx.user.user_id)
+ if resolve_effective_ttl(*pair, root, after).enabled
+ and not resolve_effective_ttl(*pair, root, before).enabled
+ ]
+ if not opening:
+ return
+ await self._register_reset_intent(ctx.account_id, opening)
+
+ async def publish_user_configuration(self, ctx, config) -> None:
+ async def publish():
+ key = (ctx.account_id, ctx.user.user_id)
+ self._user_loads.pop(key, None)
+ self._cache_user_policy(key, config)
+
+ await self._dispatcher.run(publish)
+
+ async def policies(self, account_id: str) -> tuple[TTLConfig | None, TTLConfig | None]:
+ return await self.manager.resolve_account(
+ account_id, lambda v: (v.cluster.ttl, v.account.ttl)
+ )
+
+ @classmethod
+ def policy_roots(cls) -> list[str]:
+ return cls.user_roots("_ttl_")
+
+ @classmethod
+ def enabled(cls, pair: tuple[TTLConfig | None, TTLConfig | None]) -> bool:
+ return any(resolve_effective_ttl(*pair, uri).enabled for uri in cls.policy_roots())
+
+ def expiry(self, item: TTLDirectory, base: float, days: int) -> float:
+ if item[0] == "sessions":
+ return base + days * DAY_MS
+ dt = datetime(*map(int, item[3]), tzinfo=self.zone) + timedelta(days=days)
+ return int(dt.timestamp() * 1000)
+
+ def base(self, item: TTLDirectory) -> int:
+ return int(datetime(*map(int, item[3]), tzinfo=self.zone).timestamp() * 1000)
+
+ async def client(self):
+ # No connection or ping at startup or on a disabled-account path.
+ if self.redis is None:
+ from redis.backoff import NoBackoff
+ from redis.retry import Retry
+
+ if not self.redis_config.endpoints:
+ raise UnavailableError("TTL Redis is not configured")
+ self.redis = create_redis_client(self.redis_config, retry=Retry(NoBackoff(), 0))
+ return self.redis
+
+ async def start(self) -> None:
+ self._dispatcher.bind_current_loop()
+ self._closed = False
+ self._started = True
+ self._recycle_after_ms = next_check_ms(self.resource_id, time.time_ns() // 1_000_000)
+ if self._retry:
+ self._wake_registrations()
+
+ async def aclose(self) -> None:
+ self._closed = True
+ self._started = False
+ self._wake.set()
+ self._registration_wake.set()
+ if self._registration_task:
+ await self._registration_task
+ self._registration_task = None
+ if self._task:
+ await self._task # Finish in-flight I/O; never cancel a Redis checkpoint write.
+ self._task = None
+ if self._retry:
+ try:
+ async with asyncio.timeout(2):
+ while self._retry:
+ await self._retry_writes(closing=True)
+ except Exception:
+ logger.warning(
+ "TTL shutdown left volatile registrations pending=%s",
+ len(self._retry),
+ exc_info=True,
+ )
+ async with self._recycle_lock:
+ if self.redis:
+ await self.redis.aclose()
+ self.redis = None
+
+ async def on_post_write(self, uri: str, ctx: RequestContext | None, written_at_ms: int) -> None:
+ if not self.writer or ctx is None:
+ return
+ item = directory(uri)
+ if item is None or (item[0] == "sessions" and uri != item[2] + "/messages.jsonl"):
+ return
+ if item[0] != "sessions" and uri.rsplit("/", 1)[-1] in ABSTRACT_OVERVIEW_FILENAMES:
+ return
+ base = written_at_ms if item[0] == "sessions" else self.base(item)
+ await self._dispatcher.run(
+ lambda: self._enqueue_registration((ctx.account_id, item[0], item[2]), base)
+ )
+
+ async def _enqueue_registration(self, key: tuple[str, TTLIndexType, str], base: int) -> None:
+ # Only an in-memory merge on the write path. The worker owns Redis I/O.
+ old = self._retry.get(key)
+ if old is None and len(self._retry) >= self.retry_limit:
+ self.registration_dropped_total += 1
+ logger.error(
+ "TTL event=registration_dropped scope=%s uri=%s pending=%s dropped_total=%s",
+ key[0],
+ key[2],
+ len(self._retry),
+ self.registration_dropped_total,
+ )
+ return
+ was_empty = not self._retry
+ self._retry[key] = max(base, old) if old is not None else base
+ # Updates to a failed batch must not defeat the worker's retry backoff.
+ if was_empty:
+ self._wake_registrations()
+
+ async def _register(self, key, base):
+ account, kind, uri = key
+ redis = await self.client()
+ await redis.zadd(
+ self.prefix(account) + kind,
+ {uri: base},
+ **({"gt": True} if kind == "sessions" else {"nx": True}),
+ )
+
+ async def _retry_writes(self, *, closing=False):
+ groups = {}
+ for (account, kind, uri), base in islice(self._retry.items(), min(self.batch_size, 256)):
+ groups.setdefault((account, kind), {})[uri] = base
+ error = None
+ for (account, kind), entries in groups.items():
+ if self._closed and not closing:
+ return
+ try:
+ async with self._registration_lock:
+ pair = await self.policies(account)
+ active = {
+ uri: base
+ for uri, base in entries.items()
+ if self._retry.get((account, kind, uri)) == base
+ and (await self.policy(account, uri, pair)).enabled
+ }
+ if active:
+ redis = await self.client()
+ await redis.zadd(
+ self.prefix(account) + kind,
+ active,
+ **({"gt": True} if kind == "sessions" else {"nx": True}),
+ )
+ if active:
+ await self._schedule(account)
+ # Also discard disabled entries; do not remove a concurrent update.
+ for uri, base in entries.items():
+ key = (account, kind, uri)
+ if self._retry.get(key) == base:
+ self._retry.pop(key, None)
+ except Exception as exc:
+ error = exc
+ # Do not let one failed account monopolize the bounded head batch.
+ for uri in entries:
+ key = (account, kind, uri)
+ if key in self._retry:
+ current = self._retry.pop(key)
+ self._retry[key] = current
+ logger.exception("TTL batch registration failed scope=%s kind=%s", account, kind)
+ if error is not None:
+ raise error
+
+ async def register_initial_backfill(self, request: TTLBackfillRequest) -> None:
+ if not self.writer or request.resource_id != self.resource_id:
+ raise FailedPreconditionError("TTL request must target this writer")
+ account = request.account_id
+ pair = await self.policies(account)
+ state = await self.read_initial(account)
+ if not await self._initial_enabled(account, pair, state):
+ raise FailedPreconditionError("TTL is disabled")
+ await self._register_initial(request, pair, state, discover=False)
+
+ async def _register_initial(
+ self, request: TTLBackfillRequest, pair, state, *, discover=True
+ ) -> None:
+ account = request.account_id
+ if not await self._schedule(account):
+ raise FailedPreconditionError("TTL Account is unavailable")
+ intents = request.reset_intents or {
+ root: token
+ for root, token in state["reset_required"].items()
+ if (await self.policy(account, root, pair)).enabled
+ }
+ reset = request.reset or bool(intents)
+ roots = request.roots
+ if not reset and not state["roots"] and (state["pending"] or state["done"]):
+ raise FailedPreconditionError("initial state is missing roots")
+ needs_discovery = roots is None and (reset or state["discover"] or not state["roots"])
+ if roots is None:
+ if needs_discovery and discover:
+ roots = await self._initial_roots(account, pair)
+ needs_discovery = False
+ elif state["roots"]:
+ try:
+ roots = json.loads(state["roots"])
+ except ValueError as exc:
+ raise FailedPreconditionError("invalid saved initial roots") from exc
+ if not isinstance(roots, list):
+ raise FailedPreconditionError("invalid saved initial roots")
+ else:
+ roots = []
+ roots = sorted(set(roots))
+ try:
+ for root in roots:
+ validate_ttl_root(root)
+ except ValueError as exc:
+ raise InvalidArgumentError(
+ "initial roots must be canonical Events or Sessions roots"
+ ) from exc
+ if request.reset_intents and not request.reset:
+ raise InvalidArgumentError("reset intents require reset")
+ # Configuration can change while discovery yields. Leave
+ # these intents pending rather than acknowledge a scan of closed roots.
+ active_intents = {
+ root: token
+ for root, token in intents.items()
+ if (await self.policy(account, root, pair)).enabled
+ }
+ if intents and not active_intents:
+ return
+ if not reset and state["done"]:
+ return
+ if reset:
+ self._stop_scan.add(account)
+ try:
+ async with self._locks.setdefault(account, asyncio.Lock()):
+ async with self._recycle_lock if reset else nullcontext():
+ prefix = self.prefix(account) + "backfill:"
+ redis = await self.client()
+ checkpoint = json.dumps({"root": 0, "stack": []}, separators=(",", ":"))
+ try:
+ result = await redis.eval(
+ RESET if reset else INITIAL,
+ 4,
+ prefix + "initial",
+ prefix + "initial:done",
+ prefix + "pending",
+ prefix + "reset_required",
+ json.dumps(roots, separators=(",", ":")),
+ checkpoint,
+ json.dumps(active_intents, separators=(",", ":")),
+ "1" if needs_discovery else "0",
+ )
+ except Exception as exc:
+ raise UnavailableError(
+ "TTL initial registration failed; query state before retry"
+ ) from exc
+ if not result:
+ raise ConflictError("initial roots changed; reset is required")
+ if result == 2:
+ return
+ self._pending.add(account)
+ finally:
+ self._stop_scan.discard(account)
+ logger.info(
+ "TTL initial registered scope=%s/%s event=%s",
+ self.resource_id,
+ account,
+ "reset" if reset else "resume",
+ )
+
+ async def _resume_initial(self, account: str, *, discover=True) -> None:
+ state = await self.read_initial(account)
+ if state["done"] and not state["reset_required"]:
+ return
+ pair = await self.policies(account)
+ if not await self._initial_enabled(account, pair, state):
+ self._pending.discard(account)
+ return
+ await self._register_initial(
+ TTLBackfillRequest(self.resource_id, account), pair, state, discover=discover
+ )
+
+ async def _run(self):
+ delay = 1
+ while not self._closed:
+ self._wake.clear()
+ failed = False
+ # Only expiry callbacks admit Accounts to a maintenance round.
+ for account in self._recycle.copy():
+ if self._closed:
+ break
+ try:
+ await self._resume_initial(account)
+ if account in self._pending and not self._closed:
+ async with self._locks.setdefault(account, asyncio.Lock()):
+ await self._scan(account)
+ elif not self._closed:
+ result = await self._recycle_registered(
+ TTLRecycleRequest(self.resource_id, account)
+ )
+ logger.info(
+ "TTL recycle resource=%s account=%s result=%s",
+ self.resource_id,
+ account,
+ result,
+ )
+ self._recycle.discard(account)
+ except Exception:
+ failed = True
+ logger.exception("TTL worker retry scope=%s delay=%s", account, delay)
+ delay = min(delay * 2, 60) if failed else 1
+ if not self._recycle:
+ return
+ try:
+ await asyncio.wait_for(self._wake.wait(), uniform(delay / 2, delay))
+ except TimeoutError:
+ pass
+
+ async def _children(self, uri, ctx):
+ await asyncio.sleep(self.batch_interval)
+ try:
+ entries = await self.fs.ls(
+ uri,
+ ctx=ctx,
+ node_limit=None,
+ include_abstract=False,
+ show_all_hidden=True,
+ )
+ except Exception as exc:
+ if is_not_found_error(exc):
+ return []
+ raise
+ return sorted(e["uri"].rstrip("/") for e in entries if e.get("isDir"))
+
+ async def _scan(self, account):
+ redis = await self.client()
+ prefix = self.prefix(account) + "backfill:"
+ if not await redis.sismember(prefix + "pending", "initial"):
+ self._pending.discard(account)
+ return
+ if await redis.get(prefix + "initial:done") == "1":
+ await redis.srem(prefix + "pending", "initial")
+ self._pending.discard(account)
+ return
+ state = await redis.hgetall(prefix + "initial")
+ if state.get("discover") == "1":
+ return # Scope discovery belongs to the scheduled maintenance round.
+ if not state.get("roots") or not state.get("checkpoint"):
+ raise FailedPreconditionError("pending initial has incomplete state")
+ roots = json.loads(state["roots"])
+ checkpoint = json.loads(state["checkpoint"])
+ ctx = self.context(account)
+ listings = {}
+ while checkpoint["root"] < len(roots):
+ if self._closed or account in self._stop_scan:
+ return
+ root = roots[checkpoint["root"]]
+ pair = await self.policies(account)
+ if not (await self.policy(account, root, pair)).enabled:
+ # A disabled root does not block other enabled roots in the account.
+ # Reopening this root requires the persisted initial reset.
+ checkpoint["root"] += 1
+ checkpoint["stack"] = []
+ listings.clear()
+ await redis.hset(
+ prefix + "initial",
+ "checkpoint",
+ json.dumps(checkpoint, separators=(",", ":")),
+ )
+ continue
+ stack = checkpoint["stack"]
+ if not stack:
+ stack.append({"uri": root, "last": ""})
+ frame = stack[-1]
+ if frame["uri"] not in listings:
+ listings[frame["uri"]] = await self._children(frame["uri"], ctx)
+ cached = listings[frame["uri"]]
+ start = bisect_right(cached, frame["last"])
+ if start == len(cached):
+ listings.pop(frame["uri"])
+ stack.pop()
+ if not stack:
+ checkpoint["root"] += 1
+ await redis.hset(
+ prefix + "initial",
+ "checkpoint",
+ json.dumps(checkpoint, separators=(",", ":")),
+ )
+ continue
+ # Stop at Session or valid date directories. Never list their contents.
+ leaf = root.endswith("/sessions") or len(stack) == 3
+ if not leaf:
+ child = cached[start]
+ frame["last"] = child
+ stack.append({"uri": child, "last": ""})
+ continue
+ batch = cached[start : start + self.batch_size]
+ items = [(u, directory(u)) for u in batch]
+ items = [(u, item) for u, item in items if item is not None]
+ if items:
+ await asyncio.sleep(self.batch_interval)
+ index = self.prefix(account) + items[0][1][0]
+ scores = await redis.zmscore(index, [u for u, _ in items])
+ additions = {}
+ for (uri, item), score in zip(items, scores, strict=True):
+ if score is not None:
+ continue
+ if item[0] == "sessions":
+ await asyncio.sleep(1 / self.stat_rate)
+ try:
+ stat = await self.fs.stat(uri + "/messages.jsonl", ctx=ctx)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ continue
+ raise
+ additions[uri] = self.modtime(stat)
+ else:
+ additions[uri] = self.base(item)
+ if additions:
+ await redis.zadd(index, additions, nx=True)
+ # A failed/unknown ZADD must never advance the persisted checkpoint.
+ frame["last"] = batch[-1]
+ await redis.hset(
+ prefix + "initial",
+ "checkpoint",
+ json.dumps(checkpoint, separators=(",", ":")),
+ )
+ if self._closed or account in self._stop_scan:
+ return
+ await redis.set(prefix + "initial:done", "1")
+ await redis.srem(prefix + "pending", "initial")
+ self._pending.discard(account)
+ logger.info("TTL initial scope=%s event=done", account)
+
+ @staticmethod
+ def modtime(stat):
+ value = stat.get("modTime") or stat.get("ModTime")
+ if not isinstance(value, str):
+ raise ValueError("messages.jsonl has no valid ModTime")
+ return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000)
+
+ async def resolve_expires_at(
+ self, uris: Sequence[str], ctx: RequestContext | None
+ ) -> dict[str, str | None]:
+ result = dict.fromkeys(uris)
+ if ctx is None:
+ return result
+ pair = await self.policies(ctx.account_id)
+ sessions = {}
+ now = {}
+ for uri in uris:
+ item = directory(uri)
+ if item is None:
+ continue
+ policy = await self.policy(ctx.account_id, item[1], pair)
+ if not policy.enabled:
+ continue
+ if item[0] == "sessions":
+ sessions.setdefault(item[2], []).append((uri, item, policy.days))
+ else:
+ now[uri] = self.expiry(item, self.base(item), policy.days)
+ if sessions:
+ try:
+ async with asyncio.timeout(self.read_timeout):
+ redis = await self.client()
+ scores = await redis.zmscore(
+ self.prefix(ctx.account_id) + "sessions", list(sessions)
+ )
+ for entries, score in zip(sessions.values(), scores, strict=True):
+ if score is not None:
+ for uri, item, days in entries:
+ now[uri] = self.expiry(item, score, days)
+ except Exception:
+ logger.warning(
+ "TTL expiry read unavailable scope=%s",
+ ctx.account_id,
+ exc_info=True,
+ )
+ for uri, ms in now.items():
+ result[uri] = (
+ datetime.fromtimestamp(ms / 1000, timezone.utc).isoformat().replace("+00:00", "Z")
+ )
+ return result
+
+ async def recycle_due(self, request: TTLRecycleRequest) -> TTLRecycleResult:
+ if not self.writer or request.resource_id != self.resource_id:
+ raise FailedPreconditionError("TTL request must target this writer")
+ if request.max_base_time_ms is None:
+ await self._resume_initial(request.account_id, discover=False)
+ return await self._recycle_registered(request)
+
+ async def _recycle_registered(self, request: TTLRecycleRequest) -> TTLRecycleResult:
+ if self._recycle_lock.locked():
+ raise ConflictError("TTL recycle round is already running")
+ counts = {"scanned": 0, "deleted": 0, "skipped": 0, "failed": 0}
+ account = request.account_id
+ async with self._recycle_lock:
+ redis = await self.client()
+ prefix = self.prefix(account)
+ if await redis.get(prefix + "backfill:initial:done") != "1":
+ return TTLRecycleResult()
+ cutoff = request.max_base_time_ms
+ if cutoff is None:
+ if not await redis.exists(*(prefix + kind for kind in TTLIndexType)):
+ return TTLRecycleResult()
+ cutoff = await self._recycle_cutoff(account)
+ if cutoff is None:
+ return TTLRecycleResult()
+ ctx = self.context(account)
+ for kind in ("user_events", "peer_events", "sessions"):
+ offset = 0
+ while not self._closed:
+ await asyncio.sleep(self.batch_interval)
+ candidates = await redis.zrangebyscore(
+ prefix + kind,
+ "-inf",
+ cutoff,
+ start=offset,
+ num=min(request.limit, self.batch_size),
+ withscores=True,
+ )
+ if not candidates:
+ break
+ if await redis.get(prefix + "backfill:initial:done") != "1":
+ return TTLRecycleResult(**counts)
+ removed = 0
+ for uri, score in candidates:
+ if self._closed or account in self._stop_scan:
+ return TTLRecycleResult(**counts)
+ counts["scanned"] += 1
+ try:
+ did_delete, did_remove = await self._delete(account, uri, score, ctx)
+ counts["deleted" if did_delete else "skipped"] += 1
+ removed += did_remove
+ except Exception:
+ counts["failed"] += 1
+ logger.exception("TTL recycle failed scope=%s uri=%s", account, uri)
+ offset += len(candidates) - removed
+ return TTLRecycleResult(**counts)
+
+ async def _recycle_cutoff(self, account: str) -> int | None:
+ # Use the current user registry, not initial roots or an incomplete cache.
+ # Changes after this snapshot enter the next hourly round.
+ pair = await self.policies(account)
+ roots = self.policy_roots()
+
+ def minimum(config, days=None):
+ for root in roots:
+ value = resolve_effective_ttl(*pair, root, config).days
+ if value is not None and (days is None or value < days):
+ days = value
+ return days
+
+ days = minimum(None)
+ if days != 1:
+ manager = getattr(self, "account_manager", None)
+ if manager is None:
+ # Local dev mode has no user registry; use the safe lower bound.
+ return time.time_ns() // 1_000_000
+ users = await manager.list_users_page(account, limit=None, expose_key=False)
+ for i, user in enumerate(users["users"]):
+ if i % self.batch_size == 0:
+ await asyncio.sleep(self.batch_interval)
+ days = minimum(await self.user_policy(account, user["user_id"]), days)
+ if days == 1:
+ break
+ if days is None:
+ return None
+ return time.time_ns() // 1_000_000 - (days - 1) * DAY_MS
+
+ async def _delete(self, account, uri, score, ctx):
+ item = directory(uri)
+ if item is None or item[2] != uri:
+ return False, 0
+ # Every valid TTL is at least one day. Skip young entries without loading
+ # user configuration; Events retain their existing calendar-day semantics.
+ if self.expiry(item, score, 1) > time.time_ns() // 1_000_000:
+ return False, 0
+ policy = await self.policy(account, item[1])
+ if (
+ not policy.enabled
+ or self.expiry(item, score, policy.days) > time.time_ns() // 1_000_000
+ ):
+ return False, 0
+ await asyncio.sleep(self.delete_interval)
+ path = self.fs._uri_to_path(uri, ctx=ctx)
+ agfs = self.fs._async_agfs
+ lease = await agfs.pathlock_acquire_tree(path)
+ try:
+ redis = await self.client()
+ if await redis.get(self.prefix(account) + "backfill:initial:done") != "1":
+ return False, 0
+ policy = await self.policy(account, item[1])
+ if not policy.enabled:
+ return False, 0
+ base = score
+ if item[0] == "sessions":
+ try:
+ base = self.modtime(await self.fs.stat(uri + "/messages.jsonl", ctx=ctx))
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ raise
+ if self.expiry(item, base, policy.days) > time.time_ns() // 1_000_000:
+ await self._register((account, item[0], uri), base)
+ return False, int(base > score)
+ try:
+ await self.fs.rm(uri, recursive=True, ctx=ctx, lease_ref=lease)
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ raise
+ removed = await redis.eval(REMOVE, 1, self.prefix(account) + item[0], uri, str(score))
+ return True, int(removed)
+ finally:
+ await agfs.pathlock_release(lease)
diff --git a/openviking/ttl/wiring.py b/openviking/ttl/wiring.py
new file mode 100644
index 000000000..2f0b5af43
--- /dev/null
+++ b/openviking/ttl/wiring.py
@@ -0,0 +1,117 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Process-wide registry and service wiring for the TTL adapter.
+
+The adapter is a singleton per server process. ``app.py`` creates the built-in
+implementation from ``server.ttl_adapter``, registers it here and installs its
+post-write hook on the VikingFS instance. The admin configuration routes consult
+:func:`ttl_config_supported` to decide whether an explicit TTL section is
+allowed or must be rejected with 501. When no adapter is registered
+(default configuration) every TTL entry point is inert.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence
+
+from openviking_cli.utils.logger import get_logger
+
+if TYPE_CHECKING:
+ from openviking.server.config import TTLAdapterConfig
+ from openviking.server.identity import RequestContext
+ from openviking.storage.viking_fs import VikingFS
+ from openviking.ttl.adapter import TTLAdapter
+
+logger = get_logger(__name__)
+
+_adapter: Optional["TTLAdapter"] = None
+
+
+def build_ttl_adapter(config: Optional["TTLAdapterConfig"]) -> Optional["TTLAdapter"]:
+ """Create the built-in Redis TTL implementation only when enabled."""
+ if config is None or not config.enabled:
+ return None
+ from openviking.ttl.runtime import DirectoryTTL
+
+ return DirectoryTTL(**config.config)
+
+
+def set_ttl_adapter(adapter: Optional["TTLAdapter"]) -> None:
+ """Register (or clear) the process-wide TTL adapter."""
+ global _adapter
+ _adapter = adapter
+
+
+def get_ttl_adapter() -> Optional["TTLAdapter"]:
+ """Return the registered TTL adapter, or ``None`` when TTL is off."""
+ return _adapter
+
+
+def ttl_config_supported() -> bool:
+ """Return True when explicit TTL configuration writes are allowed.
+
+ False when the Redis TTL implementation is not enabled, so admin routes
+ return 501 / ``UNIMPLEMENTED`` for an explicit TTL section.
+ """
+ return _adapter is not None and _adapter.supports_config()
+
+
+def install_ttl_hook(viking_fs: "VikingFS", adapter: Optional["TTLAdapter"]) -> None:
+ """Install the adapter's post-write hook, or leave the default-off hook.
+
+ Called once after the VikingFS instance exists. When no adapter is present
+ the hook stays unset, so a committed write never reads the clock or does any
+ TTL I/O (default-off behavior).
+ """
+ if adapter is None:
+ return
+ viking_fs.set_post_write_hook(adapter.on_post_write)
+ logger.info("Installed TTL post-write hook")
+
+
+def _entry_uri(entry: Any) -> Optional[str]:
+ if isinstance(entry, Mapping):
+ uri = entry.get("uri")
+ return uri if isinstance(uri, str) and uri else None
+ return None
+
+
+async def attach_expires_at(
+ entries: Sequence[Any],
+ ctx: Optional["RequestContext"],
+) -> None:
+ """Default-off read-path ``expires_at`` echo-back (see the technical design).
+
+ Attaches ``expires_at`` in place to each dict entry that carries a ``uri``;
+ a single ``stat`` result is just a one-element sequence. No-op in the
+ default configuration (no adapter), so the read path does zero TTL I/O. When a
+ Redis TTL implementation is enabled it computes the value from the current
+ effective policy; the field is only written for URIs the adapter actually
+ covers (value string or ``None``), so the kernel never fabricates a field
+ the product layer did not compute. Any adapter failure degrades to "no echo
+ this page" instead of disturbing the underlying listing/stat result.
+ """
+ adapter = _adapter
+ if adapter is None or not entries:
+ return
+ uris = list(dict.fromkeys(u for u in (_entry_uri(e) for e in entries) if u is not None))
+ if not uris:
+ return
+ try:
+ resolved = await adapter.resolve_expires_at(uris, ctx)
+ except Exception: # noqa: BLE001 - echo-back must never break stat/ls/tree
+ logger.exception("TTL adapter resolve_expires_at() failed; omitting expires_at echo")
+ return
+ for entry in entries:
+ uri = _entry_uri(entry)
+ if uri is not None and uri in resolved:
+ entry["expires_at"] = resolved[uri]
+
+
+__all__ = [
+ "attach_expires_at",
+ "get_ttl_adapter",
+ "install_ttl_hook",
+ "set_ttl_adapter",
+ "ttl_config_supported",
+]
diff --git a/openviking_cli/expiry_bootstrap.py b/openviking_cli/expiry_bootstrap.py
index a7d8cc6b7..9320646c1 100644
--- a/openviking_cli/expiry_bootstrap.py
+++ b/openviking_cli/expiry_bootstrap.py
@@ -21,15 +21,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
def load_expiry_config(path: Path):
+ from openviking.server.config import TTLAdapterConfig
from openviking_cli.utils.config.open_viking_config import OpenVikingConfig
raw = json.loads(os.path.expandvars(path.expanduser().read_text(encoding="utf-8-sig")))
config = OpenVikingConfig.from_dict(raw)
- if not config.uses_redis_expiry():
+ ttl = TTLAdapterConfig.model_validate(raw.get("server", {}).get("ttl_adapter") or {})
+ if not (config.uses_redis_expiry() or ttl.writer_enabled):
raise ValueError(
"expiry publisher requires memory.session_auto_commit.idle_backend='redis' "
- "or compile_api.scheduler_enabled=true (Redis)"
+ "or compile_api.scheduler_enabled=true or an enabled TTL writer (Redis)"
)
+ config.expiry.validate_for_runtime()
if not config.expiry.publisher_separate_process:
raise ValueError("expiry publisher process is disabled by configuration")
return config.expiry
diff --git a/openviking_cli/utils/config/config_utils.py b/openviking_cli/utils/config/config_utils.py
index 91f4a8c6a..9c1ac05a1 100644
--- a/openviking_cli/utils/config/config_utils.py
+++ b/openviking_cli/utils/config/config_utils.py
@@ -105,7 +105,9 @@ def warn_unknown_config_fields(
"""
warn_unknown_fields(
data=data,
- valid_fields=set(model.model_fields) | (extra_valid_fields or set()),
+ valid_fields=set(model.model_fields)
+ | {field.alias for field in model.model_fields.values() if isinstance(field.alias, str)}
+ | (extra_valid_fields or set()),
logger=logger,
path_prefix=path_prefix,
)
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index c66fdc25f..2ec00f733 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -64,6 +64,7 @@ from .rules_config import RulesConfig
from .runtime_field import RuntimeField
from .storage_config import StorageConfig
from .telemetry_config import TelemetryConfig
+from .ttl_config import TTLConfig
from .vlm_config import VLMConfig
@@ -525,6 +526,13 @@ class OpenVikingConfig(BaseModel):
description="Distributed expiry scheduling configuration",
)
+ ttl: TTLConfig = RuntimeField(
+ default_factory=TTLConfig,
+ description=(
+ "Directory TTL recycling policy. Default-off; requires the enabled Redis TTL runtime."
+ ),
+ )
+
@model_validator(mode="after")
def _validate_expiry_backend(self) -> "OpenVikingConfig":
self._check_expiry_backend()
diff --git a/openviking_cli/utils/config/ttl_config.py b/openviking_cli/utils/config/ttl_config.py
new file mode 100644
index 000000000..407041880
--- /dev/null
+++ b/openviking_cli/utils/config/ttl_config.py
@@ -0,0 +1,45 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Directory TTL (time-to-live) recycling configuration.
+
+TTL execution is provided by the optional, default-off Redis implementation in
+``openviking.ttl``. The runtime configuration manager publishes policies at the
+cluster (库/实例) and account (数据空间) scopes. No Redis-free backend is provided.
+
+A policy value is either a positive integer number of days or the literal
+``"disabled"``. An absent node means "inherit"; ``"disabled"`` stops the
+fallback at that node. Effective-policy selection (User > Account > cluster for
+each node, then type-default > ``global`` > default-off) lives in
+:mod:`openviking.config.ttl` so this module stays a pure data shape.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .runtime_field import RuntimeField
+
+#: A single TTL policy: a positive day count or an explicit ``disabled``.
+TTLPolicy = Union[Annotated[int, Field(strict=True, ge=1, le=365000)], Literal["disabled"]]
+
+
+class TTLConfig(BaseModel):
+ """Declarative TTL policy shared by cluster, account and user configs.
+
+ Every field is a :func:`RuntimeField` so the config API can PATCH it, and
+ every field defaults to ``None`` (unset / inherit) so an untouched document
+ keeps TTL fully disabled. ``global_`` is exposed to the API and persisted
+ under its ``global`` alias because ``global`` is a reserved word in Python.
+ """
+
+ model_config = ConfigDict(populate_by_name=True, extra="forbid")
+
+ global_: Optional[TTLPolicy] = RuntimeField(default=None, alias="global")
+ user_events: Optional[TTLPolicy] = RuntimeField(default=None)
+ peer_events: Optional[TTLPolicy] = RuntimeField(default=None)
+ sessions: Optional[TTLPolicy] = RuntimeField(default=None)
+
+
+__all__ = ["TTLConfig", "TTLPolicy"]
diff --git a/pyproject.toml b/pyproject.toml
index 5a224c4ff..9aef2352a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -117,6 +117,7 @@ test = [
"redis>=5.3.1,<7",
"pytest>=7.0.0",
"pytest-asyncio>=0.21.0",
+ "fakeredis[lua]>=2.23",
"pytest-xdist>=3.5.0",
"boto3>=1.42.44",
"pytest-cov>=4.0.0",
diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.py
index e6ce3e9c9..aefd2ec16 100644
--- a/tests/server/test_admin_api.py
+++ b/tests/server/test_admin_api.py
@@ -2146,19 +2146,24 @@ async def test_create_user_paths_accept_initial_user_config(
assert bob_settings.resource_uri == "viking://user/bob/resources/bob"
-async def test_user_memory_policy_can_be_initialized_and_hot_updated(
- lightweight_admin_client: httpx.AsyncClient,
- lightweight_admin_app: FastAPI,
+async def test_user_settings_preserve_policy_inheritance_backup_and_sparse_updates(
+ lightweight_admin_client, lightweight_admin_app, monkeypatch
):
+ import fakeredis.aioredis
+
+ from openviking.ttl import wiring
+ from openviking.ttl.runtime import DirectoryTTL
+ from openviking_cli.utils.config.open_viking_config import set_openviking_config
+
+ client = lightweight_admin_client
acct = _uid()
- create_account = await lightweight_admin_client.post(
+ created = await client.post(
"/api/v1/admin/accounts",
json={"account_id": acct, "admin_user_id": "alice"},
headers=root_headers(),
)
- assert create_account.status_code == 200, create_account.text
-
- create_user = await lightweight_admin_client.post(
+ assert created.status_code == 200, created.text
+ created = await client.post(
f"/api/v1/admin/accounts/{acct}/users",
json={
"user_id": "bob",
@@ -2167,80 +2172,107 @@ async def test_user_memory_policy_can_be_initialized_and_hot_updated(
},
headers=root_headers(),
)
- assert create_user.status_code == 200, create_user.text
-
- get_settings = await lightweight_admin_client.get(
- f"/api/v1/admin/accounts/{acct}/users/bob/settings",
- headers=root_headers(),
- )
- assert get_settings.status_code == 200, get_settings.text
- assert get_settings.json()["result"]["memory_policy"] == {
- "self": {"enabled": True},
- "peer": {"enabled": True},
- "working_memory": {"enabled": False},
- "memory_types": ["profile"],
- }
-
- patch_settings = await lightweight_admin_client.patch(
- f"/api/v1/admin/accounts/{acct}/users/bob/settings",
- json={"memory_policy": {"memory_types": ["experiences"]}},
- headers=root_headers(),
- )
- assert patch_settings.status_code == 200, patch_settings.text
- memory_policy = patch_settings.json()["result"]["memory_policy"]
- assert memory_policy["memory_types"] == [
- "cases",
- "experiences",
- "trajectories",
- ]
-
- viking_fs = lightweight_admin_app.state.fake_service.viking_fs
- persisted = await read_user_config(
- viking_fs,
- RequestContext(user=UserIdentifier(acct, "bob"), role=Role.USER),
- )
- assert persisted.memory_policy["memory_types"] == [
- "cases",
- "experiences",
- "trajectories",
- ]
+ assert created.status_code == 200, created.text
+ url = f"/api/v1/admin/accounts/{acct}/users/bob/settings"
+ service = lightweight_admin_app.state.fake_service
+ fs = service.viking_fs
user_ctx = RequestContext(user=UserIdentifier(acct, "bob"), role=Role.USER)
- backup = json.loads(viking_fs.files[user_config_backup_uri(user_ctx)])
- assert backup["memory_policy"]["memory_types"] == ["profile"]
-
- writes_before_noop = len(viking_fs.writes)
- noop_patch = await lightweight_admin_client.patch(
- f"/api/v1/admin/accounts/{acct}/users/bob/settings",
- json={"memory_policy": persisted.memory_policy},
- headers=root_headers(),
- )
- assert noop_patch.status_code == 200, noop_patch.text
- assert len(viking_fs.writes) == writes_before_noop
-
- lightweight_admin_app.state.config.user_config_defaults = UserConfig(
- memory_policy={"memory_types": ["events"]}
- )
- reset_settings = await lightweight_admin_client.patch(
- f"/api/v1/admin/accounts/{acct}/users/bob/settings",
- json={"memory_policy": None},
- headers=root_headers(),
- )
- assert reset_settings.status_code == 200, reset_settings.text
- assert reset_settings.json()["result"]["memory_policy"] == {
- "self": {"enabled": True},
- "peer": {"enabled": True},
- "working_memory": {"enabled": False},
- "memory_types": ["events"],
- }
+ monkeypatch.setattr(wiring, "_adapter", None)
+ before = dict(fs.files)
+ rejected = await client.patch(url, json={"ttl": {"sessions": 7}}, headers=root_headers())
+ assert rejected.status_code == 501 and fs.files == before
+
+ engine = DirectoryTTL(resource_id="settings-contract", timezone_name="UTC")
+ engine.bind_runtime(service, lightweight_admin_app.state.api_key_manager)
+ engine.redis = fakeredis.aioredis.FakeRedis(decode_responses=True)
+ monkeypatch.setattr(wiring, "_adapter", engine)
+ previous_config = get_openviking_config()
+ try:
+ await service.runtime_config_manager.patch_cluster({"ttl": {"global": 60, "sessions": 30}})
+ configured = await client.patch(
+ f"/api/v1/admin/accounts/{acct}/configuration",
+ json={"settings": {"ttl": {"global": 45, "peer_events": 14}}},
+ headers=root_headers(),
+ )
+ assert configured.status_code == 200, configured.text
+ patched = await client.patch(
+ url,
+ json={
+ "memory_policy": {"memory_types": ["experiences"]},
+ "ttl": {"sessions": 7, "peer_events": 3, "user_events": "disabled"},
+ },
+ headers=root_headers(),
+ )
+ assert patched.status_code == 200, patched.text
+ settings = (await client.get(url, headers=root_headers())).json()["result"]
+ assert settings["ttl"] == {"sessions": 7, "peer_events": 3, "user_events": "disabled"}
+ assert settings["memory_policy"]["memory_types"] == ["cases", "experiences", "trajectories"]
+ persisted = await read_user_config(fs, user_ctx)
+ assert persisted.ttl.sessions == 7
+ assert persisted.memory_policy["memory_types"] == settings["memory_policy"]["memory_types"]
+ backup = json.loads(fs.files[user_config_backup_uri(user_ctx)])
+ assert backup["memory_policy"]["memory_types"] == ["profile"]
+
+ first_peer = "viking://user/bob/peers/first/memories/events/2026/01/01"
+ second_peer = "viking://user/bob/peers/second/memories/events/2026/01/01"
+ user_event = "viking://user/bob/memories/events/2026/01/01"
+ assert await engine.resolve_expires_at([first_peer, second_peer, user_event], user_ctx) == {
+ first_peer: "2026-01-04T00:00:00Z",
+ second_peer: "2026-01-04T00:00:00Z",
+ user_event: None,
+ }
+ queried = await client.get(
+ f"/api/v1/admin/accounts/{acct}/configuration",
+ params={"path": "viking://user/bob/peers"},
+ headers=root_headers(),
+ )
+ assert queried.status_code == 200 and queried.json()["result"]["effective_policy"] == 3
- persisted = await read_user_config(viking_fs, user_ctx)
- assert persisted.memory_policy is None
- backup = json.loads(viking_fs.files[user_config_backup_uri(user_ctx)])
- assert backup["memory_policy"]["memory_types"] == [
- "cases",
- "experiences",
- "trajectories",
- ]
+ writes_before_noop = len(fs.writes)
+ noop = await client.patch(
+ url, json={"memory_policy": persisted.memory_policy}, headers=root_headers()
+ )
+ assert noop.status_code == 200 and len(fs.writes) == writes_before_noop
+ patched = await client.patch(
+ url, json={"ttl": {"sessions": None, "peer_events": "disabled"}}, headers=root_headers()
+ )
+ assert patched.status_code == 200, patched.text
+ assert patched.json()["result"]["ttl"] == {
+ "peer_events": "disabled",
+ "user_events": "disabled",
+ }
+ assert (await engine.policy(acct, "viking://user/bob/sessions")).days == 30
+ assert not (await engine.policy(acct, first_peer)).enabled
+ assert patched.json()["result"]["memory_policy"] == settings["memory_policy"]
+ assert json.loads(fs.files[user_config_backup_uri(user_ctx)])["ttl"]["sessions"] == 7
+
+ # Enabling retention must persist recovery intent before changing UserConfig.
+ before = dict(fs.files)
+ with monkeypatch.context() as failure:
+ failure.setattr(engine, "client", AsyncMock(side_effect=OSError("Redis unavailable")))
+ rejected = await client.patch(
+ url, json={"ttl": {"user_events": 7}}, headers=root_headers()
+ )
+ assert rejected.status_code == 503 and fs.files == before
+ cleared = await client.patch(url, json={"ttl": None}, headers=root_headers())
+ assert cleared.status_code == 200, cleared.text
+ assert cleared.json()["result"]["ttl"] is None
+ assert cleared.json()["result"]["memory_policy"] == settings["memory_policy"]
+ assert (await engine.policy(acct, first_peer)).days == 14
+
+ lightweight_admin_app.state.config.user_config_defaults = UserConfig(
+ memory_policy={"memory_types": ["events"]}
+ )
+ reset = await client.patch(url, json={"memory_policy": None}, headers=root_headers())
+ assert reset.status_code == 200, reset.text
+ assert reset.json()["result"]["memory_policy"]["memory_types"] == ["events"]
+ assert (await read_user_config(fs, user_ctx)).memory_policy is None
+ assert json.loads(fs.files[user_config_backup_uri(user_ctx)])["memory_policy"][
+ "memory_types"
+ ] == ["cases", "experiences", "trajectories"]
+ finally:
+ set_openviking_config(previous_config)
+ await engine.aclose()
async def test_user_memory_policy_uses_server_default_without_user_override(
@@ -2592,13 +2624,15 @@ async def test_delete_account(
assert (await manager.resolve_identity(replacement_key)).user_id == "bob"
-async def test_delete_account_retries_when_runtime_config_cleanup_fails(
- admin_client: httpx.AsyncClient,
- admin_service: OpenVikingService,
- admin_app: FastAPI,
- monkeypatch: pytest.MonkeyPatch,
+async def test_delete_account_retries_cleanup_and_retires_ttl_callbacks(
+ admin_client, admin_service, admin_app, monkeypatch
):
- """Config cleanup failure keeps the deletion fence and can be retried."""
+ import fakeredis.aioredis
+
+ from openviking.scheduling.expiry.api import CallbackRequest, ItemIdentity
+ from openviking.ttl import wiring
+ from openviking.ttl.runtime import DirectoryTTL
+
acct = _uid()
resp = await admin_client.post(
"/api/v1/admin/accounts",
@@ -2606,41 +2640,61 @@ async def test_delete_account_retries_when_runtime_config_cleanup_fails(
headers=root_headers(),
)
user_key = resp.json()["result"]["user_key"]
- runtime_config = admin_service.runtime_config_manager
+ engine = DirectoryTTL(resource_id="account-cleanup")
+ engine.bind_runtime(admin_service, admin_app.state.api_key_manager)
+ engine.redis = fakeredis.aioredis.FakeRedis(decode_responses=True)
+ scheduler = Mock(report=AsyncMock(), unregister=AsyncMock())
+ engine.install_expiry(scheduler)
+ monkeypatch.setattr(wiring, "_adapter", engine)
+ prefix = engine.prefix(acct)
+ other = engine.prefix("other") + "sessions"
+ await engine.redis.zadd(prefix + "sessions", {"viking://user/alice/sessions/s": 1})
+ await engine.redis.hset(prefix + "backfill:initial", "roots", "[]")
+ await engine.redis.set(prefix + "backfill:initial:done", "1")
+ await engine.redis.zadd(other, {"viking://user/alice/sessions/kept": 1})
+ unlink = engine.redis.unlink
+ monkeypatch.setattr(engine.redis, "unlink", AsyncMock(side_effect=OSError("Redis unavailable")))
delete_config = AsyncMock(side_effect=OSError("provider unavailable"))
- monkeypatch.setattr(runtime_config, "delete_account", delete_config)
-
- resp = await admin_client.delete(
- f"/api/v1/admin/accounts/{acct}",
- headers=root_headers(),
- )
- assert resp.status_code == 202
- failed = await _wait_for_task(admin_client, resp.json()["result"]["task_id"])
- assert failed["status"] == "failed"
- assert "provider unavailable" in failed["error"]
- denied = await admin_client.get(
- "/api/v1/fs/ls?uri=viking://",
- headers={"X-API-Key": user_key},
- )
- assert denied.status_code == 401
- assert await admin_app.state.api_key_manager.get_deletion(acct) is not None
-
- conflict = await admin_client.post(
- "/api/v1/admin/accounts",
- json={"account_id": acct, "admin_user_id": "replacement"},
- headers=root_headers(),
- )
- assert conflict.status_code == 409
-
- delete_config.side_effect = None
- retry = await admin_client.delete(
- f"/api/v1/admin/accounts/{acct}",
- headers=root_headers(),
- )
- completed = await _wait_for_task(admin_client, retry.json()["result"]["task_id"])
- assert completed["status"] == "completed"
- assert await admin_app.state.api_key_manager.get_deletion(acct) is None
- assert delete_config.await_count == 2
+ monkeypatch.setattr(admin_service.runtime_config_manager, "delete_account", delete_config)
+ try:
+ resp = await admin_client.delete(f"/api/v1/admin/accounts/{acct}", headers=root_headers())
+ assert resp.status_code == 202
+ failed = await _wait_for_task(admin_client, resp.json()["result"]["task_id"])
+ assert failed["status"] == "failed" and "Redis unavailable" in failed["error"]
+ assert await engine.redis.zcard(prefix + "sessions") == 1
+ denied = await admin_client.get(
+ "/api/v1/fs/ls?uri=viking://", headers={"X-API-Key": user_key}
+ )
+ assert denied.status_code == 401
+ assert await admin_app.state.api_key_manager.get_deletion(acct) is not None
+ conflict = await admin_client.post(
+ "/api/v1/admin/accounts",
+ json={"account_id": acct, "admin_user_id": "replacement"},
+ headers=root_headers(),
+ )
+ assert conflict.status_code == 409
+
+ monkeypatch.setattr(engine.redis, "unlink", unlink)
+ retry = await admin_client.delete(f"/api/v1/admin/accounts/{acct}", headers=root_headers())
+ failed = await _wait_for_task(admin_client, retry.json()["result"]["task_id"])
+ assert failed["status"] == "failed" and "provider unavailable" in failed["error"]
+ assert await admin_app.state.api_key_manager.get_deletion(acct) is not None
+ delete_config.side_effect = None
+ retry = await admin_client.delete(f"/api/v1/admin/accounts/{acct}", headers=root_headers())
+ completed = await _wait_for_task(admin_client, retry.json()["result"]["task_id"])
+ assert completed["status"] == "completed"
+ assert await admin_app.state.api_key_manager.get_deletion(acct) is None
+ assert delete_config.await_count == 2
+ assert not [key async for key in engine.redis.scan_iter(match=prefix + "*")]
+ assert await engine.redis.zcard(other) == 1
+
+ # A delivery already in flight cannot revive maintenance for the deleted account.
+ callback = CallbackRequest(ItemIdentity("ttl", acct, "_system", "maintenance"))
+ assert (await engine.handle(callback)).reason == "account_deleted"
+ scheduler.report.assert_not_awaited()
+ assert not [key async for key in engine.redis.scan_iter(match=prefix + "*")]
+ finally:
+ await engine.aclose()
async def test_create_duplicate_account_fails(admin_client: httpx.AsyncClient):
diff --git a/tests/server/test_bootstrap.py b/tests/server/test_bootstrap.py
index a7c666a0a..dd04ef346 100644
--- a/tests/server/test_bootstrap.py
+++ b/tests/server/test_bootstrap.py
@@ -392,7 +392,12 @@ def test_main_restart_cleans_up_managed_children_and_preserves_launch_command(
):
import sys
- config = ServerConfig(host="127.0.0.1", port=1933, with_bot=True)
+ config = ServerConfig(
+ host="127.0.0.1",
+ port=1933,
+ with_bot=True,
+ ttl_adapter={"enabled": True},
+ )
steps = []
monkeypatch.setattr(sys, "argv", ["openviking-server", "--with-bot", "--bot-port", "19000"])
original_argv = [sys.executable, "/venv/bin/openviking-server", *sys.argv[1:]]
@@ -405,7 +410,7 @@ def test_main_restart_cleans_up_managed_children_and_preserves_launch_command(
classmethod(
lambda cls, **kw: SimpleNamespace(
memory=SimpleNamespace(
- session_auto_commit=SessionAutoCommitConfig(enabled=True, idle_backend="redis")
+ session_auto_commit=SessionAutoCommitConfig(enabled=False, idle_backend="redis")
),
expiry=ExpiryConfig(),
)
diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.py
index 9f0bf9329..9b9840798 100644
--- a/tests/server/test_server_health.py
+++ b/tests/server/test_server_health.py
@@ -6,7 +6,7 @@
import asyncio
import time
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, Mock
import httpx
import pytest
@@ -334,8 +334,7 @@ async def test_slow_init_does_not_block_health(monkeypatch):
async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):
- """System assets reconcile after auth loading and before deletion setup."""
-
+ """Install TTL hooks before workers; preserve system-asset reconciliation."""
events = []
ov_config = SimpleNamespace(reconcile_system_assets_on_startup=True)
monkeypatch.setattr("openviking.server.app.get_openviking_config", lambda: ov_config)
@@ -346,9 +345,10 @@ async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):
self.viking_fs = object()
self._queue_manager = None
- async def initialize(self):
+ async def initialize(self, **kwargs):
+ await kwargs["before_background_start"]()
+ events.extend(["workers", "service"])
self._initialized = True
- events.append("service")
async def apply_agent_evolution_config(self):
events.append("agent-evolution")
@@ -358,12 +358,10 @@ async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):
events.append("system-assets")
return 1
- manager = object()
-
- async def setup(**kwargs):
- assert kwargs["manager"] is manager
- events.append("deletion")
-
+ manager = SimpleNamespace(resolve_identity=Mock(), get_user_group_ids=Mock())
+ ttl_init = AsyncMock(side_effect=lambda *args: events.append("ttl"))
+ monkeypatch.setattr("openviking.server.app._initialize_ttl_adapter", ttl_init)
+ setup = AsyncMock(side_effect=lambda **kwargs: events.append("deletion"))
monkeypatch.setattr("openviking.service.deletion.setup_deletion", setup)
async def initialize_plugin(app, service, config):
@@ -374,19 +372,21 @@ async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):
app = SimpleNamespace(state=SimpleNamespace(api_key_manager=None))
service = MockService()
- config = ServerConfig(root_api_key="root-key-for-test")
-
+ service.compile_scheduler = SimpleNamespace(set_identity_resolver=Mock())
+ config = ServerConfig(
+ root_api_key="root-key-for-test",
+ ttl_adapter={"enabled": True},
+ )
await _initialize_runtime_state(app, service, config)
+ startup = ["auth", "ttl", "workers", "service", "agent-evolution"]
+ assert events == startup + ["system-assets", "deletion"]
assert service._initialized is True
- assert app.state.api_key_manager is manager
- assert events == [
- "service",
- "agent-evolution",
- "auth",
- "system-assets",
- "deletion",
- ]
+ assert setup.await_args.kwargs["manager"] is manager
+ ttl_init.assert_awaited_once_with(app, service, config)
+ service.compile_scheduler.set_identity_resolver.assert_called_once_with(
+ manager.resolve_identity, manager.get_user_group_ids
+ )
events.clear()
ov_config.reconcile_system_assets_on_startup = False
diff --git a/tests/storage/test_session_delete_storage.py b/tests/storage/test_session_delete_storage.py
index 6fb5217a0..0d56e0385 100644
--- a/tests/storage/test_session_delete_storage.py
+++ b/tests/storage/test_session_delete_storage.py
@@ -13,7 +13,9 @@ from openviking.server.identity import RequestContext, Role
from openviking.service.fs_service import FSService
from openviking.storage.acl import AclEntry, AclLevel, AclMode, DirectAcl, EffectiveAcl
from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
+from openviking.storage.expr import And, Eq, Or, PathScope
from openviking.storage.viking_fs import VikingFS
+from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
from openviking_cli.exceptions import InvalidArgumentError, PermissionDeniedError
from openviking_cli.session.user_id import UserIdentifier
@@ -153,16 +155,83 @@ async def test_session_rm_rechecks_existence_after_waiting_for_tree_lock(storage
storage.agfs.pathlock_release.assert_awaited_once_with(storage.lease)
-async def test_session_rm_reuses_outer_lease_without_releasing_it(storage):
- outer_lease = {"lease_ref": "outer-lease"}
-
- await storage.fs.rm(
- SESSION_URI, recursive=True, ctx=storage.ctx, lease_ref=outer_lease, no_vector=True
+async def test_ttl_recycles_under_native_tree_lease_and_retries_failed_deletion(
+ tmp_path, monkeypatch
+):
+ import fakeredis.aioredis
+
+ from openviking.scheduling.expiry.api import CallbackRequest, ItemIdentity
+ from openviking.ttl.adapter import TTLBackfillRequest, TTLRecycleRequest
+ from openviking.ttl.runtime import DirectoryTTL
+ from openviking.ttl.wiring import install_ttl_hook
+ from openviking.utils.agfs_utils import RagfsBindingConfig, create_agfs_client
+ from openviking_cli.exceptions import NotFoundError
+ from openviking_cli.utils.config.agfs_config import AGFSConfig
+ from openviking_cli.utils.config.ttl_config import TTLConfig
+
+ agfs = create_agfs_client(
+ RagfsBindingConfig(agfs=AGFSConfig(path=str(tmp_path), backend="local"))
)
-
- storage.agfs.pathlock_acquire_tree.assert_not_awaited()
- storage.agfs.pathlock_release.assert_not_awaited()
- assert storage.agfs.rm.await_args.kwargs["fs_ctx"]["lease_ref"] == "outer-lease"
+ vectors = SimpleNamespace(count=AsyncMock(return_value=1), delete_uris=AsyncMock())
+ fs = VikingFS(agfs=agfs, vector_store=vectors)
+ engine = DirectoryTTL(resource_id="recycle-contract", batch_interval=0, delete_interval=0)
+ engine.fs = fs
+ engine.policies = AsyncMock(return_value=(TTLConfig(sessions=7, user_events=7), None))
+ engine.redis = fakeredis.aioredis.FakeRedis(decode_responses=True)
+ ctx = engine.context("account")
+ session = "viking://user/alice/sessions/fresh"
+ event = "viking://user/alice/memories/events/2020/01/01"
+ request = TTLRecycleRequest(engine.resource_id, ctx.account_id)
+ callback = CallbackRequest(ItemIdentity("ttl", ctx.account_id, "_system", "maintenance"))
+ try:
+ # A historical Session missed incremental registration; backfill must recover it.
+ await fs.write_file(session + "/messages.jsonl", "recent message", ctx=ctx)
+ install_ttl_hook(fs, engine)
+ await engine.start()
+ # Soft TTL permits writing old Events until the background deletion happens.
+ await fs.write_file(event + "/event.md", "historical event", ctx=ctx)
+ await engine._registration_task
+ prefix = engine.prefix(ctx.account_id)
+ event_score = await engine.redis.zscore(prefix + "user_events", event)
+ assert event_score is not None
+ await engine.register_initial_backfill(
+ TTLBackfillRequest(
+ engine.resource_id,
+ ctx.account_id,
+ ("viking://user/alice/sessions", "viking://user/alice/memories/events"),
+ )
+ )
+ assert (await engine.recycle_due(request)).deleted == 0
+ assert await fs.read_file(event + "/event.md", ctx=ctx) == "historical event"
+ monkeypatch.setattr(
+ "openviking.ttl.runtime.time.time_ns", lambda: engine._recycle_after_ms * 1_000_000
+ )
+ await engine.handle(callback)
+ await engine._task
+ assert (await engine.read_initial(ctx.account_id))["done"]
+ assert await engine.redis.zscore(prefix + "sessions", session) is not None
+
+ # The file's fresh ModTime wins over a stale Session index entry.
+ await engine.redis.zadd(prefix + "sessions", {session: 1})
+ vectors.delete_uris.side_effect = OSError("vector deletion unavailable")
+ failed = await engine.recycle_due(request)
+ assert failed.failed == 1 and failed.skipped >= 1
+ assert await engine.redis.zscore(prefix + "sessions", session) > 1
+ assert await engine.redis.zscore(prefix + "user_events", event) == event_score
+ assert await fs.read_file(event + "/event.md", ctx=ctx) == "historical event"
+ assert (await fs._async_agfs.pathlock_observe())["active_locks"] == 0
+
+ vectors.delete_uris.side_effect = None
+ recycled = await engine.recycle_due(request)
+ assert recycled.deleted == 1 and recycled.failed == 0
+ with pytest.raises(NotFoundError):
+ await fs.stat(event, ctx=ctx)
+ assert await engine.redis.zscore(prefix + "user_events", event) is None
+ assert await fs.read_file(session + "/messages.jsonl", ctx=ctx) == "recent message"
+ assert (await fs._async_agfs.pathlock_observe())["active_locks"] == 0
+ finally:
+ await engine.aclose()
+ agfs.close()
async def test_session_rm_preserves_retryable_busy_error(storage):
@@ -200,6 +269,12 @@ async def test_session_rm_preserves_namespace_and_path_validation(storage, acl_e
async def test_default_rm_still_cleans_vector_records(storage, missing, uri):
child_uri = f"{uri}/document.md"
storage.fs._collect_uris.return_value = [child_uri]
+ delete = AsyncMock()
+ backend = object.__new__(VikingVectorIndexBackend)
+ backend._get_backend_for_context = AsyncMock(
+ return_value=SimpleNamespace(delete_by_filter=delete)
+ )
+ storage.vector.delete_uris = backend.delete_uris
if missing:
storage.agfs.stat.side_effect = FileNotFoundError(RESOURCE_PATH)
@@ -209,7 +284,14 @@ async def test_default_rm_still_cleans_vector_records(storage, missing, uri):
storage.fs._collect_uris.assert_awaited_once()
storage.vector.count.assert_awaited_once()
assert storage.vector.count.await_args.kwargs["ctx"] == storage.ctx
- storage.vector.delete_uris.assert_awaited_once_with(storage.ctx, [child_uri, uri])
+ delete.assert_awaited_once_with(
+ And(
+ [
+ Eq("account_id", storage.ctx.account_id),
+ Or([Eq("uri", uri), PathScope("uri", uri, depth=-1)]),
+ ]
+ )
+ )
if missing:
storage.agfs.rm.assert_not_awaited()
else:
@@ -245,4 +327,9 @@ async def test_resource_rm_authorizes_the_whole_subtree_at_its_root(storage, all
await service._rm(RESOURCE_URI, ctx=storage.ctx, recursive=True)
storage.agfs.rm.assert_not_awaited()
storage.vector.delete_uris.assert_not_awaited()
- storage.acl.resolve_many.assert_awaited_once_with([RESOURCE_URI], storage.ctx)
+ else:
+ storage.vector.count.assert_awaited_once()
+ storage.vector.delete_uris.assert_awaited_once_with(
+ storage.ctx, [RESOURCE_URI], recursive=True
+ )
+ storage.agfs.pathlock_release.assert_awaited_once_with(storage.lease)
diff --git a/tests/storage/test_viking_fs_write_locking.py b/tests/storage/test_viking_fs_write_locking.py
index e3feea1cb..4e6936cf1 100644
--- a/tests/storage/test_viking_fs_write_locking.py
+++ b/tests/storage/test_viking_fs_write_locking.py
@@ -182,7 +182,11 @@ async def test_append_file_holds_exact_lease_across_read_and_write(monkeypatch):
)
monkeypatch.setattr(fs, "_ctx_or_default", lambda _ctx=None: _default_ctx())
+ # Notification failure cannot undo a committed append or keep its lease held.
+ hook = AsyncMock(side_effect=RuntimeError("registration unavailable"))
+ fs.set_post_write_hook(hook)
await fs.append_file("viking://a.md", "+new")
+ hook.assert_awaited_once()
assert fake.events[0] == ("ensure_parent", "/local/default/resources/a.md", None)
assert fake.events[1][0] == "acquire"
diff --git a/tests/unit/service/test_core_consistency.py b/tests/unit/service/test_core_consistency.py
index 3548345c0..225ab4c73 100644
--- a/tests/unit/service/test_core_consistency.py
+++ b/tests/unit/service/test_core_consistency.py
@@ -115,14 +115,26 @@ async def test_check_consistency_preserves_directory_behavior() -> None:
@pytest.mark.asyncio
async def test_close_stops_queue_manager_before_ragfs_binding(monkeypatch) -> None:
+ from openviking.ttl import wiring
+
events: list[str] = []
+ async def close_ttl():
+ events.append("ttl")
+
+ adapter = SimpleNamespace(aclose=close_ttl)
+ monkeypatch.setattr(wiring, "_adapter", adapter)
+ monkeypatch.setattr(
+ "openviking.service.core.get_task_tracker", lambda: SimpleNamespace(close=AsyncMock())
+ )
+
class ResourceService:
async def close_background_tasks(self) -> None:
events.append("resource_tasks")
class QueueManager:
def stop(self) -> None:
+ assert wiring.get_ttl_adapter() is adapter
events.append("queue_manager")
class RagfsClient:
@@ -132,6 +144,13 @@ async def test_close_stops_queue_manager_before_ragfs_binding(monkeypatch) -> No
service = OpenVikingService.__new__(OpenVikingService)
service._resource_service = ResourceService()
service._config = SimpleNamespace(vlm=VLMConfig())
+ service._compile_scheduler = None
+ service._ttl_adapter = adapter
+ service._expiry_runtime = SimpleNamespace(
+ stop_roles=AsyncMock(side_effect=lambda **_: events.append("expiry_roles")),
+ close=AsyncMock(side_effect=lambda: events.append("expiry_storage")),
+ )
+ service._session_service = SimpleNamespace(set_session_expiry_adapter=lambda _: None)
service._runtime_config_manager = None
service._watch_scheduler = None
service._session_auto_commit_scheduler = None
@@ -154,4 +173,8 @@ async def test_close_stops_queue_manager_before_ragfs_binding(monkeypatch) -> No
await service.close()
assert events.index("queue_manager") < events.index("ragfs_binding")
+ assert wiring.get_ttl_adapter() is None
+ assert events.index("expiry_roles") < events.index("queue_manager")
+ assert events.index("queue_manager") < events.index("ttl") < events.index("expiry_storage")
+ assert events.index("expiry_storage") < events.index("ragfs_binding")
assert events[-1] == "data_dir_lock"
diff --git a/tests/unit/session/test_session_commit_resume.py b/tests/unit/session/test_session_commit_resume.py
index 1ae18e66a..758a05f1c 100644
--- a/tests/unit/session/test_session_commit_resume.py
+++ b/tests/unit/session/test_session_commit_resume.py
@@ -100,7 +100,7 @@ class _MemoryVikingFS:
[(10, None, 5), (10, 1, 2), (10, 0, 15), (0, None, 15)],
)
async def test_commit_retention_boundary_and_pending_tokens_after_reload(
- monkeypatch, keep_count, keep_turns, archive_count
+ monkeypatch, tmp_path, keep_count, keep_turns, archive_count
):
session_uri = "viking://user/default/sessions/session-1"
messages = [
@@ -118,11 +118,14 @@ async def test_commit_retention_boundary_and_pending_tokens_after_reload(
for i in range(12)
],
]
- storage = _MemoryVikingFS(
- {
- f"{session_uri}/messages.jsonl": "\n".join(message.to_jsonl() for message in messages),
- }
+ from openviking.storage.viking_fs import VikingFS
+ from openviking.utils.agfs_utils import RagfsBindingConfig, create_agfs_client
+ from openviking_cli.utils.config.agfs_config import AGFSConfig
+
+ agfs = create_agfs_client(
+ RagfsBindingConfig(agfs=AGFSConfig(path=str(tmp_path), backend="local"))
)
+ storage = VikingFS(agfs=agfs)
tracker = TaskTracker(_TaskStore())
monkeypatch.setattr("openviking.session.session._enabled_memory_types", lambda: set())
monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)
@@ -130,33 +133,45 @@ async def test_commit_retention_boundary_and_pending_tokens_after_reload(
"openviking.storage.queuefs.get_queue_manager",
lambda: SimpleNamespace(enqueue=AsyncMock()),
)
- session = Session(viking_fs=storage, session_id="session-1", session_uri=session_uri)
- turn_options = (
- {"retention_mode": "turn_budget", "keep_recent_turn_count": keep_turns}
- if keep_turns is not None
- else {}
- )
- result = await session.commit_async(keep_recent_count=keep_count, **turn_options)
- archived = await session._archives.read_messages(result["archive_uri"])
- retained = messages[archive_count:]
- assert [message.id for message in archived] == [m.id for m in messages[:archive_count]]
- assert [message.id for message in session.messages] == [m.id for m in retained]
- assert session.meta.pending_tokens == 0
-
- reloaded = Session(viking_fs=storage, session_id="session-1", session_uri=session_uri)
- await reloaded.load()
- assert reloaded.meta.pending_tokens == 0
- appended = await reloaded.add_message_async("user", [TextPart("next question")])
- if keep_turns == 0 or keep_count == 0:
- expected = appended.estimated_tokens
- elif keep_turns == 1:
- expected = sum(message.estimated_tokens for message in retained)
- else:
- expected = retained[0].estimated_tokens
- assert reloaded.meta.pending_tokens == expected
- fresh = Session(viking_fs=storage, session_id="session-1", session_uri=session_uri)
- await fresh.load()
- assert fresh.meta.pending_tokens == expected
+ try:
+ session = Session(viking_fs=storage, session_id="session-1", session_uri=session_uri)
+ await session.ensure_exists()
+ await storage.write_file(
+ f"{session_uri}/messages.jsonl",
+ "\n".join(message.to_jsonl() for message in messages),
+ ctx=session.ctx,
+ )
+ turn_options = (
+ {"retention_mode": "turn_budget", "keep_recent_turn_count": keep_turns}
+ if keep_turns is not None
+ else {}
+ )
+ result = await session.commit_async(keep_recent_count=keep_count, **turn_options)
+ archived = await session._archives.read_messages(result["archive_uri"])
+ saved = await session._merge_and_save_commit_meta(archive_count, {"events": 1}, None)
+ assert saved is not None and saved.memories_extracted["events"] == 1
+ retained = messages[archive_count:]
+ assert [message.id for message in archived] == [m.id for m in messages[:archive_count]]
+ assert [message.id for message in session.messages] == [m.id for m in retained]
+ assert session.meta.pending_tokens == 0
+
+ reloaded = Session(viking_fs=storage, session_id="session-1", session_uri=session_uri)
+ await reloaded.load()
+ assert reloaded.meta.pending_tokens == 0
+ appended = await reloaded.add_message_async("user", [TextPart("next question")])
+ if keep_turns == 0 or keep_count == 0:
+ expected = appended.estimated_tokens
+ elif keep_turns == 1:
+ expected = sum(message.estimated_tokens for message in retained)
+ else:
+ expected = retained[0].estimated_tokens
+ assert reloaded.meta.pending_tokens == expected
+ fresh = Session(viking_fs=storage, session_id="session-1", session_uri=session_uri)
+ await fresh.load()
+ assert fresh.meta.pending_tokens == expected
+ finally:
+ await tracker.close()
+ agfs.close()
def test_phase2_auto_commit_policy_parameters_are_appended():
diff --git a/uv.lock b/uv.lock
index 389011902..74ad39374 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1164,6 +1164,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },
]
+[[package]]
+name = "fakeredis"
+version = "2.40.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "redis" },
+ { name = "sortedcontainers" },
+ { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/61/d0/8cbd1339c2a606a0ceda74e1a181248d372bb2c66bc6cf9d954871839ff9/fakeredis-2.40.0.tar.gz", hash = "sha256:16eb05a3e97c37a033c73d1da7e885eb2aa47ba7604cc377144339efa2780a02", size = 332674, upload-time = "2026-10-14T12:46:01.851Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c7/e4/6919d3653d72c53d1fb22c97ceb6fa3664cad302994e90ee52279f7eb394/fakeredis-2.40.0-py3-none-any.whl", hash = "sha256:b155ef2442134372eb1cc5664cf5638ccbe0a6dde9d1942153708e2782f315c9", size = 204148, upload-time = "2026-10-14T12:46:00.014Z" },
+]
+
+[package.optional-dependencies]
+lua = [
+ { name = "lupa" },
+]
+
[[package]]
name = "fastapi"
version = "0.135.1"
@@ -2487,6 +2506,69 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" },
]
+[[package]]
+name = "lupa"
+version = "2.8"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/a6/0f869fbb07c393f15473b1eefefb7b5bec162fb7481803d040ed4dc46002/lupa-2.8.tar.gz", hash = "sha256:d8022641b9ec8ecf2c5ecbe9f47e5a70e0b87c4b5ae921b92cb02a638e0acd08", size = 6156370, upload-time = "2026-04-15T20:08:30.534Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/09/21/9be4516ddd22f8eadba336d9ba065d17d79108465ae1b7f71424ab99b9d0/lupa-2.8-cp310-abi3-win32.whl", hash = "sha256:c2a5fd15dc62374e1661a55f01744c9ec1c56f291ba4a0749d3af2174556e78f", size = 1594887, upload-time = "2026-04-15T20:05:23.377Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/99/1557c9685d7034d9ce8dd2b54c40a26d6deb7c67c1fdb5c801abd1a02c3f/lupa-2.8-cp310-abi3-win_arm64.whl", hash = "sha256:9e304fb1c50cf23fd8882afbe1aa87525ef8a72667bcab3b37b2bbb2bc542269", size = 1371742, upload-time = "2026-04-15T20:05:27.417Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/34/05ce4745b191633f90ff1ab50f1a19a37da282bb0a41fb500d9157fc9b8f/lupa-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:97bd01e90b8031e56a5fd5bb70605aea09f1dba675c1140308a52780f93d06f1", size = 1202714, upload-time = "2026-04-15T20:05:31.088Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/d2/f70fdbeec2d4c69ee6a469e6cddde9635fff4af4e13fb652e6a1229eef51/lupa-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b5ebe1a13c45767919c86750b84fe2da9f6288b6f3cea4ce7660bb2abc9d921", size = 1857453, upload-time = "2026-04-15T20:05:34.611Z" },
+ { url = "https://files.pythonhosted.org/packages/97/dc/6fcda0e36e75eb6cb98dc9190fa4737d727eeae29e58f892980b2c96b656/lupa-2.8-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:097e7d0f1719a88020b67c82e05d53d7973c166952393afcecfd8434c7e19a15", size = 2408890, upload-time = "2026-04-15T20:05:37.994Z" },
+ { url = "https://files.pythonhosted.org/packages/58/29/7ea176eac3c1dac83d059762daa875ad1390decc0bf2c3b4c7bbfc1f1665/lupa-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:7bb223ee8f72d0dc076b0d65296ee72f1c69450f9d2fed5315f7707d98c4a03d", size = 1910396, upload-time = "2026-04-15T20:05:41.163Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/0a/5a740717f27aa77481e6a61b97cf79d1e0c1ede729b1268caacded915326/lupa-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b12e43c1fb787189dfc28cd604aef0baa2cb95e27da19498d520361d0ace070a", size = 1202376, upload-time = "2026-04-15T20:05:44.049Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/75/6b64d0098c64275a801896cb7a6a30e7e653d25fa102c64e747292afcdbb/lupa-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6f603391dffb256e36a79fd2044084d5f4b8a0a4c0e5ad291cd3ab3aaf1fd0a", size = 1839271, upload-time = "2026-04-15T20:05:47.399Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/2f/0d4f00563046ff616ef6a421f8b776a5ffb327f7b32ed69e856d52b917a8/lupa-2.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f6f41c91366e7d0d474f87d81c1274af861f40812bf729c9f97ab4c8f3c7ac8", size = 2376251, upload-time = "2026-04-15T20:05:49.891Z" },
+ { url = "https://files.pythonhosted.org/packages/4c/8e/caa83237f427d9e85b7f02c816e7270c9c9571dec1673e06b0180402f70e/lupa-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:f5a6af145b0ea818f01d27bfe2583a4b538570bef61d22c8773e0eccf011234c", size = 1923488, upload-time = "2026-04-15T20:05:52.954Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/0b/368f2f0bc750b25c69d4563e44f677925ab5dd3d2887f9b0c15465d21a2a/lupa-2.8-cp312-abi3-macosx_10_13_x86_64.whl", hash = "sha256:f4342f4de76ae7ce2ab0672d36003bdb7e1a33252f293b569298ddd792e70e33", size = 1194056, upload-time = "2026-04-15T20:05:55.794Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/0f/c89eb8dd36fdea4e50ae3f7f5275bea3b0cc5d4057b8ee7b3bbc78010422/lupa-2.8-cp312-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:4203fa1659315e939a5304e75001b8cc14234fb3cbb3ed86c049b0cc5d90fcee", size = 1434278, upload-time = "2026-04-15T20:05:57.94Z" },
+ { url = "https://files.pythonhosted.org/packages/47/30/c3b4d2cd8733621b404b8a4214e5f852955c4ba632546dc84123bea9ee89/lupa-2.8-cp312-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:81f2d843ce668b653146c007467570210ae44be51dac6926666c51d49536f307", size = 1150068, upload-time = "2026-04-15T20:06:01.04Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/d2/bac12c398519efafc6af84be1974edd0d7a4895fb4735b5c8d615d298595/lupa-2.8-cp312-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d3d0cde2c77588d1c60875a4f34f059513476c6e1775351897195b51e0f3df08", size = 1409532, upload-time = "2026-04-15T20:06:03.592Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/6a/18b52e11962014026e07813530b0b108ee8bc0a2a13ef0eaea5d41dce023/lupa-2.8-cp312-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e0d11b8f3a8dac6413f704fef7161d048bb10c58bdac6cbffa5e60efa56e9a3", size = 1242687, upload-time = "2026-04-15T20:06:06.863Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/8e/7fd4eb049875f61429b96780d2eae4700f0e78fe0a52db8edb231b1cd09f/lupa-2.8-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54cff414f21f8cd8c6be4aae52541f3b9cd39602b59e3a3db9b5c9f9f674ff18", size = 1856038, upload-time = "2026-04-15T20:06:09.358Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/f9/37ad9d2773d30f2931890d310a4bdce28d45484206e6f48bc18b0325eabd/lupa-2.8-cp312-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:24b4d8af5558e549b70daf1547f5c1c1d664ecea9fc790f83efe5d75e9a93797", size = 1128982, upload-time = "2026-04-15T20:06:12.312Z" },
+ { url = "https://files.pythonhosted.org/packages/57/31/c0fd7984c24844ea79caa45c0235f61a06b38fd69a839f6c62770f8d684a/lupa-2.8-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:ce86dff1ee7f7cf45f5622065ae991949dd7bb1703581cbc58a630137bb7ccf9", size = 1457594, upload-time = "2026-04-15T20:06:15.881Z" },
+ { url = "https://files.pythonhosted.org/packages/11/f5/a28e411be30ec1bf0db1eb0c087eebc73be9e7a1adcfe6ac209861ccc446/lupa-2.8-cp312-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:f4d01b2a08c70bbb883a9e082b6b36b89121ed5910b710f1ba11c73295ff4fba", size = 1425721, upload-time = "2026-04-15T20:06:18.009Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/c1/359f767c4ae024be30d909fe8a9f0e9af266bad47ce2bd2ed248fb986fcf/lupa-2.8-cp312-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:7f210d5a8353e510ea1199c42cf3cbdd630553bf2bc8fb4c00fea06fdec7c798", size = 1253258, upload-time = "2026-04-15T20:06:21.17Z" },
+ { url = "https://files.pythonhosted.org/packages/17/52/473f11790c261fd02bbf318a546fe040e9ec9f677181272fa78d3b4112a4/lupa-2.8-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4f81a02806e7c7ad26d8c6fa222c8bef1b0c1b124347c879be880b41339d41e4", size = 2395272, upload-time = "2026-04-15T20:06:24.137Z" },
+ { url = "https://files.pythonhosted.org/packages/94/bf/75c8795655a8836eab6a11a630352c4b7c5dc5c54d075077bc9bffdeee45/lupa-2.8-cp312-abi3-win32.whl", hash = "sha256:360056453a7a4eaa4ac5a204c31a5a014b1eb2ee5490603234d2ba831684f1f2", size = 1606136, upload-time = "2026-04-15T20:06:27.815Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/29/11a2cdd612b6f55e506292dfb6ba343216e80a693e7fe3f876ef204ce9c6/lupa-2.8-cp312-abi3-win_arm64.whl", hash = "sha256:1628371c6592a6d5650497a9e31fb2bb3a7e9883c1f301d1111265e484045af9", size = 1364495, upload-time = "2026-04-15T20:06:30.254Z" },
+ { url = "https://files.pythonhosted.org/packages/4d/17/fa834b6b09ad17e7df5d0f7715d64877a125a3776ada689751a1f9dc2959/lupa-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:450650f91c48c2415b0d59ab3abfcfda3b6efb5b858205f4d4bda8ad141fa529", size = 1190111, upload-time = "2026-04-15T20:06:32.84Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/43/45589901b7d1a0e3a9d91d19a311fb6a56924e8571536c3f2212160fd953/lupa-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27044f3363047f946b3d3aab9157cbd172b3538ada9ec1baef43432bf7d03a78", size = 1812999, upload-time = "2026-04-15T20:06:35.664Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/ac/4ade7d15ff5c61758d7943ac6f0a496bf1cc65b6c09f842b52a0702e664c/lupa-2.8-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cf4f064a0e5531afce2d7d750120c10c10f9529139af6ca6150d13151034398", size = 2368731, upload-time = "2026-04-15T20:06:37.959Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/27/05f950d15b8ab120b39c43588b438ff3ace70c1b1b0225a960393a497483/lupa-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:281bedc5deb92d31e649a3552edd662449365a635904fa4d5cb4509c7245e34e", size = 1941809, upload-time = "2026-04-15T20:06:40.302Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/3f/19f83c3a0c84dc8bea8a58e7416dca6a3ede662c33c8d1ec758e5afc754a/lupa-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45fc9da0145ecb0083ef5ff9975116cc784bd0258bdc2bd131ba15483ce18398", size = 1201203, upload-time = "2026-04-15T20:06:42.169Z" },
+ { url = "https://files.pythonhosted.org/packages/89/0f/a14f0073f09610158038582e230618a48c14da6bd88185289461aa4cb854/lupa-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:58e18afed57955b41130e269c78f53d4123ab86e236b53816f4cbffa25cb5d30", size = 1806210, upload-time = "2026-04-15T20:06:45.486Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/14/48fff156c63a136001a7620878af7d31aa07e66b495ed621e3eddd73c294/lupa-2.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc47f536ac13a79cef47d29a2b205576a22841f042a2bcec1676b95806e7706a", size = 2359005, upload-time = "2026-04-15T20:06:47.819Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/18/3ac638ec90edf178242b8a2b2f00f8adae694248c03a26341ef941bb746e/lupa-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:ce9404c661dbac65cc9bed351ad45e797af93d30d70be309a3fa8209ac86d93b", size = 1936754, upload-time = "2026-04-15T20:06:50.448Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/ef/5ee5fed6ea7459a671196359ce04bfeeaf26be1dac8ff24bf28e5c7a6e81/lupa-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:348c3f8ecabb6324dcbc05c2740d762ef8fcec7b06c79e45262ab97a217684e3", size = 1209388, upload-time = "2026-04-15T20:06:53.022Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/b1/67a940d5542cb0384b443fe951b5a83ea9340d1333a733a258fdd1c619ba/lupa-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:951496471056061598a7d1729a6cdf48d662fec777a9f2d8aa5a1e62fd30e5a5", size = 1826821, upload-time = "2026-04-15T20:06:55.699Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/a2/b354e5ba3b911ec50686003dc8897e892b9e8c5c036b33219b03d54c4daf/lupa-2.8-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a591b9947ca347b41a63370e121d6e2b1458fe6dde9ae065029ec10a37f25ff4", size = 2366893, upload-time = "2026-04-15T20:06:58.9Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/52/d76066401f29539df5352f70ecded66576f32933b6045cd0bfc56cb770b9/lupa-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:3903c9cf628dae2f56405503247b77a61a3a61bd2dda470e336950c74776d55d", size = 1994716, upload-time = "2026-04-15T20:07:19.194Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/bd/3efc437a4361c16d25e66478c50357c9a8e8ecfb718fe749eb9ca3176ef6/lupa-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f711a8ab0486b9ac6fdda94a22ddcfbc9f0d4a27e3a8cf1bf79c6e48b33017c1", size = 1251217, upload-time = "2026-04-15T20:07:01.64Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/f4/2e9f8ecbaca854bfdf14af8a9b505ec0cbc640377b3b218921594b7563cd/lupa-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc51250e76367a3e27fcd01dc769b9bfcbbc34f48df48dde53d6af6e75b7eaa5", size = 1814701, upload-time = "2026-04-15T20:07:04.149Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/53/4000b1acaa8b1f3827fcff0cfcdff44d3befddda42cab7e685a49689b5a1/lupa-2.8-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f8a22088a552828958603323f0a5c4b3e11e03b75d0bf4c965ef879de9b60a8d", size = 2348414, upload-time = "2026-04-15T20:07:07.285Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/78/26ee48d3890cddf03cefb65f433e3492759c0b3c0582180755bddbaab7bd/lupa-2.8-cp314-cp314t-win32.whl", hash = "sha256:4f7c553c1d8cfffbe85d81daef730d12cae4b6002d457542914da0ac8a1145b3", size = 1831611, upload-time = "2026-04-15T20:07:09.752Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/d1/4a5cc64a3cad22821ae4c3f7a90456a08ca19457d8354f4abf46ad03c7e8/lupa-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:d8766aff03a78c80ad2d188a8bdb216de5ec838359cd87e05bbdfa56394a6105", size = 2209250, upload-time = "2026-04-15T20:07:11.906Z" },
+ { url = "https://files.pythonhosted.org/packages/37/7c/cdcb654daf668192aaf36b0aeb94f2281dad092aaa5003688691131736ea/lupa-2.8-cp314-cp314t-win_arm64.whl", hash = "sha256:91d622777febda3ab1bed1d45295f2f32a4680c7b3d7caf8c669998ed5c44118", size = 1126735, upload-time = "2026-04-15T20:07:15.434Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/44/de1961ad38e17cd326a53c246c7e3b91178ed578f4cf22ffcd5e7e11b041/lupa-2.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b036738282a5acd2e71fdddb317c9df8b87c1673aa57f403d05fcc2be8abc4ba", size = 1186020, upload-time = "2026-04-15T20:07:35.017Z" },
+ { url = "https://files.pythonhosted.org/packages/13/c2/276f0b9dc8bcc5a8a58af5316dfa0e6f56be3613dd6dbcc8d3d2cb6559ba/lupa-2.8-cp39-abi3-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:ac6b6e8d0e617e26a98cbb44880bcd75de5d32b3ad7b3b3793583909292b47ed", size = 1468944, upload-time = "2026-04-15T20:07:37.782Z" },
+ { url = "https://files.pythonhosted.org/packages/63/38/52934e52a5180dc6425d20284d004fe4b27a4f9171a82dc99fb67af250bf/lupa-2.8-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ba3a7dd839f90c3d2e53bebe3c192b1f3f9fd720a6781256405123211fd0dce6", size = 1172998, upload-time = "2026-04-15T20:07:40.812Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/82/76b3809bd0839d9b3b4ec58d06591e08f17337b6d9576877cb9d48b34e94/lupa-2.8-cp39-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d7edb13a7a5250b5c6c22d1495d9e842b5c9fc5081c8fe6b5efe2112fe3e41f9", size = 1449975, upload-time = "2026-04-15T20:07:44.262Z" },
+ { url = "https://files.pythonhosted.org/packages/16/07/2f89d54f747c67c23b4b9ae4aa8c8dd06bb409155dedcf406157f2736b66/lupa-2.8-cp39-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:891f72e0bffbed1e4175f975aeb2a083956586a100066525e1be485f617f7b25", size = 1281944, upload-time = "2026-04-15T20:07:46.458Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/bd/7375d2b0fcae79d806baf52a76f26c96964593f58e1372d13ae5ac09c676/lupa-2.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a295f87b5b7ebbfd5191932e8cb0e51df3c7769101ac6b6c7d7c9fb27bfd1307", size = 1910455, upload-time = "2026-04-15T20:07:49.75Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/0c/8abb3bc0e08b311fc01db05b6e9f9ff31a8f65e4fc3f0aeb05cfef75c8ac/lupa-2.8-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4fe5d7a810b64ea8511eb885fc8cdde042ee5ff7b7d08ae78f32449756acb177", size = 1155548, upload-time = "2026-04-15T20:07:52.657Z" },
+ { url = "https://files.pythonhosted.org/packages/80/2e/9eeecd3f493099721c1d3f31beeca23a4237db1a54223684df4dc96aa1bd/lupa-2.8-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:bfc470012ef66ad064c7bd77416af03a3452ef630b04b9012595ea13f2e54518", size = 1489232, upload-time = "2026-04-15T20:07:54.92Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/13/731c99dc2e7652ae818a6de45bdf0142049f7cb566049061c898355f1891/lupa-2.8-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:250e035fdaffe8c87093e3ebc206ac29a26131b1568ea711d780c26001ce96e7", size = 1466321, upload-time = "2026-04-15T20:07:57.627Z" },
+ { url = "https://files.pythonhosted.org/packages/de/71/3ad8cc4fc05a77dc0d3f7079348bd1cad4675a0d14c24f8e6a3ce5f008f7/lupa-2.8-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:b9bddb09acfffb4f828f790f444b11dc0cca591afea1a244d9329eea2d20c003", size = 1288577, upload-time = "2026-04-15T20:07:59.913Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/b2/1175f6d0aa7b68627fbe2f58bd1e8bea36a89d10dfd67671d2b024c96162/lupa-2.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2e64acbbd47e9b82a64405a39e0d2b36a5a7dad8ab41c0f3437f572f7d282ba3", size = 2444866, upload-time = "2026-04-15T20:08:02.753Z" },
+ { url = "https://files.pythonhosted.org/packages/92/f7/e78df680c7a0ea452daac07467ca188d63c2c00ca1c884c0a50e27eb83b5/lupa-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32e4e5103bbddcdd2458fb2ccae6c8ba11c9997c711d7e379e0d45551d109c76", size = 1778509, upload-time = "2026-04-15T20:08:21.784Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/23/0e53cabb16b2a8aa9cf1fde499c097d8942c5dab709fc8e921f3b824b18b/lupa-2.8-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7667001804657496dee9feced2daae5000b4604a3218dd8e6b7b754982ba88b8", size = 2300480, upload-time = "2026-04-15T20:08:24.394Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/85/0271227eab939921a12ebba5d17aa4cd18346aa534ca7f5da09cd0b63dd4/lupa-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:86f6f668966965b15247dc32d064cfe7be67b71e584ccfacbe2f637575296878", size = 1847445, upload-time = "2026-04-15T20:08:27.031Z" },
+]
+
[[package]]
name = "lxml"
version = "6.1.1"
@@ -3695,6 +3777,7 @@ test = [
{ name = "boto3" },
{ name = "datasets" },
{ name = "diff-match-patch" },
+ { name = "fakeredis", extra = ["lua"] },
{ name = "hvac" },
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
@@ -3737,6 +3820,7 @@ requires-dist = [
{ name = "defusedxml", specifier = ">=0.7.1" },
{ name = "diff-match-patch", marker = "extra == 'test'", specifier = ">=20200713" },
{ name = "dingtalk-stream", marker = "extra == 'bot'", specifier = ">=0.4.0" },
+ { name = "fakeredis", extras = ["lua"], marker = "extra == 'test'", specifier = ">=2.23" },
{ name = "fastapi", specifier = ">=0.128.0" },
{ name = "feedparser", specifier = ">=6.0.0" },
{ name = "firecrawl-anydoc", specifier = ">=0.2.4,<0.3" },
@@ -6091,6 +6175,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/37/c3/6eeb6034408dac0fa653d126c9204ade96b819c936e136c5e8a6897eee9c/socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3", size = 12763, upload-time = "2020-04-17T15:50:31.878Z" },
]
+[[package]]
+name = "sortedcontainers"
+version = "2.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" },
+]
+
[[package]]
name = "soupsieve"
version = "2.8.3"