Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Typed R/F/V inputs for planning maintenance reindex operations."""
from __future__ import annotations
import asyncio
import json
from dataclasses import dataclass, field
from typing import Any, Mapping
from openviking.concurrency import bounded_map
from openviking.core.namespace import context_type_for_uri
from openviking.server.error_mapping import is_not_found_error
from openviking.storage.abstract_overview import body_for_preview
from openviking.storage.internal_names import is_storage_internal_name
from openviking.storage.resource_diff import (
ContentState,
IndexState,
ResourceDiffEntry,
ResourceDiffResult,
)
from openviking.storage.resource_rnfv import (
RequestIntent,
VectorIndexSnapshot,
VectorRecordSnapshot,
canonical_vector_records_by_level,
)
from openviking.utils.content_hash import content_md5
from openviking_cli.exceptions import NotFoundError
@dataclass(frozen=True)
class RFVEntry:
"""One current formal-tree node and its per-level source fingerprints."""
uri: str
relative_path: str
is_dir: bool
level_md5s: Mapping[int, str] = field(default_factory=dict)
def __post_init__(self) -> None:
expected = {0, 1} if self.is_dir else {2}
invalid = set(self.level_md5s) - expected
if invalid:
raise ValueError(f"invalid RFV levels for {self.relative_path}: {sorted(invalid)}")
@dataclass(frozen=True)
class RFVFormalSnapshot:
entries: Mapping[str, RFVEntry]
complete: bool = True
@dataclass(frozen=True)
class RFVSnapshot:
request: RequestIntent
formal: RFVFormalSnapshot
vectors: VectorIndexSnapshot
source_contents: Mapping[tuple[str, int], str | bytes] = field(default_factory=dict)
source_raw_contents: Mapping[tuple[str, int], str | bytes] = field(default_factory=dict)
source_metadata: Mapping[str, Any] | None = None
input_only_paths: frozenset[str] = frozenset()
def _relative_uri(root: str, uri: str) -> str | None:
root = root.rstrip("/")
if uri == root:
return ""
prefix = root + "/"
return uri[len(prefix) :] if uri.startswith(prefix) else None
def _is_sidecar(relative_path: str) -> bool:
return relative_path.rsplit("/", 1)[-1] in {".abstract.md", ".overview.md"}
def _is_control_file(relative_path: str) -> bool:
name = relative_path.rsplit("/", 1)[-1]
return name in {".source.json", ".image_mappings.json", ".artifact_manifest.json"}
async def build_rfv_snapshot(
*,
viking_fs: Any,
vikingdb: Any,
target_uri: str,
ctx: Any,
request_intent: RequestIntent,
recursive: bool = True,
source_read_concurrency: int = 8,
root_is_dir: bool | None = None,
formal_inventory: tuple[bool, list[Mapping[str, Any]], bool] | None = None,
vector_inventory: Mapping[str, Mapping[str, Any]] | None = None,
root_exists: bool = True,
) -> RFVSnapshot:
"""Read one complete F/V inventory and each selected embedding source once.
``root_exists=False`` means the target root is already absent from the formal
tree (for example, an entire ``resources`` or skill package removed outside the
normal APIs). No formal node is synthesized for it, so every vector record under
the root resolves to an orphan and is deleted during the same reindex.
"""
root = target_uri.rstrip("/")
projection = request_intent.required_vector_fields() | {"abstract"}
if target_uri.rstrip("/").startswith(("viking://user/", "viking://agent/skills/")):
projection = projection | {"name", "description", "tags"}
resolved_root_is_dir = root_is_dir
if formal_inventory is not None:
resolved_root_is_dir = bool(formal_inventory[0])
if resolved_root_is_dir is None and not recursive:
stat = await viking_fs.stat(root, ctx=ctx, skip_count=True)
resolved_root_is_dir = bool(stat.get("isDir", stat.get("is_dir")))
include_direct_children = bool(resolved_root_is_dir) and not recursive
async def read_formal_inventory() -> tuple[bool, list[Mapping[str, Any]], bool]:
if formal_inventory is not None:
return formal_inventory
is_dir = resolved_root_is_dir
if is_dir is None:
stat = await viking_fs.stat(root, ctx=ctx, skip_count=True)
is_dir = bool(stat.get("isDir", stat.get("is_dir")))
if not is_dir:
return is_dir, [], True
if not recursive:
entries = await viking_fs.ls(
root,
node_limit=None,
show_all_hidden=True,
ctx=ctx,
)
return is_dir, list(entries), True
raw_entries = await viking_fs.tree(
root,
output="original",
show_all_hidden=True,
node_limit=None,
level_limit=None,
ctx=ctx,
)
complete = not any(entry.get("access") == "denied" for entry in raw_entries)
return is_dir, list(raw_entries), complete
async def read_vector_inventory() -> Mapping[str, Mapping[str, Any]]:
if vector_inventory is not None:
return vector_inventory
return await vikingdb.get_incremental_inventory_under_uri(
root,
ctx=ctx,
output_fields=sorted(projection),
recursive=recursive,
include_direct_children=include_direct_children,
)
formal_task = asyncio.create_task(read_formal_inventory())
vector_task = asyncio.create_task(read_vector_inventory())
try:
(root_is_dir, raw_entries, complete), inventory = await asyncio.gather(
formal_task, vector_task
)
except BaseException:
for task in (formal_task, vector_task):
if not task.done():
task.cancel()
await asyncio.gather(formal_task, vector_task, return_exceptions=True)
raise
complete = complete and not any(entry.get("access") == "denied" for entry in raw_entries)
if not complete:
raise ValueError("cannot build RFV snapshot from an incomplete formal tree")
nodes: dict[str, tuple[str, bool]] = (
{"": (root, root_is_dir)} if root_exists else {}
)
sidecars: dict[tuple[str, int], str] = {}
source_metadata_uri: str | None = None
for raw in raw_entries:
if raw.get("access") == "denied":
continue
uri = str(raw.get("uri") or "")
relative = _relative_uri(root, uri)
if relative is None:
complete = False
continue
relative_path = relative.strip("/")
if not uri or not relative_path:
continue
name = relative_path.rsplit("/", 1)[-1]
if _is_sidecar(relative_path):
parent = relative_path.rsplit("/", 1)[0] if "/" in relative_path else ""
level = 0 if name == ".abstract.md" else 1
sidecars[(parent, level)] = uri
continue
if relative_path == ".source.json":
source_metadata_uri = uri
continue
if is_storage_internal_name(name) or _is_control_file(relative_path):
continue
nodes[relative_path] = (uri, bool(raw.get("isDir")))
if root_exists and root_is_dir and not recursive:
for level, name in ((0, ".abstract.md"), (1, ".overview.md")):
sidecars[("", level)] = f"{root}/{name}"
records_by_uri_level: dict[tuple[str, int], Mapping[str, Any]] = {}
for record in inventory.values():
uri = str(record.get("uri") or "")
try:
level = int(record.get("level", -1))
except (TypeError, ValueError):
continue
records_by_uri_level.setdefault((uri, level), record)
source_contents: dict[tuple[str, int], str | bytes] = {}
source_raw_contents: dict[tuple[str, int], str | bytes] = {}
level_md5s: dict[str, dict[int, str]] = {path: {} for path in nodes}
source_jobs: list[tuple[str, str, int, bool]] = []
for path, (uri, is_dir) in nodes.items():
if include_direct_children and path:
continue
if is_dir:
for level in (0, 1):
source_uri = sidecars.get((path, level))
if source_uri:
source_jobs.append((path, source_uri, level, True))
else:
source_jobs.append((path, uri, 2, False))
async def read_source(job: tuple[str, str, int, bool]) -> None:
path, source_uri, level, is_sidecar = job
try:
raw = await viking_fs.read_file_bytes(source_uri, ctx=ctx)
except Exception as exc:
if is_sidecar and (
isinstance(exc, (FileNotFoundError, KeyError, NotFoundError))
or is_not_found_error(exc)
):
return
raise
node_uri = nodes[path][0]
if is_sidecar:
source_raw_contents[(node_uri, level)] = raw
body = body_for_preview(raw)
if not body.strip():
return
source_contents[(node_uri, level)] = body
source_md5 = content_md5(body.encode("utf-8"))
level_md5s[path][level] = source_md5
source = body
else:
source_md5 = content_md5(raw)
level_md5s[path][level] = source_md5
source = raw
record = records_by_uri_level.get((node_uri, level))
keep_file_source = (
not is_sidecar
and path.rsplit("/", 1)[-1] == "SKILL.md"
and context_type_for_uri(root) == "skill"
)
if keep_file_source or (
not is_sidecar
and (
request_intent.processing_mode != "vectors_only"
or request_intent.force
or record is None
or str(record.get("md5") or "") != source_md5
)
):
source_contents[(node_uri, level)] = source
await bounded_map(
source_jobs,
read_source,
concurrency=max(1, source_read_concurrency),
)
source_metadata = None
if source_metadata_uri is not None:
try:
raw_metadata = await viking_fs.read_file_bytes(source_metadata_uri, ctx=ctx)
parsed_metadata = json.loads(raw_metadata)
except (UnicodeDecodeError, json.JSONDecodeError):
parsed_metadata = None
if isinstance(parsed_metadata, dict):
kind = parsed_metadata.get("kind") or parsed_metadata.get("type")
source_uri = parsed_metadata.get("uri") or parsed_metadata.get("source")
if kind and source_uri:
source_metadata = {
"kind": str(kind),
"uri": str(source_uri),
"path": str(source_uri),
}
vector_records: dict[str, VectorRecordSnapshot] = {}
for record_id, record in inventory.items():
uri = str(record.get("uri") or "")
relative_path = _relative_uri(root, uri)
if relative_path is None or (
not recursive
and relative_path
and (not include_direct_children or "/" in relative_path)
):
raise ValueError(f"RFV vector record is outside target: {uri}")
level = int(record.get("level", -1))
fields = {
name: record[name] for name in projection - {"id", "uri", "level"} if name in record
}
vector_records[str(record_id)] = VectorRecordSnapshot(
str(record_id), uri, relative_path, level, fields
)
formal_entries = {
path: RFVEntry(uri, path, is_dir, level_md5s[path]) for path, (uri, is_dir) in nodes.items()
}
return RFVSnapshot(
request=request_intent,
formal=RFVFormalSnapshot(formal_entries, complete=complete),
vectors=VectorIndexSnapshot(vector_records, projection, complete=True),
source_contents=source_contents,
source_raw_contents=source_raw_contents,
source_metadata=source_metadata,
input_only_paths=frozenset(
path for path in nodes if include_direct_children and path and "/" not in path
),
)
def _summarize_index_state(
*,
expected_levels: set[int],
level_states: Mapping[int, IndexState],
invalid_levels: set[int],
) -> IndexState:
if invalid_levels:
return IndexState.LEVEL_CONFLICT
values = set(level_states.values())
if not expected_levels:
return IndexState.ABSENT
if not level_states:
return IndexState.MISSING
if values == {IndexState.ABSENT}:
return IndexState.ABSENT
if IndexState.ORPHAN in values:
return IndexState.ORPHAN
if IndexState.STALE in values:
return IndexState.STALE
if IndexState.MISSING in values:
return IndexState.PARTIAL if len(level_states) > 1 else IndexState.MISSING
return IndexState.COMPLETE
def resolve_rfv_state(snapshot: RFVSnapshot) -> ResourceDiffResult:
"""Resolve current formal nodes and vectors without inventing an N snapshot."""
if not snapshot.formal.complete or not snapshot.vectors.complete:
raise ValueError("cannot resolve an incomplete RFV snapshot")
required = snapshot.request.required_vector_fields()
if not snapshot.vectors.projected_fields.issuperset(required):
raise ValueError("RFV vector projection misses required fields")
root = snapshot.request.target_uri.rstrip("/")
for path, formal in snapshot.formal.entries.items():
expected_uri = root if not path else f"{root}/{path}"
if formal.relative_path != path or formal.uri.rstrip("/") != expected_uri:
raise ValueError(f"RFV formal entry is outside target: {formal.uri}")
for record in snapshot.vectors.records_by_id.values():
expected_uri = root if not record.relative_path else f"{root}/{record.relative_path}"
if record.uri.rstrip("/") != expected_uri:
raise ValueError(f"RFV vector record is outside target: {record.uri}")
records_by_path: dict[str, list[VectorRecordSnapshot]] = {}
for record in snapshot.vectors.records_by_id.values():
records_by_path.setdefault(record.relative_path, []).append(record)
entries: dict[str, ResourceDiffEntry] = {}
all_paths = set(snapshot.formal.entries) | set(records_by_path)
for path in sorted(all_paths):
formal = snapshot.formal.entries.get(path)
records = records_by_path.get(path, [])
canonical, _duplicates = canonical_vector_records_by_level(records)
if formal is None:
entries[path] = ResourceDiffEntry(
relative_path=path,
content_state=ContentState.ABSENT,
index_state=IndexState.ORPHAN,
old_kind=None,
new_kind=None,
)
continue
kind = "directory" if formal.is_dir else "file"
expected_levels = {0, 1} if formal.is_dir else {2}
invalid_levels = set(canonical) - expected_levels
level_states: dict[int, IndexState] = {}
for level in sorted(expected_levels):
source_md5 = str(formal.level_md5s.get(level) or "")
record = canonical.get(level)
if formal.is_dir and not source_md5:
level_states[level] = (
IndexState.MISSING
if snapshot.request.processing_mode != "vectors_only"
else IndexState.ABSENT
)
continue
if record is None:
level_states[level] = IndexState.MISSING
continue
stored_md5 = str(record.fields.get("md5") or "")
level_states[level] = (
IndexState.STALE
if snapshot.request.force or not source_md5 or stored_md5 != source_md5
else IndexState.COMPLETE
)
index_state = _summarize_index_state(
expected_levels=expected_levels,
level_states=level_states,
invalid_levels=invalid_levels,
)
entries[path] = ResourceDiffEntry(
relative_path=path,
content_state=ContentState.UNCHANGED,
index_state=index_state,
old_kind=kind,
new_kind=kind,
md5=(formal.level_md5s.get(2) if not formal.is_dir else None),
level_states=level_states,
level_md5s=dict(formal.level_md5s),
)
return ResourceDiffResult(entries=entries)
__all__ = [
"RFVEntry",
"RFVFormalSnapshot",
"RFVSnapshot",
"build_rfv_snapshot",
"resolve_rfv_state",
]