Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Read the R / N / F / V snapshots used by resource update planning.
This module gathers the typed R/N/F/V inputs used by the canonical resource
update plan.
- ``R`` normalized request intent — target, processing mode, and scalar fields
such as tags that this request explicitly wants to mutate.
- ``N`` new-artifact manifest — the files a parser produced, read from the
parse output store. Canonical add-resource prepares it once before planning:
image references are rewritten to their final URI and their final-byte md5 is
retained in the resulting inventory.
- ``F`` target file tree — one unbounded ``tree`` call. Any permission-denied
subtree (or a truncated scan) marks the snapshot incomplete so the planner
refuses deletions instead of treating unreadable files as removed.
- ``V`` target vectors — lightweight L0/L1/L2 records under the target URI.
The canonical ``add_resources`` path resolves these snapshots into a
``ResourceDiffResult`` before building a ``ContextUpdatePlan``.
"""
from __future__ import annotations
import asyncio
import json
import logging
from collections import Counter
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, Literal, Mapping, Tuple
from openviking.concurrency import bounded_map
from openviking.storage.internal_names import is_storage_internal_name
from openviking.storage.resource_rnfv import (
CONTROL_BASENAMES,
FormalEntry,
FormalTreeSnapshot,
NewArtifactSnapshot,
NewEntry,
RequestIntent,
RNFVSnapshot,
VectorIndexSnapshot,
VectorRecordSnapshot,
canonical_vector_records_by_level,
)
from openviking.utils.content_hash import content_md5
from openviking.utils.log_correlation import log_correlation
logger = logging.getLogger(__name__)
class ContentState(str, Enum):
"""Relationship between the new artifact (N) and formal tree (F).
F is the source of truth for whether content currently exists. Vector
records are deliberately not part of this axis; their presence is expressed
independently by :class:`IndexState`.
"""
# Neither N nor F contains this path.
ABSENT = "absent"
# N and F contain the same file bytes, or the same directory node.
UNCHANGED = "unchanged"
# N contains a path that has no formal file and no prior vector records.
ADDED = "added"
# N and F both contain a file, but their bytes differ.
MODIFIED = "modified"
# F contains a path omitted from N.
DELETED = "deleted"
# N contains a path absent from F while V still has records for that path.
RESTORE = "restore"
# N and F contain the path with different node kinds (file versus directory).
REPLACE_KIND = "replace_kind"
class IndexState(str, Enum):
"""Health of V for the node kind selected by N/F.
A file is valid only with L2. A directory is valid only with both L0 and
L1. This axis is independent from content change, so an unchanged file can
still require index repair.
"""
# No formal node and no vector record exist for the path.
ABSENT = "absent"
# All and only the levels expected for the current node kind are present.
COMPLETE = "complete"
# None of the levels expected for the current node kind are present.
MISSING = "missing"
# Some expected levels are present, but the complete expected set is not.
PARTIAL = "partial"
# Expected levels exist but are derived from content known to be out of date.
STALE = "stale"
# V belongs to content removed by the current update.
OBSOLETE = "obsolete"
# V has records while neither N nor F has a corresponding content node.
ORPHAN = "orphan"
# V contains a level that is invalid for the current file/directory kind.
LEVEL_CONFLICT = "level_conflict"
@dataclass(frozen=True)
class ResourceDiffEntry:
"""One path's independent content and index facts after RNFV resolution.
``content_state`` determines formal-tree mutation. ``index_state`` determines
vector repair or cleanup. ``md5`` is the final new-file fingerprint when the
path is a file; directories intentionally never carry an aggregate MD5.
"""
relative_path: str
content_state: ContentState
index_state: IndexState
old_kind: str | None = None
new_kind: str | None = None
md5: str | None = None
level_states: Mapping[int, IndexState] = field(default_factory=dict)
level_md5s: Mapping[int, str] = field(default_factory=dict)
@dataclass(frozen=True)
class ResourceDiffResult:
entries: Mapping[str, ResourceDiffEntry]
@dataclass(frozen=True)
class ArtifactInventory:
"""Prepared N snapshot plus paths relative to its parse artifact.
``entries`` are rooted at the final resource URI; ``artifact_paths`` point
to the same files under the parse artifact. The latter keeps diff body
fallbacks independent from the parser's optional document wrapper.
"""
entries: Mapping[str, NewEntry]
artifact_paths: Mapping[str, str]
rewritten_paths: frozenset[str] = frozenset()
class InlineBytesStore:
"""Minimal parse-output store backed by a single file's final bytes.
The single-file write path has no parse artifact; it already holds the
fully rendered bytes. This store lets the shared RNFV plan pipeline read the
same bytes for body fallbacks and content commit without inventing a real
artifact backend. The ref is an opaque placeholder; only ``read_bytes`` is
exercised.
"""
backend = "inline"
def __init__(self, data: bytes) -> None:
self._data = data
async def read_bytes(self, ref: Any, rel_path: str) -> bytes:
del ref, rel_path
return self._data
def make_inline_file_inventory(final_bytes: bytes) -> ArtifactInventory:
"""Build a single-file N snapshot from already-rendered bytes."""
return ArtifactInventory(
entries={"": NewEntry(md5=content_md5(final_bytes), is_dir=False)},
artifact_paths={"": ""},
)
def count_tree_entry_kinds(
entries: Mapping[str, NewEntry | FormalEntry],
) -> tuple[int, int, bool]:
"""Return business file/dir counts and whether a logical root is present.
Directory plans add ``relative_path=""`` as a traversal-only root. Keeping
it separate prevents logs from presenting that synthetic node as a file or
directory stored below the target. A flat resource uses the same empty path
with ``is_dir=False`` and is therefore counted as one real file.
"""
logical_root = bool((root := entries.get("")) is not None and root.is_dir)
files = sum(not entry.is_dir for entry in entries.values())
directories = sum(entry.is_dir for path, entry in entries.items() if path)
return files, directories, logical_root
def _index_state(
records: tuple[VectorRecordSnapshot, ...],
*,
kind: str | None,
content_state: ContentState,
md5: str | None,
) -> IndexState:
"""Classify V with strict precedence.
Content removed by this update makes its V record obsolete. A vector with
neither new nor formal content is an orphan. For an existing node, invalid
levels win over missing or stale checks; then missing expected levels, then
content/fingerprint staleness, and finally complete.
"""
if content_state is ContentState.DELETED:
return IndexState.OBSOLETE if records else IndexState.ABSENT
if kind is None:
return IndexState.ORPHAN if records else IndexState.ABSENT
valid_levels = {2} if kind == "file" else {0, 1}
levels = {record.level for record in records}
if levels - valid_levels:
return IndexState.LEVEL_CONFLICT
if not levels:
return IndexState.MISSING
if levels != valid_levels:
return IndexState.PARTIAL
if content_state == ContentState.MODIFIED:
return IndexState.STALE
if kind == "file" and md5:
record_md5 = next(
(str(record.fields.get("md5") or "") for record in records if record.level == 2),
"",
)
if record_md5 and record_md5 != md5:
return IndexState.STALE
return IndexState.COMPLETE
async def resolve_resource_diff(
snapshot: RNFVSnapshot,
*,
store: Any,
artifact_ref: Any,
target: Any,
artifact_paths: Mapping[str, str] | None = None,
concurrency: int | None = None,
) -> ResourceDiffResult:
"""Resolve R/N/F/V into final states, including bounded body fallbacks."""
snapshot.validate_for_planning()
if (
not snapshot.new.complete
or not snapshot.formal.complete
or (snapshot.request.vectorize and not snapshot.vectors.complete)
):
raise ValueError("cannot resolve an incomplete RNFV snapshot")
new = snapshot.new.entries
formal = snapshot.formal.entries
vector_records = snapshot.vectors.records_by_id if snapshot.request.vectorize else {}
records_by_rel: dict[str, list[VectorRecordSnapshot]] = {}
for record in vector_records.values():
records_by_rel.setdefault(record.relative_path, []).append(record)
states: dict[str, tuple[ContentState, str | None]] = {}
compare_paths: list[str] = []
hash_paths: list[str] = []
md5_fast_path_count = 0
keys = set(new) | set(formal) | set(records_by_rel)
for rel_path in sorted(keys):
new_entry = new.get(rel_path)
old_entry = formal.get(rel_path)
if new_entry is not None and old_entry is not None:
new_kind = "directory" if new_entry.is_dir else "file"
old_kind = "directory" if old_entry.is_dir else "file"
if new_kind != old_kind:
states[rel_path] = (ContentState.REPLACE_KIND, new_entry.md5 or None)
elif new_entry.is_dir:
states[rel_path] = (ContentState.UNCHANGED, None)
else:
canonical_records, _ = canonical_vector_records_by_level(
records_by_rel.get(rel_path, ())
)
l2_record = canonical_records.get(2)
l2_md5 = str(l2_record.fields.get("md5") or "") if l2_record else ""
if new_entry.md5 and l2_md5:
md5_fast_path_count += 1
state = (
ContentState.UNCHANGED if new_entry.md5 == l2_md5 else ContentState.MODIFIED
)
states[rel_path] = (state, new_entry.md5)
else:
compare_paths.append(rel_path)
elif new_entry is not None:
state = ContentState.RESTORE if records_by_rel.get(rel_path) else ContentState.ADDED
states[rel_path] = (state, new_entry.md5 or None)
if not new_entry.is_dir and not new_entry.md5:
hash_paths.append(rel_path)
elif old_entry is not None:
states[rel_path] = (ContentState.DELETED, None)
else:
states[rel_path] = (ContentState.ABSENT, None)
async def compare(rel_path: str) -> tuple[str, ContentState, str]:
artifact_path = (artifact_paths or {}).get(rel_path, rel_path)
new_bytes, old_bytes = await asyncio.gather(
store.read_bytes(artifact_ref, artifact_path),
target.read_file(rel_path),
)
from openviking.utils.content_hash import content_md5
return (
rel_path,
ContentState.UNCHANGED if new_bytes == old_bytes else ContentState.MODIFIED,
content_md5(new_bytes),
)
async def hash_new(rel_path: str) -> tuple[str, str]:
artifact_path = (artifact_paths or {}).get(rel_path, rel_path)
new_bytes = await store.read_bytes(artifact_ref, artifact_path)
from openviking.utils.content_hash import content_md5
return rel_path, content_md5(new_bytes)
if concurrency is None:
from openviking_cli.utils.config import get_openviking_config
concurrency = int(
get_openviking_config().queue_workers.add_resource.file_operation_concurrency
)
for rel_path, state, md5 in await bounded_map(
compare_paths, compare, concurrency=max(1, concurrency)
):
states[rel_path] = (state, md5)
for rel_path, md5 in await bounded_map(hash_paths, hash_new, concurrency=max(1, concurrency)):
state, _ = states[rel_path]
states[rel_path] = (state, md5)
result: dict[str, ResourceDiffEntry] = {}
for rel_path in sorted(keys):
state, md5 = states[rel_path]
new_entry = new.get(rel_path)
old_entry = formal.get(rel_path)
new_kind = ("directory" if new_entry.is_dir else "file") if new_entry is not None else None
old_kind = ("directory" if old_entry.is_dir else "file") if old_entry is not None else None
current_kind = (
new_kind if state not in {ContentState.DELETED, ContentState.ABSENT} else None
)
records = tuple(records_by_rel.get(rel_path, ()))
canonical_records, _ = canonical_vector_records_by_level(records)
result[rel_path] = ResourceDiffEntry(
relative_path=rel_path,
content_state=state,
index_state=_index_state(
tuple(canonical_records.values()),
kind=current_kind,
content_state=state,
md5=md5,
),
old_kind=old_kind,
new_kind=new_kind,
md5=md5,
)
resolved = ResourceDiffResult(entries=result)
n_files, n_dirs, n_has_root = count_tree_entry_kinds(new)
f_files, f_dirs, f_has_root = count_tree_entry_kinds(formal)
file_entries = [
entry
for path, entry in resolved.entries.items()
if path and (entry.new_kind or entry.old_kind) == "file"
]
directory_entries = [
entry
for path, entry in resolved.entries.items()
if path and (entry.new_kind or entry.old_kind) == "directory"
]
root_entry = resolved.entries.get("")
file_content_counts = Counter(entry.content_state.value for entry in file_entries)
directory_content_counts = Counter(entry.content_state.value for entry in directory_entries)
file_index_counts = Counter(entry.index_state.value for entry in file_entries)
directory_index_counts = Counter(entry.index_state.value for entry in directory_entries)
logger.info(
"[ResourceDiffResult] %s target=%s n_files=%d n_dirs=%d f_files=%d f_dirs=%d "
"logical_root=%s v_records=%d root_state=%s root_index=%s "
"md5_fast_path=%d body_compared=%d new_files_hashed=%d "
"states={file:%s,dir:%s} index={file:%s,dir:%s}",
log_correlation(),
snapshot.request.target_uri,
n_files,
n_dirs,
f_files,
f_dirs,
str(n_has_root or f_has_root).lower(),
len(vector_records),
root_entry.content_state.value if root_entry is not None else "none",
root_entry.index_state.value if root_entry is not None else "none",
md5_fast_path_count,
len(compare_paths),
len(hash_paths),
dict(file_content_counts),
dict(directory_content_counts),
dict(file_index_counts),
dict(directory_index_counts),
)
return resolved
def _is_excluded_rel_path(rel_path: str) -> bool:
"""Control sidecars and storage-internal entries are never business files."""
if not rel_path:
return True
for segment in rel_path.split("/"):
if is_storage_internal_name(segment) or segment in CONTROL_BASENAMES:
return True
return False
async def read_target_file_snapshot(
viking_fs: Any,
target_uri: str,
*,
ctx: Any,
root_is_file: bool = False,
) -> Tuple[Dict[str, FormalEntry], bool]:
"""Return ``(rel_path -> FormalEntry, complete)`` for the formal tree.
``complete`` is False when any entry is permission-denied, because a subtree
we cannot see must not be interpreted as absent (which would drive deletion).
"""
if root_is_file:
stat = await viking_fs.stat(target_uri, ctx=ctx, skip_count=True)
return {"": FormalEntry(is_dir=bool(stat.get("isDir")))}, True
entries = await viking_fs.tree(
target_uri,
output="original",
show_all_hidden=True,
node_limit=None,
level_limit=None,
ctx=ctx,
)
files: Dict[str, FormalEntry] = {}
complete = True
for entry in entries:
if entry.get("access") == "denied":
complete = False
continue
rel_path = str(entry.get("rel_path") or "").strip("/")
if _is_excluded_rel_path(rel_path):
continue
files[rel_path] = FormalEntry(is_dir=bool(entry.get("isDir")))
return files, complete
async def prepare_artifact_inventory(
store: Any,
ref: Any,
*,
doc_rel: str = "",
target_root_uri: str | None = None,
root_is_file: bool = False,
) -> ArtifactInventory:
"""Prepare an artifact once and return its final-byte N snapshot.
The one recursive walk supplies the file inventory consumed by RNFV and,
when a final target URI is known, rewrites mapped markdown image references
in place. Rewritten markdown immediately receives an updated manifest MD5,
so the next diff compares the exact bytes that would be uploaded.
"""
from openviking.parse.image_rewrite import IMAGE_MAPPINGS_FILENAME, _rewrite_content
from openviking.parse.output import read_artifact_manifest, write_artifact_manifest
from openviking.utils.content_hash import content_md5
base = doc_rel.strip("/")
prefix = f"{base}/" if base else ""
md5_by_artifact_rel = await read_artifact_manifest(store, ref)
entries: Dict[str, NewEntry] = {}
artifact_paths: Dict[str, str] = {}
rewritten_paths: set[str] = set()
def target_relative(artifact_rel: str) -> str:
if base and artifact_rel.startswith(prefix):
return artifact_rel[len(prefix) :]
return artifact_rel
if root_is_file:
return ArtifactInventory(
entries={"": NewEntry(md5=md5_by_artifact_rel.get(base, ""), is_dir=False)},
artifact_paths={"": base},
)
async def walk(
directory: str,
mapping_dir: str = "",
inherited_mappings: Mapping[str, Mapping[str, str]] | None = None,
) -> None:
directory_entries = await store.list(ref, directory)
current_mappings = inherited_mappings or {}
current_mapping_dir = mapping_dir
mapping_entry = next(
(entry for entry in directory_entries if entry.name == IMAGE_MAPPINGS_FILENAME),
None,
)
if mapping_entry is not None:
try:
loaded = json.loads(
(await store.read_bytes(ref, mapping_entry.rel_path)).decode("utf-8")
)
except Exception as exc:
logger.warning(
"[ArtifactInventory] Ignoring unreadable image mapping sidecar %s: %s",
mapping_entry.rel_path,
exc,
)
loaded = None
if not isinstance(loaded, dict):
if loaded is not None:
logger.warning(
"[ArtifactInventory] Ignoring invalid image mapping sidecar %s",
mapping_entry.rel_path,
)
else:
current_mappings = loaded
current_mapping_dir = directory
available_files = {
entry.name
for entry in directory_entries
if not entry.is_dir and not entry.name.startswith(".")
}
for entry in directory_entries:
if _is_excluded_rel_path(entry.rel_path):
continue
relative_path = target_relative(entry.rel_path)
if entry.is_dir:
if relative_path:
entries[relative_path] = NewEntry(is_dir=True)
await walk(entry.rel_path, current_mapping_dir, current_mappings)
continue
if not relative_path:
continue
md5 = md5_by_artifact_rel.get(entry.rel_path, "")
if target_root_uri and entry.name.lower().endswith((".md", ".markdown")):
mapping_key = (
entry.rel_path[len(current_mapping_dir) + 1 :]
if current_mapping_dir and entry.rel_path.startswith(current_mapping_dir + "/")
else entry.rel_path
)
path_mappings = current_mappings.get(mapping_key)
if isinstance(path_mappings, dict) and path_mappings:
content = (await store.read_bytes(ref, entry.rel_path)).decode("utf-8")
target_uri = f"{target_root_uri.rstrip('/')}/{relative_path}"
rewritten, count = _rewrite_content(
content,
target_uri.rsplit("/", 1)[0],
available_files,
{str(key): str(value) for key, value in path_mappings.items()},
)
if count:
final_bytes = rewritten.encode("utf-8")
await store.write_bytes(ref, entry.rel_path, final_bytes)
md5 = content_md5(final_bytes)
md5_by_artifact_rel[entry.rel_path] = md5
rewritten_paths.add(relative_path)
entries[relative_path] = NewEntry(md5=md5, is_dir=False)
artifact_paths[relative_path] = entry.rel_path
await walk(base)
if rewritten_paths:
await write_artifact_manifest(store, ref, md5_by_artifact_rel)
return ArtifactInventory(
entries=entries,
artifact_paths=artifact_paths,
rewritten_paths=frozenset(rewritten_paths),
)
async def build_rnfv_snapshot(
*,
viking_fs: Any,
vikingdb: Any,
store: Any,
artifact_ref: Any,
target_uri: str,
ctx: Any,
doc_rel: str = "",
request_intent: RequestIntent | None = None,
root_is_file: bool = False,
target_preexisting: bool = True,
formal_snapshot: tuple[Dict[str, FormalEntry], bool] | None = None,
artifact_inventory: ArtifactInventory | None = None,
vector_scope: Literal["subtree", "self"] = "subtree",
vector_inventory: Mapping[str, Mapping[str, Any]] | None = None,
) -> RNFVSnapshot:
"""Read the complete R/N/F/V inputs without deriving an executable plan.
``vector_scope="self"`` restricts the subtree-strategy V read to the target URI itself
(all levels), which the single-file write path uses so an incremental
update never reads sibling records under a shared parent directory.
"""
request = request_intent or RequestIntent(
target_uri=target_uri, processing_mode="semantic_and_vectors"
)
async def read_artifact() -> ArtifactInventory:
return artifact_inventory or await prepare_artifact_inventory(
store, artifact_ref, doc_rel=doc_rel, root_is_file=root_is_file
)
async def read_formal() -> tuple[Dict[str, FormalEntry], bool]:
if formal_snapshot is not None:
return formal_snapshot
if not target_preexisting:
return {}, True
return await read_target_file_snapshot(
viking_fs, target_uri, ctx=ctx, root_is_file=root_is_file
)
projection = request.required_vector_fields()
if request.vectorize and (root_is_file or request.processing_mode == "vectors_only"):
projection = projection | {"abstract"}
tasks = [
asyncio.create_task(read_artifact()),
asyncio.create_task(read_formal()),
]
if request.vectorize and vector_inventory is None:
tasks.append(
asyncio.create_task(
_read_incremental_vector_inventory(
vikingdb,
target_uri=target_uri,
ctx=ctx,
projection=projection,
vector_scope=vector_scope,
)
)
)
try:
results = await asyncio.gather(*tasks)
except BaseException:
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
raise
artifact, formal = results[:2]
if not request.vectorize:
inventory: Dict[str, Dict[str, Any]] = {}
elif vector_inventory is not None:
inventory = dict(vector_inventory)
else:
inventory = results[2]
target_files, files_complete = formal
base = target_uri.rstrip("/")
prefix = base + "/"
vector_records: Dict[str, VectorRecordSnapshot] = {}
for record_id, record in inventory.items():
level = int(record.get("level", -1))
uri = str(record.get("uri") or "")
rel = "" if uri == base else uri[len(prefix) :] if uri.startswith(prefix) else None
if rel is None:
raise RuntimeError(f"Vector inventory returned an out-of-scope URI: {uri}")
fields = {
field: record[field] for field in projection - {"id", "uri", "level"} if field in record
}
vector_records[record_id] = VectorRecordSnapshot(
record_id=record_id,
uri=uri,
relative_path=rel,
level=level,
fields=fields,
)
return RNFVSnapshot(
request=request,
new=NewArtifactSnapshot(entries=artifact.entries),
formal=FormalTreeSnapshot(entries=target_files, complete=files_complete),
vectors=VectorIndexSnapshot(
records_by_id=vector_records,
projected_fields=projection,
),
)
async def _read_incremental_vector_inventory(
vikingdb: Any,
*,
target_uri: str,
ctx: Any,
projection: frozenset[str],
vector_scope: Literal["subtree", "self"] = "subtree",
) -> Dict[str, Dict[str, Any]]:
"""Read the lightweight V snapshot used to build RNFV."""
return await vikingdb.get_incremental_inventory_under_uri(
target_uri,
ctx=ctx,
output_fields=sorted(projection),
# The shared inventory API expresses an exact URI as a non-recursive
# scope without direct children. Keep its richer recursive/direct-child
# contract instead of adding a parallel depth parameter solely for the
# single-file write path.
recursive=vector_scope == "subtree",
include_direct_children=False,
)
__all__ = [
"ArtifactInventory",
"ContentState",
"IndexState",
"InlineBytesStore",
"ResourceDiffEntry",
"ResourceDiffResult",
"build_rnfv_snapshot",
"count_tree_entry_kinds",
"make_inline_file_inventory",
"prepare_artifact_inventory",
"read_target_file_snapshot",
]