Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""Compile file editing, revision reads, validation and accepted artifact storage.
These helpers serve record validation, Reduce drafts and Finalize publication preparation.
They share the task runtime without owning pipeline scheduling or publication.
"""
from __future__ import annotations
import base64
import json
from typing import TYPE_CHECKING
import yaml
from openviking.core.skill_loader import validate_skill_format
from openviking.utils.path_safety import safe_join_viking_uri
from openviking_cli.exceptions import OpenVikingError
from vikingbot.compile.hashing import content_hash, digest
from vikingbot.compile.renderer import (
_split_frontmatter,
validate_relative_file_path,
)
from vikingbot.compile.results import FileDraft, FileResponse, Group
if TYPE_CHECKING:
from vikingbot.compile.pipeline import Pipeline
def encode_content(value: str | bytes) -> dict:
"""Snapshot text or binary bytes for JSON storage; only UTF-8 bodies become text."""
if isinstance(value, bytes):
try:
value = value.decode("utf-8")
except UnicodeDecodeError:
return {"content": None, "content_base64": base64.b64encode(value).decode("ascii")}
return {"content": value, "content_base64": None}
def file_bytes(file: dict) -> bytes:
"""Restore exactly one text or Base64 body, rejecting missing or ambiguous content."""
if file is None:
raise ValueError("Missing stored file")
text, encoded = file.get("content"), file.get("content_base64")
if (text is None) == (encoded is None):
raise ValueError("A file requires exactly one text or binary body")
return text.encode("utf-8") if text is not None else base64.b64decode(encoded, validate=True)
def file_view(file: dict, reference: str | None = None) -> dict:
"""Expose text or binary metadata to models; byte snapshots remain runtime-only."""
if file is None:
raise ValueError("Missing stored file")
view = {k: v for k, v in file.items() if k != "content_base64"}
if file.get("content_base64") is not None:
raw = file_bytes(file)
view.update(binary=True, size_bytes=len(raw), sha256=content_hash(raw))
if reference is not None and file.get("content_base64") is not None:
view["content_ref"] = reference
return view
def reuse_files(response: FileResponse, snapshots: dict[str, dict]) -> None:
"""Resolve only supplied snapshot references, preserving bytes and checking their hashes."""
for draft in response.files:
if draft.content_ref is None:
continue
source = snapshots.get(draft.content_ref)
if (
source is None
or draft.content is not None
or draft.content_base64 is not None
or draft.patches
):
raise ValueError("Reuse only supplied content_ref without content or patches")
raw = file_bytes(source)
actual = content_hash(raw)
if source.get("sha256", actual) != actual:
raise ValueError("Stored file hash mismatch")
draft.content, draft.content_base64 = source["content"], source.get("content_base64")
draft.content_sha256 = draft.content_sha256 or actual
draft.content_ref = None
def apply_file(draft: FileDraft, old: str | bytes | None) -> str | bytes:
"""Assemble a revision-bound file, rejecting ambiguous, overlapping or stale patches.
Bytes outside patch anchors are preserved exactly, including whitespace and frontmatter.
"""
content = draft.content
if draft.content_base64 is not None:
content = file_bytes(draft.model_dump())
if old is None:
if draft.base_hash is not None or draft.patches:
raise ValueError("New files must not include base_hash or patches")
if content is None:
raise ValueError(
f'File "{draft.path}" is missing content. '
"Provide inline content, or set content_ref to the file you wrote with write_file, "
"relative to your scratch root."
)
return content
if draft.base_hash != content_hash(old):
raise ValueError("Old target revision does not match the requested change")
if content is not None:
if draft.patches:
raise ValueError("Full replacement cannot be mixed with patches")
return content
if isinstance(old, bytes):
raise ValueError("Binary files require full replacement, not text patches")
if not draft.patches:
raise ValueError("Existing files require content or patches")
edits = []
for patch in draft.patches:
if old.count(patch.old) != 1:
raise ValueError("Patch anchor must occur exactly once in its permitted old scope")
start = old.index(patch.old)
edits.append((start, start + len(patch.old), patch.new))
edits.sort()
if any(a[1] > b[0] for a, b in zip(edits, edits[1:], strict=False)):
raise ValueError("Patch anchors overlap")
value = old
for start, end, replacement in reversed(edits):
value = value[:start] + replacement + value[end:]
return value
async def load_old(runtime: Pipeline, path: str) -> str | bytes | None:
"""Read a selected target body once in this execution; only NOT_FOUND means absence."""
path = validate_relative_file_path(path)
if path not in runtime.old:
uri = safe_join_viking_uri(runtime.target, path)
try:
entry = await runtime.client.stat(uri)
if entry.get("isDir"):
raise ValueError("Selected old target must be a file")
payload = await runtime.client.download_bytes(uri)
try:
runtime.old[path] = payload.decode("utf-8")
except UnicodeDecodeError:
runtime.old[path] = payload
runtime.metrics["history_body_reads"] += 1
except OpenVikingError as exc:
if exc.code != "NOT_FOUND":
raise
runtime.old[path] = None
return runtime.old[path]
def validate_files(runtime: Pipeline, response, group, records, old):
"""Check output paths, revisions and file bodies before staging."""
paths = [f.path for f in response.files]
if len(set(paths)) != len(paths):
raise ValueError("A group cannot submit a path more than once")
for draft in response.files:
validate_relative_file_path(draft.path)
current = old.get(draft.path)
if draft.content_ref is not None:
raise ValueError(
"File references require execution=agent and validated scratch resolution"
)
value = apply_file(draft, current)
if draft.content_sha256 and content_hash(value) != draft.content_sha256:
raise ValueError("Artifact content hash mismatch")
if isinstance(value, bytes) and draft.path.lower().endswith((".md", ".json")):
value.decode("utf-8")
if draft.path.endswith(".json"):
json.loads(value)
async def save_files(runtime: Pipeline, response, group, records, old, *, origin=None):
"""Persist candidates with lineage; only resolved paths enter publication via accept_files."""
output = []
for draft in response.files:
previous = old.get(draft.path)
value = apply_file(draft, previous)
inputs = draft.inputs
artifact = {
"path": draft.path,
**encode_content(value),
"sha256": content_hash(value),
"base_hash": content_hash(previous) if previous is not None else None,
"owner": group.group_id,
"origin": origin or draft.path,
"source_refs": sorted(
{ref for r in records if r.record_id in inputs for ref in r.source_refs}
),
"inputs": inputs,
}
reference = f"artifacts/{digest([group.group_id, draft.path])}"
await runtime.files.put(reference, artifact)
output.append(reference)
supported = {record_id for draft in response.files for record_id in draft.inputs}
for record in records:
if runtime.status[record.record_id] not in {"failed", "prepared"}:
runtime.status[record.record_id] = (
"prepared" if record.record_id in supported else "unreferenced"
)
return output
async def accept_files(runtime: Pipeline, references):
"""Accept artifacts and optional display metadata; Markdown schemas belong to the Skill."""
for reference in references:
artifact = await runtime.files.get(reference)
metadata = {}
if artifact["path"].lower().endswith(".md"):
try:
metadata, _ = _split_frontmatter(artifact["content"])
except (ValueError, yaml.YAMLError):
pass
owner = runtime.owners.get(artifact["path"])
previous = f"artifacts/{digest([owner, artifact['path']])}"
runtime.artifacts = [ref for ref in runtime.artifacts if ref != previous]
runtime.artifacts.append(reference)
runtime.owners[artifact["path"]] = artifact["owner"]
title, description = metadata.get("title"), metadata.get("description")
runtime.catalog[artifact["path"]] = {
"path": artifact["path"],
"title": title if isinstance(title, str) and title.strip() else artifact["path"],
"description": description if isinstance(description, str) else "",
"source_refs": artifact["source_refs"],
}
def validate_skill_output(files: dict[str, str | None]) -> None:
"""Check one Skill directory and its SKILL.md using the shared format validator.
Paths are already checked by file validation. Raise ValueError for structural
errors; content quality and task-specific attachments remain the Skill's concern.
"""
directories = {path.split("/")[0] for path in files}
if len(directories) != 1 or any("/" not in path for path in files):
raise ValueError("Output must contain one <skill-name>/ directory with SKILL.md")
name = directories.pop()
main = f"{name}/SKILL.md"
if main not in files:
raise ValueError(f"Missing {main}")
if files[main] is None:
raise ValueError("SKILL.md must contain UTF-8 text")
result = validate_skill_format(files[main], strict=True, skill_dir_name=name, source_path=main)
if not result["valid"]:
raise ValueError("; ".join(issue["message"] for issue in result["errors"]))
async def repair_skill_output(runtime: Pipeline, artifacts: list[dict], error: str) -> list[str]:
"""Repair rejected files and return saved references, without publishing them.
The model receives the current files, diagnostic and source handles. Its existing
validation loop permits three submissions total. Accepted replacements retain
source lineage and target revision guards; failures propagate to the caller.
"""
ids = {i for a in artifacts for i in a["inputs"]} or set(runtime.evidence)
records = [runtime.records[i] for i in sorted(ids)]
group = Group("skill-repair", records)
snapshots = {f"artifacts/{digest([a['owner'], a['path']])}": a for a in artifacts}
old = {}
for artifact in artifacts:
if artifact["base_hash"] is not None:
previous = runtime.old.get(artifact["path"])
if previous is None or content_hash(previous) != artifact["base_hash"]:
raise ValueError(f"Stale prepared artifact: {artifact['path']}")
old[artifact["path"]] = previous
def validate(response):
"""Accept complete files only when provenance, revisions and package format agree."""
reuse_files(response, snapshots)
validate_files(runtime, response, group, records, old)
validate_skill_output(
{draft.path: apply_file(draft, old.get(draft.path)) for draft in response.files}
)
response = await runtime.model.ask(
"skill_repair",
runtime.system
+ "\nFix the reported errors according to the original Skill and user instruction. "
"Preserve valid content. Submit all files, including unchanged ones, with supporting "
"input IDs. Read assigned evidence if needed. Keep existing base_hash values. "
"Preserve binary attachments using their supplied content_ref; never generate Base64.",
{
"error": error,
"files": [file_view(a, ref) for ref, a in snapshots.items()],
"inputs": [
{"id": r.record_id, "payload": {"source_ranges": r.source_refs}} for r in records
],
},
FileResponse,
validate,
)
repaired = await save_files(runtime, response, group, records, old)
runtime.artifacts.clear()
runtime.owners.clear()
runtime.catalog.clear()
await accept_files(runtime, repaired)
return repaired