Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Post-commit image URI rewriting for OpenViking.
Scans markdown files in VikingFS after source commit and rewrites local
image references to viking:// URIs, driven by the ``.image_mappings.json``
sidecars that ``_ingest_local_images`` writes at each document root (the
images themselves are stored next to the markdown file referencing them).
"""
import json
import re
from pathlib import Path
from typing import Any, Dict, Optional, Set
from openviking.server.identity import RequestContext
from openviking.storage.viking_fs import get_viking_fs
from openviking_cli.utils import get_logger
logger = get_logger(__name__)
_IMAGE_PATTERN = re.compile(r"!\[([^\]]*)\]\(((?:[^()]|\([^()]*\))+)\)")
_REMOTE_PREFIXES = ("http://", "https://", "viking://", "data:", "ftp://")
# Sidecar written by MarkdownParser._ingest_local_images at each document root,
# consumed (and deleted) here. Shared so merge/sync code can recognize it.
IMAGE_MAPPINGS_FILENAME = ".image_mappings.json"
# HTML <img src="..."> embeds, common in markdown for sizing control. Shared
# with the parser so ingestion and rewriting see the same references.
HTML_IMG_PATTERN = re.compile(r"""(<img\s[^>]*?src=["'])([^"']+)(["'][^>]*>)""", re.IGNORECASE)
_FENCE_PATTERN = re.compile(r"^(\s{0,3})(`{3,}|~{3,})")
_LIST_ITEM_PATTERN = re.compile(r"^(\s{0,3})([-*+]|\d{1,9}[.)])(\s+)")
def build_artifact_image_mappings(root_dir: Path) -> Dict[str, Dict[str, str]]:
"""Build the sidecar mapping for an already-materialized parser artifact.
Understanding API artifacts already place each image next to the markdown
file that references it. Keep this helper deliberately narrower than
MarkdownParser's local-image ingestion: it records only existing sibling
files and never copies or renames artifact content.
"""
root = root_dir.resolve()
mappings: Dict[str, Dict[str, str]] = {}
for md_path in root.rglob("*.md"):
if not md_path.is_file():
continue
try:
content = md_path.read_text(encoding="utf-8")
except (OSError, UnicodeError):
logger.warning(f"[image_rewrite] Failed to read artifact markdown: {md_path}")
continue
protected = _protected_ranges(content)
refs = [(match.start(), match.group(2)) for match in _IMAGE_PATTERN.finditer(content)]
refs.extend((match.start(), match.group(2)) for match in HTML_IMG_PATTERN.finditer(content))
file_mappings: Dict[str, str] = {}
md_dir = md_path.parent.resolve()
for pos, original_ref in refs:
if any(start <= pos < end for start, end in protected):
continue
ref = original_ref.strip()
if not ref or _is_remote_uri(ref):
continue
# Queries/fragments are not part of the local filesystem path, but
# the exact original reference remains the mapping key used later.
path_part = re.split(r"[?#]", ref, maxsplit=1)[0]
ref_path = Path(path_part)
if not path_part or ref_path.is_absolute():
continue
try:
candidate = (md_dir / ref_path).resolve()
candidate.relative_to(root)
except (OSError, ValueError):
continue
# The existing sidecar contract stores only the final filename and
# rewrite_image_uris resolves it beside the markdown file.
if candidate.parent != md_dir or not candidate.is_file():
continue
file_mappings[original_ref] = candidate.name
if file_mappings:
mappings[md_path.relative_to(root).as_posix()] = file_mappings
return mappings
def _is_remote_uri(path: str) -> bool:
return any(path.startswith(p) for p in _REMOTE_PREFIXES)
def _inline_code_ranges(line: str):
"""Yield (start, end) offsets of inline code spans within a single line.
A code span is a run of N backticks closed by another run of exactly N
backticks. Unterminated runs are not treated as code.
"""
i = 0
n = len(line)
while i < n:
if line[i] != "`":
i += 1
continue
j = i
while j < n and line[j] == "`":
j += 1
run = j - i
k = j
closed = False
while k < n:
if line[k] != "`":
k += 1
continue
m = k
while m < n and line[m] == "`":
m += 1
if m - k == run:
yield (i, m)
i = m
closed = True
break
k = m
if not closed:
i = j
def _protected_ranges(content: str):
"""Compute character ranges that must not be rewritten.
Covers fenced code blocks, indented code blocks and inline code spans so
that Markdown image examples inside code are left untouched.
"""
ranges = []
offset = 0
in_fence = False
fence_char = ""
fence_len = 0
in_indent_code = False
in_list = False
prev_blank = True # start of document behaves like "after a blank line"
for line in content.splitlines(keepends=True):
start = offset
end = offset + len(line)
offset = end
line_content = line.rstrip("\n").rstrip("\r")
stripped = line_content.strip()
is_blank = stripped == ""
if in_fence:
ranges.append((start, end))
m = _FENCE_PATTERN.match(line_content)
if (
m
and m.group(2)[0] == fence_char
and len(m.group(2)) >= fence_len
and stripped == m.group(2)
):
in_fence = False
prev_blank = is_blank
continue
m = _FENCE_PATTERN.match(line_content)
if m:
in_fence = True
in_indent_code = False
fence_char = m.group(2)[0]
fence_len = len(m.group(2))
ranges.append((start, end))
prev_blank = is_blank
continue
indent_width = 0
for ch in line_content:
if ch == " ":
indent_width += 1
elif ch == "\t":
indent_width += 4
else:
break
# Track list scope: a list item opens a list; it stays open across
# blank lines and indented continuation, and closes when a non-blank,
# non-list line returns to the left margin.
if _LIST_ITEM_PATTERN.match(line_content):
in_list = True
elif in_list and not is_blank and indent_width == 0:
in_list = False
if in_indent_code:
if is_blank or indent_width >= 4:
ranges.append((start, end))
prev_blank = is_blank
continue
in_indent_code = False
elif not in_list and not is_blank and indent_width >= 4 and prev_blank:
in_indent_code = True
ranges.append((start, end))
prev_blank = is_blank
continue
for s, e in _inline_code_ranges(line_content):
ranges.append((start + s, start + e))
prev_blank = is_blank
return ranges
async def _discover_mappings(
viking_fs,
root_prefix: str,
md_uris: list,
ctx: Optional[RequestContext] = None,
) -> Dict[str, Dict[str, Dict[str, str]]]:
"""Locate every ``.image_mappings.json`` under *root_prefix*.
``_ingest_local_images`` writes one sidecar per document root, and all of a
document's markdown files live under that root — so probing the ancestor
directories of the markdown files finds every sidecar regardless of how
deep the ingest placed the document (single-file ingest leaves it at the
resource root; directory ingest nests it per document).
Returns ``{mapping_dir: {rel_md_path: {original_path: image_filename}}}``
where ``rel_md_path`` is relative to ``mapping_dir``.
"""
candidates = set()
for md_uri in md_uris:
d = md_uri.rsplit("/", 1)[0]
while d == root_prefix or d.startswith(root_prefix + "/"):
candidates.add(d)
if d == root_prefix:
break
d = d.rsplit("/", 1)[0]
found: Dict[str, Dict[str, Dict[str, str]]] = {}
for d in candidates:
try:
content = await viking_fs.read_file(f"{d}/{IMAGE_MAPPINGS_FILENAME}", ctx=ctx)
found[d] = json.loads(content)
except Exception:
continue
return found
async def rewrite_image_uris(
root_uri: str,
ctx: Optional[RequestContext] = None,
lease_ref: Optional[Dict[str, Any]] = None,
) -> Dict[str, int]:
"""Rewrite local image references in markdown files to viking:// URIs.
After ``persist_temp_tree`` copies content to the final VikingFS location,
this function scans all ``.md`` files under *root_uri* for image references
recorded in the ``.image_mappings.json`` sidecars written by
``_ingest_local_images`` (one per document root, holding
``{rel_md_path -> {original_path -> image_filename}}``), and replaces each
recorded path with the full viking:// URI of the image stored next to the
referencing markdown file. Each sidecar is interpreted in the coordinate
system of the directory holding it, so both single-file ingest (sidecar at
the resource root) and directory ingest (sidecar per document subdirectory)
are covered.
Args:
root_uri: The final VikingFS root URI (e.g. ``viking://resources/doc``)
ctx: Optional request context for permissions
lease_ref: Optional lease reference held by the caller. When the caller
already owns a TREE lock over *root_uri*, forwarding it lets the
cleanup ``rm`` reuse that lock instead of conflicting with it.
Returns:
Dict with ``files_processed`` and ``references_rewritten`` counts.
"""
viking_fs = get_viking_fs()
root_prefix = root_uri.rstrip("/")
# Find all .md files recursively
glob_result = await viking_fs.glob("**/*.md", uri=root_uri, ctx=ctx)
md_uris = glob_result.get("matches", [])
if not md_uris:
return {"files_processed": 0, "references_rewritten": 0}
mappings_by_dir = await _discover_mappings(viking_fs, root_prefix, md_uris, ctx)
files_processed = 0
references_rewritten = 0
for md_uri in md_uris:
# Resolve this markdown file's mapping from the sidecar of the document
# root containing it (the key is relative to that root).
path_to_image_name: Dict[str, str] = {}
for map_dir, file_mappings in mappings_by_dir.items():
if md_uri.startswith(map_dir + "/"):
entry = file_mappings.get(md_uri[len(map_dir) + 1 :])
if entry:
path_to_image_name = entry
break
if not path_to_image_name:
continue
md_dir = md_uri.rsplit("/", 1)[0]
# Build the set of available images that sit beside this markdown file
available_images: Set[str] = set()
try:
entries = await viking_fs.ls(md_dir, ctx=ctx)
available_images = {
e["name"] for e in entries if not e.get("isDir") and not e["name"].startswith(".")
}
except Exception:
logger.debug(f"[image_rewrite] Failed to list directory {md_dir}")
try:
content = await viking_fs.read_file(md_uri, ctx=ctx)
except Exception:
logger.warning(f"[image_rewrite] Failed to read {md_uri}, skipping")
continue
new_content, rewrite_count = _rewrite_content(
content, md_dir, available_images, path_to_image_name
)
if rewrite_count > 0:
try:
await viking_fs.write_file(
md_uri,
new_content,
ctx=ctx,
lease_ref=lease_ref,
)
files_processed += 1
references_rewritten += rewrite_count
logger.debug(f"[image_rewrite] Rewrote {rewrite_count} image ref(s) in {md_uri}")
except Exception:
logger.warning(f"[image_rewrite] Failed to write {md_uri}")
# Clean up mapping sidecars — no longer needed after rewrite
for map_dir in mappings_by_dir:
try:
await viking_fs.rm(
f"{map_dir}/{IMAGE_MAPPINGS_FILENAME}",
ctx=ctx,
lease_ref=lease_ref,
)
except Exception as e:
logger.warning(
f"[image_rewrite] Failed to delete {map_dir}/{IMAGE_MAPPINGS_FILENAME}: {e}"
)
logger.info(
f"[image_rewrite] Processed {len(md_uris)} .md files, "
f"rewrote {references_rewritten} image reference(s) in {files_processed} file(s)"
)
return {"files_processed": files_processed, "references_rewritten": references_rewritten}
def _rewrite_content(
content: str,
image_dir: str,
available_images: Set[str],
path_to_image_name: Optional[Dict[str, str]] = None,
) -> tuple[str, int]:
"""Rewrite local image references in markdown content.
Returns (new_content, rewrite_count).
"""
rewrite_count = 0
mappings = path_to_image_name or {}
protected = _protected_ranges(content)
def _in_protected(pos: int) -> bool:
for s, e in protected:
if s <= pos < e:
return True
return False
def _mapped_uri(path: str) -> Optional[str]:
"""viking:// URI for *path* if the mapping covers it, else None."""
image_name = mappings.get(path)
if image_name and image_name in available_images:
return f"{image_dir}/{image_name}"
logger.warning(
f"[image_rewrite] Image not found in VikingFS: path = {path}, "
f"image_dir = {image_dir}, leaving reference unchanged"
)
return None
def replacer(match: re.Match) -> str:
nonlocal rewrite_count
alt_text = match.group(1)
path = match.group(2)
# Skip image references that live inside code blocks / inline code.
if _in_protected(match.start()):
return match.group(0)
if _is_remote_uri(path):
return match.group(0)
uri = _mapped_uri(path)
if uri is None:
return match.group(0)
rewrite_count += 1
return f""
def img_tag_replacer(match: re.Match) -> str:
nonlocal rewrite_count
path = match.group(2)
if _in_protected(match.start()):
return match.group(0)
if _is_remote_uri(path):
return match.group(0)
uri = _mapped_uri(path)
if uri is None:
return match.group(0)
rewrite_count += 1
return f"{match.group(1)}{uri}{match.group(3)}"
new_content = _IMAGE_PATTERN.sub(replacer, content)
# The markdown pass may have shifted offsets; recompute protected ranges
# against the updated text before rewriting <img> tags.
protected = _protected_ranges(new_content)
new_content = HTML_IMG_PATTERN.sub(img_tag_replacer, new_content)
return new_content, rewrite_count