Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(parse): 目录/重入入库拆分图片并改写为 viking:// URI、Markdown 图片引用 viking 化 / split images and rewrite refs to viking:// URIs on directory & re-ingest (#2557)
Kchen8bd2b45d4cd9
diff --git a/openviking/parse/image_rewrite.py b/openviking/parse/image_rewrite.pyindex 02c24d441..cdc297e3b 100644--- a/openviking/parse/image_rewrite.py+++ b/openviking/parse/image_rewrite.py@@ -3,8 +3,9 @@"""Post-commit image URI rewriting for OpenViking.Scans markdown files in VikingFS after source commit and rewrites local-image references to viking:// URIs based on images stored in the ./images/-directory.+image references to viking:// URIs, driven by the ``.image_mappings.json``+sidecars that ``_ingest_local_images`` writes at each document root (the+images themselves are stored next to the markdown file referencing them)."""import json@@ -22,6 +23,16 @@ logger = get_logger(__name__)_IMAGE_PATTERN = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")_REMOTE_PREFIXES = ("http://", "https://", "viking://", "data:", "ftp://")++# Sidecar written by MarkdownParser._ingest_local_images at each document root,+# consumed (and deleted) here. Shared so merge/sync code can recognize it.+IMAGE_MAPPINGS_FILENAME = ".image_mappings.json"++# HTML <img src="..."> embeds, common in markdown for sizing control. Shared+# with the parser so ingestion and rewriting see the same references.+HTML_IMG_PATTERN = re.compile(+ r"""(<img\s[^>]*?src=["'])([^"']+)(["'][^>]*>)""", re.IGNORECASE+)_FENCE_PATTERN = re.compile(r"^(\s{0,3})(`{3,}|~{3,})")_LIST_ITEM_PATTERN = re.compile(r"^(\s{0,3})([-*+]|\d{1,9}[.)])(\s+)")@@ -144,6 +155,42 @@ def _protected_ranges(content: str):return ranges+async def _discover_mappings(+ viking_fs,+ root_prefix: str,+ md_uris: list,+ ctx: Optional[RequestContext] = None,+) -> Dict[str, Dict[str, Dict[str, str]]]:+ """Locate every ``.image_mappings.json`` under *root_prefix*.++ ``_ingest_local_images`` writes one sidecar per document root, and all of a+ document's markdown files live under that root — so probing the ancestor+ directories of the markdown files finds every sidecar regardless of how+ deep the ingest placed the document (single-file ingest leaves it at the+ resource root; directory ingest nests it per document).++ Returns ``{mapping_dir: {rel_md_path: {original_path: image_filename}}}``+ where ``rel_md_path`` is relative to ``mapping_dir``.+ """+ candidates = set()+ for md_uri in md_uris:+ d = md_uri.rsplit("/", 1)[0]+ while d == root_prefix or d.startswith(root_prefix + "/"):+ candidates.add(d)+ if d == root_prefix:+ break+ d = d.rsplit("/", 1)[0]++ found: Dict[str, Dict[str, Dict[str, str]]] = {}+ for d in candidates:+ try:+ content = await viking_fs.read_file(f"{d}/{IMAGE_MAPPINGS_FILENAME}", ctx=ctx)+ found[d] = json.loads(content)+ except Exception:+ continue+ return found++async def rewrite_image_uris(root_uri: str,ctx: Optional[RequestContext] = None,@@ -153,8 +200,14 @@ async def rewrite_image_uris(After ``persist_temp_tree`` copies content to the final VikingFS location,this function scans all ``.md`` files under *root_uri* for image references- pointing to local paths. For each one, it looks for a corresponding file- in ``{root_uri}/.images/`` and replaces the path with the full viking:// URI.+ recorded in the ``.image_mappings.json`` sidecars written by+ ``_ingest_local_images`` (one per document root, holding+ ``{rel_md_path -> {original_path -> image_filename}}``), and replaces each+ recorded path with the full viking:// URI of the image stored next to the+ referencing markdown file. Each sidecar is interpreted in the coordinate+ system of the directory holding it, so both single-file ingest (sidecar at+ the resource root) and directory ingest (sidecar per document subdirectory)+ are covered.Args:root_uri: The final VikingFS root URI (e.g. ``viking://resources/doc``)@@ -177,24 +230,21 @@ async def rewrite_image_uris(if not md_uris:return {"files_processed": 0, "references_rewritten": 0}- # Load mapping file from _ingest_local_images:- # {rel_md_path -> {original_path_str -> image_filename}}- # The mapping file lives at the root directory and images are stored next to- # their referencing markdown file.- file_mappings: Dict[str, Dict[str, str]] = {}- try:- mapping_content = await viking_fs.read_file(f"{root_prefix}/.image_mappings.json", ctx=ctx)- file_mappings = json.loads(mapping_content)- except Exception:- pass+ mappings_by_dir = await _discover_mappings(viking_fs, root_prefix, md_uris, ctx)files_processed = 0references_rewritten = 0for md_uri in md_uris:- # Resolve this markdown file's mapping and its containing directory- rel_md_path = md_uri[len(root_prefix) + 1 :] if md_uri.startswith(root_prefix) else md_uri- path_to_image_name = file_mappings.get(rel_md_path, {})+ # Resolve this markdown file's mapping from the sidecar of the document+ # root containing it (the key is relative to that root).+ path_to_image_name: Dict[str, str] = {}+ for map_dir, file_mappings in mappings_by_dir.items():+ if md_uri.startswith(map_dir + "/"):+ entry = file_mappings.get(md_uri[len(map_dir) + 1 :])+ if entry:+ path_to_image_name = entry+ breakif not path_to_image_name:continue@@ -230,12 +280,12 @@ async def rewrite_image_uris(except Exception:logger.warning(f"[image_rewrite] Failed to write {md_uri}")- # Clean up mapping file — no longer needed after rewrite- if file_mappings:+ # Clean up mapping sidecars — no longer needed after rewrite+ for map_dir in mappings_by_dir:try:- await viking_fs.rm(f"{root_prefix}/.image_mappings.json", ctx=ctx, lock_handle=lock_handle)+ await viking_fs.rm(f"{map_dir}/{IMAGE_MAPPINGS_FILENAME}", ctx=ctx, lock_handle=lock_handle)except Exception as e:- logger.warning(f"[image_rewrite] Failed to delete .image_mappings.json: {e}")+ logger.warning(f"[image_rewrite] Failed to delete {map_dir}/{IMAGE_MAPPINGS_FILENAME}: {e}")logger.info(f"[image_rewrite] Processed {len(md_uris)} .md files, "@@ -266,6 +316,17 @@ def _rewrite_content(return Truereturn False+ def _mapped_uri(path: str) -> Optional[str]:+ """viking:// URI for *path* if the mapping covers it, else None."""+ image_name = mappings.get(path)+ if image_name and image_name in available_images:+ return f"{image_dir}/{image_name}"+ logger.warning(+ f"[image_rewrite] Image not found in VikingFS: path = {path}, "+ f"image_dir = {image_dir}, leaving reference unchanged"+ )+ return None+def replacer(match: re.Match) -> str:nonlocal rewrite_countalt_text = match.group(1)@@ -278,18 +339,31 @@ def _rewrite_content(if _is_remote_uri(path):return match.group(0)- # Prefer exact path mapping from .image_mappings.json- if path in mappings:- image_name = mappings[path]- if image_name in available_images:- rewrite_count += 1- return f""+ uri = _mapped_uri(path)+ if uri is None:+ return match.group(0)+ rewrite_count += 1+ return f""- logger.warning(- f"[image_rewrite] Image not found in VikingFS: path = {path}, "- f"image_dir = {image_dir}, leaving reference unchanged"- )- return match.group(0)+ def img_tag_replacer(match: re.Match) -> str:+ nonlocal rewrite_count+ path = match.group(2)++ if _in_protected(match.start()):+ return match.group(0)++ if _is_remote_uri(path):+ return match.group(0)++ uri = _mapped_uri(path)+ if uri is None:+ return match.group(0)+ rewrite_count += 1+ return f"{match.group(1)}{uri}{match.group(3)}"new_content = _IMAGE_PATTERN.sub(replacer, content)+ # The markdown pass may have shifted offsets; recompute protected ranges+ # against the updated text before rewriting <img> tags.+ protected = _protected_ranges(new_content)+ new_content = HTML_IMG_PATTERN.sub(img_tag_replacer, new_content)return new_content, rewrite_countdiff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.pyindex 47ebf1af1..fd9447c97 100644--- a/openviking/parse/parsers/directory.py+++ b/openviking/parse/parsers/directory.py@@ -26,6 +26,7 @@ from openviking.parse.base import (ResourceNode,create_parse_result,)+from openviking.parse.image_rewrite import IMAGE_MAPPINGS_FILENAMEfrom openviking.parse.parsers.base_parser import BaseParserfrom openviking.parse.parsers.media.constants import MEDIA_EXTENSIONSfrom openviking.storage.viking_fs import LS_ALL_NODES@@ -37,6 +38,10 @@ if TYPE_CHECKING:logger = get_logger(__name__)+# Hidden files a parser's temp tree is allowed to carry through the merge.+# Everything else hidden stays filtered, like a default ls.+_MERGE_SIDECAR_ALLOWLIST = frozenset({IMAGE_MAPPINGS_FILENAME})+class DirectoryParser(BaseParser):"""@@ -379,6 +384,10 @@ class DirectoryParser(BaseParser):# in flat mode link targets don't exist at their original paths.enable_link_rewrite=preserve_structure,link_rewrite_root=import_root,+ # The whole ingested tree is fair game for image ingestion:+ # an md may reference shared images outside its own directory+ # (e.g. ../images/x.gif) that still live inside the import.+ allowed_media_dirs=[Path(import_root)] if import_root else None,)if sub_result.temp_dir_path:if preserve_structure:@@ -460,13 +469,22 @@ class DirectoryParser(BaseParser):) -> None:"""Move all content from a parser's temp directory into *dest_uri*.- After the move the source temp is deleted.+ After the move the source temp is deleted. Hidden files stay filtered,+ except the sidecars in :data:`_MERGE_SIDECAR_ALLOWLIST` that downstream+ steps depend on (e.g. ``.image_mappings.json`` for the post-commit+ image rewrite)."""- entries = await viking_fs.ls(src_temp_uri, node_limit=LS_ALL_NODES)+ entries = await viking_fs.ls(src_temp_uri, show_all_hidden=True, node_limit=LS_ALL_NODES)for entry in entries:name = entry.get("name", "")if not name or name in (".", ".."):continue+ if (+ not DirectoryParser._is_dir_entry(entry)+ and name.startswith(".")+ and name not in _MERGE_SIDECAR_ALLOWLIST+ ):+ continuesrc = entry.get("uri", f"{src_temp_uri.rstrip('/')}/{name}")dst = f"{dest_uri.rstrip('/')}/{name}"if DirectoryParser._is_dir_entry(entry):@@ -529,13 +547,20 @@ class DirectoryParser(BaseParser):src_uri: str,dst_uri: str,) -> None:- """Recursively move a VikingFS directory tree."""+ """Recursively move a VikingFS directory tree (hidden files filtered,+ allowlisted sidecars carried)."""await viking_fs.mkdir(dst_uri, exist_ok=True)- entries = await viking_fs.ls(src_uri, node_limit=LS_ALL_NODES)+ entries = await viking_fs.ls(src_uri, show_all_hidden=True, node_limit=LS_ALL_NODES)for entry in entries:name = entry.get("name", "")if not name or name in (".", ".."):continue+ if (+ not DirectoryParser._is_dir_entry(entry)+ and name.startswith(".")+ and name not in _MERGE_SIDECAR_ALLOWLIST+ ):+ continues = f"{src_uri.rstrip('/')}/{name}"d = f"{dst_uri.rstrip('/')}/{name}"if DirectoryParser._is_dir_entry(entry):diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.pyindex 0afcd5fc3..c53d6cfad 100644--- a/openviking/parse/parsers/markdown.py+++ b/openviking/parse/parsers/markdown.py@@ -266,12 +266,16 @@ class MarkdownParser(BaseParser):# Set up relative-link rewrite context consumed by _write_section during# apply. Link rewriting is opt-in via kwargs (DirectoryParser sets these);# single-file ingestion never passes link_rewrite_root, so it never rewrites.+ # base_dir/allowed_media_dirs let the rewrite ask _ingest_will_handle_image+ # which image embeds ingestion will take (those are left for #2429).self._rewrite_ctx = {"enabled": bool(kwargs.get("enable_link_rewrite", False)),"source_path": source_path,"doc_name": layout.doc_name,"root_dir": layout.root_dir,"import_root": kwargs.get("link_rewrite_root"),+ "base_dir": base_dir,+ "allowed_media_dirs": allowed_media_dirs,}# Phase 2 — write only: replay the plan against the real VikingFS,@@ -574,9 +578,13 @@ class MarkdownParser(BaseParser):logger.warning(f"[MarkdownParser] Failed to read markdown file: {md_uri}")continue- # Collect all image references in this markdown file- images = list(self._image_pattern.finditer(content))- if not images:+ # Collect all image references in this markdown file: markdown+ # embeds (![...]) and HTML <img src="..."> tags alike.+ from openviking.parse.image_rewrite import HTML_IMG_PATTERN++ image_refs = [m.group(2) for m in self._image_pattern.finditer(content)]+ image_refs += [m.group(2) for m in HTML_IMG_PATTERN.finditer(content)]+ if not image_refs:continue# Resolve local image paths, skipping remote URIs and duplicates@@ -584,8 +592,7 @@ class MarkdownParser(BaseParser):origin_images_links = []seen_paths = set()- for match in images:- path_str = match.group(2)+ for path_str in image_refs:# Skip remote URIsif self._is_remote_uri(path_str):@@ -648,8 +655,11 @@ class MarkdownParser(BaseParser):# Write a single mapping file at the root directory for rewrite_image_urisif mappings:import json++ from openviking.parse.image_rewrite import IMAGE_MAPPINGS_FILENAME+await viking_fs.write_file(- f"{root_prefix}/.image_mappings.json",+ f"{root_prefix}/{IMAGE_MAPPINGS_FILENAME}",json.dumps(mappings, ensure_ascii=False),)@@ -693,6 +703,23 @@ class MarkdownParser(BaseParser):if not allowed_roots:return None+ # Markdown semantics first: the reference is relative to the file's+ # own directory. Accept it when the resolved target stays inside ANY+ # allowed root (e.g. ../images/x.gif escaping base_dir but still+ # inside the import root a DirectoryParser passed down).+ if base_dir:+ candidate = (base_dir / path).resolve()+ if candidate.exists():+ for root in allowed_roots:+ try:+ candidate.relative_to(root.resolve())+ return candidate+ except ValueError:+ continue++ # Derived-media semantics: the reference may instead be relative to+ # one of the media roots themselves (e.g. images extracted by the+ # PDF/DOC parser into a media dir).for root in allowed_roots:candidate = (root / path).resolve()@@ -927,6 +954,31 @@ class MarkdownParser(BaseParser):# D) Directory landing → point at the directory and drop any suffix.return _to_rel(os.path.join(target_parent, landing), keep_suffix=False)+ async def _ingest_will_handle_image(self, link: str) -> bool:+ """Whether ``_ingest_local_images`` will take this image reference: it must+ resolve within base_dir/allowed_media_dirs AND pass image validation — the+ exact conditions ingestion itself applies. Such embeds are left untouched so+ #2429 can copy them next to the section and ``rewrite_image_uris`` can turn+ them into viking:// URIs; everything else falls back to depth adjustment.+ Results are cached per parse_content call."""+ ctx = self._rewrite_ctx or {}+ cache = ctx.setdefault("_image_probe_cache", {})+ if link not in cache:+ handled = False+ resolved = self._resolve_image_path(+ link, ctx.get("base_dir"), ctx.get("allowed_media_dirs")+ )+ if resolved is not None:+ try:+ image_bytes = await asyncio.to_thread(resolved.read_bytes)+ handled = await asyncio.to_thread(+ self._is_valid_image, image_bytes, resolved+ )+ except Exception:+ handled = False+ cache[link] = handled+ return cache[link]+async def _rewrite_relative_links(self,content: str,@@ -936,12 +988,17 @@ class MarkdownParser(BaseParser):section_subpath: str,import_root: str,) -> str:- """Rewrite relative markdown *document* links in `content` (disk-coordinate).-- Image embeds (``![...]``) are left untouched. Image ingestion and image-path- rewriting are owned entirely by ``_ingest_local_images`` (PR #2429); link- rewriting only re-bases links between documents.+ """Rewrite relative markdown links in `content` (disk-coordinate).++ Image embeds (``![...]``) that ingestion will take are left untouched —+ ``_ingest_local_images`` (PR #2429) copies them next to the section and+ ``rewrite_image_uris`` later rewrites them to viking:// URIs. Image embeds+ ingestion will NOT take (outside base_dir, missing, failing validation) get+ the same depth adjustment as document links so they stay valid after the+ document moves into its ingest directory."""+ from openviking.parse.image_rewrite import HTML_IMG_PATTERN+src_dir = os.path.dirname(os.path.abspath(source_path))import_root_abs = os.path.abspath(import_root)source_new_dir = os.path.join(src_dir, doc_name, section_subpath)@@ -951,18 +1008,33 @@ class MarkdownParser(BaseParser):for match in _MD_LINK_RE.finditer(content):out.append(content[last : match.start()])bang, text, link = match.group(1), match.group(2), match.group(3)- # Image embeds are #2429's to ingest and rewrite; never touch them here.- new_link = (- link- if bang- else await self._rewrite_single_link(+ if bang and await self._ingest_will_handle_image(link):+ new_link = link+ else:+ new_link = await self._rewrite_single_link(link, src_dir, import_root_abs, source_new_dir)- )out.append(f"{bang}[{text}]({new_link})")last = match.end()out.append(content[last:])- return "".join(out)+ rewritten = "".join(out)++ # HTML <img src="..."> embeds get the same ownership split as ![...].+ pieces: List[str] = []+ last = 0+ for match in HTML_IMG_PATTERN.finditer(rewritten):+ pieces.append(rewritten[last : match.start()])+ src = match.group(2)+ if await self._ingest_will_handle_image(src):+ new_src = src+ else:+ new_src = await self._rewrite_single_link(+ src, src_dir, import_root_abs, source_new_dir+ )+ pieces.append(f"{match.group(1)}{new_src}{match.group(3)}")+ last = match.end()+ pieces.append(rewritten[last:])+ return "".join(pieces)async def _target_split_files(self, target_path: str) -> Optional[Dict[str, str]]:"""The target's real ingest layout {"<doc_dir>/<section...>": text} via adiff --git a/openviking/parse/parsers/word.py b/openviking/parse/parsers/word.pyindex 0dd4752c1..e2c7ae20e 100644--- a/openviking/parse/parsers/word.py+++ b/openviking/parse/parsers/word.py@@ -58,14 +58,24 @@ class WordParser(BaseParser):result = await self._md_parser.parse_content(markdown_content,source_path=str(path),+ # Forward the original upload name explicitly (mirrors pdf.py) so+ # the resource is named after it, not the temp upload path. Pass+ # the raw kwargs values rather than the path.stem-backed local+ # resource_name, leaving MarkdownParser's naming logic unchanged.+ resource_name=kwargs.get("resource_name"),+ source_name=kwargs.get("source_name"),instruction=instruction,base_dir=path.parent,+ # docx images are extracted into storage.media_dir, so (like pdf)+ # that is the only derived-media root needed.allowed_media_dirs=[storage.media_dir],- **kwargs,)else:result = await self._md_parser.parse_content(- str(source), instruction=instruction, **kwargs+ str(source),+ instruction=instruction,+ resource_name=kwargs.get("resource_name"),+ source_name=kwargs.get("source_name"),)result.source_format = "docx"result.parser_name = "WordParser"diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex a4549663f..73f3f9dbd 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -3,6 +3,7 @@"""SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""import asyncio+import jsonimport threadingfrom contextlib import nullcontextfrom dataclasses import dataclass, field@@ -19,7 +20,10 @@ from openviking.parse.parsers.constants import (FILE_TYPE_DOCUMENTATION,FILE_TYPE_OTHER,)-from openviking.parse.image_rewrite import rewrite_image_uris+from openviking.parse.image_rewrite import (+ IMAGE_MAPPINGS_FILENAME,+ rewrite_image_uris,+)from openviking.parse.parsers.media.utils import (generate_audio_summary,generate_image_summary,@@ -951,34 +955,57 @@ class SemanticProcessor(DequeueHandlerBase):) -> None:"""Rewrite local image refs in the target after a temp-to-target sync.- ``_sync_topdown_recursive`` skips hidden files, so the- ``.image_mappings.json`` sidecar written by the parser at the temp root- is not copied into the target. Carry it over (when missing) so+ ``_sync_topdown_recursive`` MOVES the visible files into the target and+ skips hidden ones, so afterwards the temp tree holds only the+ ``.image_mappings.json`` sidecars written by the parser (one per+ document root, possibly nested for directory ingests). Discovery is+ therefore driven by the markdown files already synced into the TARGET:+ their ancestor directories, mirrored back onto the temp tree, are where+ sidecars can live. Carry each one over (when missing) so:func:`rewrite_image_uris` can resolve local image paths against theimages that were synced into the final target."""viking_fs = get_viking_fs()root_prefix = root_uri.rstrip("/")target_prefix = target_uri.rstrip("/")- mapping_name = ".image_mappings.json"+ mapping_name = IMAGE_MAPPINGS_FILENAMEif root_prefix != target_prefix:try:- await viking_fs.stat(f"{target_prefix}/{mapping_name}", ctx=ctx)- target_has_mapping = True+ glob_result = await viking_fs.glob("*.md", uri=target_prefix, ctx=ctx)+ target_md_uris = glob_result.get("matches", [])except Exception:- target_has_mapping = False-- if not target_has_mapping:+ target_md_uris = []++ # Ancestor dirs of the target md files, as paths relative to the+ # target root — the candidate sidecar locations on both trees.+ candidate_rels = set()+ for md_uri in target_md_uris:+ d = md_uri.rsplit("/", 1)[0]+ while d == target_prefix or d.startswith(target_prefix + "/"):+ candidate_rels.add(d[len(target_prefix) :].lstrip("/"))+ if d == target_prefix:+ break+ d = d.rsplit("/", 1)[0]++ for rel in candidate_rels:+ src_mapping = f"{root_prefix}/{rel}/{mapping_name}" if rel else f"{root_prefix}/{mapping_name}"+ target_mapping = (+ f"{target_prefix}/{rel}/{mapping_name}" if rel else f"{target_prefix}/{mapping_name}"+ )try:- mapping_content = await viking_fs.read_file(- f"{root_prefix}/{mapping_name}", ctx=ctx- )- await viking_fs.write_file(- f"{target_prefix}/{mapping_name}", mapping_content, ctx=ctx- )+ await viking_fs.stat(target_mapping, ctx=ctx)+ continue # already carried over+ except Exception:+ pass+ try:+ mapping_content = await viking_fs.read_file(src_mapping, ctx=ctx)+ except Exception:+ continue # no sidecar at this level+ try:+ await viking_fs.write_file(target_mapping, mapping_content, ctx=ctx)except Exception:- # No sidecar to carry over (e.g. no local images ingested).+ # Target subtree may not exist (doc removed in sync); skip.passtry:diff --git a/tests/parse/test_document_parser_threading.py b/tests/parse/test_document_parser_threading.pyindex 9b4162e73..4a93050d8 100644--- a/tests/parse/test_document_parser_threading.py+++ b/tests/parse/test_document_parser_threading.py@@ -56,7 +56,7 @@ async def test_word_parser_offloads_docx_conversion(monkeypatch, tmp_path: Path)fake_docx = SimpleNamespace()monkeypatch.setitem(sys.modules, "docx", fake_docx)- def convert(path: Path, docx_module) -> str:+ def convert(path: Path, docx_module, resource_name=None, storage=None) -> str:assert docx_module is fake_docxreturn "# converted docx"@@ -66,12 +66,44 @@ async def test_word_parser_offloads_docx_conversion(monkeypatch, tmp_path: Path)result = await parser.parse(source)- assert calls == [(convert, (source, fake_docx), {})]+ # #2429 added resource_name/storage to the offloaded conversion call; assert+ # the conversion was offloaded with the doc + docx module, without pinning the+ # identity of the storage singleton.+ assert len(calls) == 1+ func, args, _ = calls[0]+ assert func is convert+ assert args[0] == source+ assert args[1] is fake_docxassert seen["content"] == "# converted docx"assert result.source_format == "docx"assert result.parser_name == "WordParser"+async def test_word_parser_forwards_original_name_to_markdown(+ monkeypatch, tmp_path: Path+):+ # On single-file upload the on-disk path is a temp name (upload_<uuid>.docx)+ # and the user's original filename arrives via source_name. WordParser must+ # forward that name to MarkdownParser explicitly; otherwise parse_content can+ # only fall back to the temp path and the resource ends up named upload_<uuid>.+ parser = word.WordParser()+ seen = _stub_markdown_parse(parser)+ _patch_to_thread(monkeypatch, word)+ monkeypatch.setitem(sys.modules, "docx", SimpleNamespace())+ monkeypatch.setattr(+ parser, "_convert_to_markdown", lambda *args, **kwargs: "# converted docx"+ )++ upload = tmp_path / "upload_abc123.docx"+ upload.write_bytes(b"placeholder")++ await parser.parse(upload, source_name="季度报告.docx")++ forwarded = seen["kwargs"].get("source_name") or seen["kwargs"].get("resource_name")+ assert forwarded == "季度报告.docx", seen["kwargs"]++@pytest.mark.asyncioasync def test_excel_parser_offloads_xlsx_conversion(monkeypatch, tmp_path: Path):parser = excel.ExcelParser()diff --git a/tests/parse/test_markdown_link_rewrite.py b/tests/parse/test_markdown_link_rewrite.pyindex 11e91c551..f9ca49e61 100644--- a/tests/parse/test_markdown_link_rewrite.py+++ b/tests/parse/test_markdown_link_rewrite.py@@ -51,12 +51,13 @@ class TestRewriteRelativeLinks:)assert out == "[x](../../目录甲/目录乙/目录丙/文档/)"- async def test_image_embed_left_to_ingest(self, tmp_path: Path):- # Image embeds (![...]) are owned entirely by #2429's _ingest_local_images;- # link rewriting must leave them untouched.+ async def test_image_not_ingestable_depth_adjusted(self, tmp_path: Path):+ # img/a.png is a stub (no decodable pixels) so #2429's _ingest_local_images+ # will not take it; the rewrite must then depth-adjust it like any relative+ # link so it stays valid after the doc moves into its directory.kb = self._make_tree(tmp_path)out = await self._rewrite(self._parser(), kb, "")- assert out == ""+ assert out == ""async def test_external_anchor_absolute_unchanged(self, tmp_path: Path):kb = self._make_tree(tmp_path)@@ -145,7 +146,7 @@ class TestRewriteRelativeLinks:)assert out == ("a [1](../目录甲/目录乙/目录丙/文档/) "- "b " # image embed left to #2429, not rewritten+ "b " # stub image: not ingestable -> depth-adjusted)async def test_future_bare_file_layout_points_at_file(self, tmp_path: Path):@@ -205,7 +206,7 @@ class FakeVikingFS:self.files[uri] = datareturn uri- async def write_file(self, uri, content):+ async def write_file(self, uri, content, **kw):if isinstance(content, str):content = content.encode("utf-8")self.files[uri] = content@@ -216,8 +217,10 @@ class FakeVikingFS:async def read(self, uri, offset=0, size=-1):return self.files.get(uri, b"")- async def read_file(self, uri):- data = self.files.get(uri, b"")+ async def read_file(self, uri, **kw):+ if uri not in self.files:+ raise FileNotFoundError(uri)+ data = self.files[uri]return data.decode("utf-8") if isinstance(data, bytes) else dataasync def glob(self, pattern, uri="", **kw):@@ -228,7 +231,15 @@ class FakeVikingFS:matches = [u for u in self.files if u.startswith(prefix) and u.endswith(suffix)]return {"matches": matches}- async def ls(self, uri, node_limit=None):+ async def rm(self, uri, **kw):+ self.files.pop(uri, None)++ async def stat(self, uri, **kw):+ if uri not in self.files:+ raise FileNotFoundError(uri)+ return {"name": uri.rsplit("/", 1)[-1], "isDir": False}++ async def ls(self, uri, node_limit=None, show_all_hidden=False, **kw):prefix = uri.rstrip("/") + "/"children = {}for key in list(self.files.keys()) + self.dirs:@@ -242,6 +253,10 @@ class FakeVikingFS:children[child_name] = is_dirresult = []for name in sorted(children):+ # Mirror VikingFS._ls_original: hidden FILES are filtered unless+ # show_all_hidden is set; directories are always listed.+ if not children[name] and name.startswith(".") and not show_all_hidden:+ continuechild_uri = f"{uri.rstrip('/')}/{name}"result.append({"name": name, "uri": child_uri,@@ -391,3 +406,341 @@ class TestDirectoryEndToEnd:written = [_decode(c) for c in fake.files.values() if "见" in _decode(c)]assert written, fake.filesassert "./sub/target.md" in written[0]+++def _write_valid_png(path: Path, size=(20, 20)) -> None:+ from PIL import Image++ Image.new("RGB", size, color=(120, 30, 30)).save(path)+++class TestImageLinkSplit:+ """Ownership split for image embeds during link rewriting:++ - Images that #2429's _ingest_local_images WILL take (resolvable within+ base_dir/allowed_media_dirs and passing validation) are left untouched —+ ingestion copies them next to the section and rewrite_image_uris later+ turns them into viking:// URIs.+ - Images it will NOT take (outside base_dir, missing, or failing validation)+ get the same depth adjustment as document links so the relative path stays+ valid after the doc moves into its ingest directory.+ """++ async def test_ingestable_image_left_untouched_and_copied(self, tmp_path: Path):+ kb = tmp_path / "kb"+ (kb / "img").mkdir(parents=True)+ _write_valid_png(kb / "img" / "photo.png")+ src = kb / "page.md"+ src.write_text("", encoding="utf-8")++ fake = FakeVikingFS()+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):+ await MarkdownParser().parse(+ str(src), enable_link_rewrite=True, link_rewrite_root=str(kb)+ )++ md = [_decode(c) for u, c in fake.files.items() if u.endswith(".md")]+ assert md and "" in md[0], fake.files+ assert any(u.endswith("/photo.png") for u in fake.files), fake.files+ assert any(u.endswith(".image_mappings.json") for u in fake.files), fake.files++ async def test_image_outside_base_dir_depth_adjusted(self, tmp_path: Path):+ # The md lives in kb/sub; the image lives in kb/img — outside base_dir+ # (= the md's own directory) but inside the import root. #2429 cannot+ # take it, so the link must be depth-adjusted to stay valid.+ kb = tmp_path / "kb"+ (kb / "img").mkdir(parents=True)+ (kb / "sub").mkdir()+ _write_valid_png(kb / "img" / "photo.png")+ src = kb / "sub" / "page.md"+ src.write_text("", encoding="utf-8")++ fake = FakeVikingFS()+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):+ await MarkdownParser().parse(+ str(src), enable_link_rewrite=True, link_rewrite_root=str(kb)+ )++ md = [_decode(c) for u, c in fake.files.items() if u.endswith(".md")]+ assert md and "" in md[0], fake.files+ assert not any(u.endswith("/photo.png") for u in fake.files), fake.files++ async def test_html_img_ingested_and_left_for_rewrite(self, tmp_path: Path):+ # HTML <img src="..."> embeds get the same treatment as ![...] embeds:+ # ingestable ones are copied next to the section and recorded in the+ # mapping sidecar, with the tag itself left untouched.+ kb = tmp_path / "kb"+ (kb / "img").mkdir(parents=True)+ _write_valid_png(kb / "img" / "photo.png")+ src = kb / "page.md"+ src.write_text(+ '<img src="./img/photo.png" width="80%">', encoding="utf-8"+ )++ fake = FakeVikingFS()+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):+ await MarkdownParser().parse(+ str(src), enable_link_rewrite=True, link_rewrite_root=str(kb)+ )++ md = [_decode(c) for u, c in fake.files.items() if u.endswith(".md")]+ assert md and '<img src="./img/photo.png" width="80%">' in md[0], fake.files+ assert any(u.endswith("/photo.png") for u in fake.files), fake.files+ mapping = [+ _decode(c)+ for u, c in fake.files.items()+ if u.endswith(".image_mappings.json")+ ]+ assert mapping and "./img/photo.png" in mapping[0], fake.files++ async def test_html_img_outside_import_root_depth_adjusted(self, tmp_path: Path):+ # An <img> pointing outside base_dir (and no allowed_media_dirs) is not+ # ingestable -> it gets the same depth adjustment as document links,+ # other attributes preserved.+ kb = tmp_path / "kb"+ (kb / "img").mkdir(parents=True)+ (kb / "sub").mkdir()+ _write_valid_png(kb / "img" / "photo.png")+ src = kb / "sub" / "page.md"+ src.write_text('<img src="../img/photo.png" width="80%">', encoding="utf-8")++ fake = FakeVikingFS()+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):+ await MarkdownParser().parse(+ str(src), enable_link_rewrite=True, link_rewrite_root=str(kb)+ )++ md = [_decode(c) for u, c in fake.files.items() if u.endswith(".md")]+ assert md and '<img src="../../img/photo.png" width="80%">' in md[0], fake.files++ async def test_directory_ingest_image_within_import_root_becomes_viking(+ self, tmp_path: Path+ ):+ # Directory ingest passes the import root as allowed_media_dirs, so an+ # image outside the md's own dir but inside the ingested tree IS taken:+ # copied next to the section and rewritten to a viking:// URI.+ from openviking.parse.image_rewrite import rewrite_image_uris++ kb = tmp_path / "kb"+ (kb / "images").mkdir(parents=True)+ (kb / "guides").mkdir()+ _write_valid_png(kb / "images" / "photo.png")+ (kb / "guides" / "page.md").write_text(+ "\n\n"+ '<img src="../images/photo.png" width="80%">',+ encoding="utf-8",+ )++ fake = FakeVikingFS()+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):+ result = await DirectoryParser().parse(str(kb))++ temp = result.temp_dir_path+ entries = await fake.ls(temp)+ doc_dirs = [e for e in entries if e["isDir"]]+ root = f"viking://resources/{doc_dirs[0]['name']}"+ src_prefix = doc_dirs[0]["uri"].rstrip("/") + "/"+ for u in list(fake.files):+ if u.startswith(src_prefix):+ fake.files[f"{root}/{u[len(src_prefix):]}"] = fake.files[u]++ import openviking.parse.image_rewrite as image_rewrite_mod++ with patch.object(image_rewrite_mod, "get_viking_fs", return_value=fake):+ stats = await rewrite_image_uris(root, lock_handle=None)++ assert stats["references_rewritten"] == 2, stats+ page = _decode(fake.files[f"{root}/guides/page/page.md"])+ assert f"" in page, page+ assert f'<img src="{root}/guides/page/photo.png" width="80%">' in page, page++ async def test_invalid_image_depth_adjusted(self, tmp_path: Path):+ # In base_dir but not a decodable image -> ingest skips it -> depth-adjust.+ kb = tmp_path / "kb"+ (kb / "img").mkdir(parents=True)+ (kb / "img" / "bad.png").write_bytes(b"\x89PNG\r\n\x1a\n")+ src = kb / "page.md"+ src.write_text("", encoding="utf-8")++ fake = FakeVikingFS()+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):+ await MarkdownParser().parse(+ str(src), enable_link_rewrite=True, link_rewrite_root=str(kb)+ )++ md = [_decode(c) for u, c in fake.files.items() if u.endswith(".md")]+ assert md and "" in md[0], fake.files+ assert not any(u.endswith(".image_mappings.json") for u in fake.files), fake.files+++class TestRewriteImageUris:+ """rewrite_image_uris must consume every .image_mappings.json in the tree,+ interpreting each one in the coordinate system of the directory holding it+ (the doc root the parser wrote it to), not only at the resource root."""++ def _patched(self, fake):+ import openviking.parse.image_rewrite as image_rewrite_mod++ return patch.object(image_rewrite_mod, "get_viking_fs", return_value=fake)++ async def test_nested_mapping_consumed(self):+ from openviking.parse.image_rewrite import rewrite_image_uris++ root = "viking://resources/res"+ fake = FakeVikingFS()+ fake.files = {+ f"{root}/index/index.md": "".encode(),+ f"{root}/index/logo.png": b"img",+ f"{root}/index/.image_mappings.json": (+ '{"index.md": {"./assets/logo.png": "logo.png"}}'.encode()+ ),+ }++ with self._patched(fake):+ stats = await rewrite_image_uris(root, lock_handle=None)++ assert stats == {"files_processed": 1, "references_rewritten": 1}+ assert _decode(fake.files[f"{root}/index/index.md"]) == (+ f""+ )+ assert f"{root}/index/.image_mappings.json" not in fake.files++ async def test_split_doc_mapping_keys_resolved(self):+ # Keys produced for a split document are paths relative to the doc root+ # holding the mapping (possibly several levels deep).+ from openviking.parse.image_rewrite import rewrite_image_uris++ root = "viking://resources/res"+ fake = FakeVikingFS()+ fake.files = {+ f"{root}/big/标题/章一/部分_1.md": "".encode(),+ f"{root}/big/标题/章一/d.jpg": b"img",+ f"{root}/big/.image_mappings.json": (+ '{"标题/章一/部分_1.md": {"./assets/d.jpg": "d.jpg"}}'.encode()+ ),+ }++ with self._patched(fake):+ stats = await rewrite_image_uris(root, lock_handle=None)++ assert stats == {"files_processed": 1, "references_rewritten": 1}+ assert _decode(fake.files[f"{root}/big/标题/章一/部分_1.md"]) == (+ f""+ )+ assert f"{root}/big/.image_mappings.json" not in fake.files++ async def test_root_mapping_still_works(self):+ # Single-file ingest leaves the mapping at the resource root; that layout+ # must keep working unchanged.+ from openviking.parse.image_rewrite import rewrite_image_uris++ root = "viking://resources/doc"+ fake = FakeVikingFS()+ fake.files = {+ f"{root}/doc.md": "".encode(),+ f"{root}/logo.png": b"img",+ f"{root}/.image_mappings.json": (+ '{"doc.md": {"./assets/logo.png": "logo.png"}}'.encode()+ ),+ }++ with self._patched(fake):+ stats = await rewrite_image_uris(root, lock_handle=None)++ assert stats == {"files_processed": 1, "references_rewritten": 1}+ assert _decode(fake.files[f"{root}/doc.md"]) == f""+ assert f"{root}/.image_mappings.json" not in fake.files++ async def test_merge_temp_carries_sidecar_but_not_other_hidden_files(self):+ # _merge_temp must carry the declared .image_mappings.json sidecar, but+ # keep filtering every other hidden file a parser (or anything else)+ # may have left in its temp tree.+ src = "viking://temp/one"+ dest = "viking://temp/dir/docs"+ fake = FakeVikingFS()+ fake.files = {+ f"{src}/doc/doc.md": b"",+ f"{src}/doc/a.png": b"img",+ f"{src}/doc/.image_mappings.json": b'{"doc.md": {"./a.png": "a.png"}}',+ f"{src}/doc/.stray_hidden": b"must not be merged",+ }++ await DirectoryParser._merge_temp(fake, src, dest)++ assert f"{dest}/doc/.image_mappings.json" in fake.files, fake.files+ assert f"{dest}/doc/doc.md" in fake.files+ assert not any(u.endswith(".stray_hidden") for u in fake.files), fake.files++ async def test_sync_path_carries_nested_mappings_to_target(self):+ # Re-ingest of an existing resource goes through SemanticProcessor's+ # temp->target sync, which MOVES the visible files into the target and+ # skips hidden ones: afterwards the temp tree holds ONLY the+ # .image_mappings.json sidecars (the md files are gone). They must+ # still be carried over so the rewrite happens in the target tree.+ import openviking.storage.queuefs.semantic_processor as sp_mod+ from openviking.storage.queuefs.semantic_processor import SemanticProcessor++ root = "viking://temp/sync_src"+ target = "viking://resources/res"+ fake = FakeVikingFS()+ fake.files = {+ # temp tree after sync: visible files moved away, sidecar left behind+ f"{root}/index/.image_mappings.json": (+ '{"index.md": {"./assets/logo.png": "logo.png"}}'.encode()+ ),+ # target tree after sync: visible files only, no hidden sidecar+ f"{target}/index/index.md": "".encode(),+ f"{target}/index/logo.png": b"img",+ }++ processor = SemanticProcessor.__new__(SemanticProcessor)+ with patch.object(sp_mod, "get_viking_fs", return_value=fake), patch.object(+ __import__("openviking.parse.image_rewrite", fromlist=["x"]),+ "get_viking_fs",+ return_value=fake,+ ):+ await processor._rewrite_target_image_uris(root, target)++ assert _decode(fake.files[f"{target}/index/index.md"]) == (+ f""+ )++ async def test_directory_ingest_images_become_viking_uris(self, tmp_path: Path):+ # End to end: directory ingest -> persist -> rewrite. Every md referencing+ # an ingestable image must end up with a viking:// URI.+ from openviking.parse.image_rewrite import rewrite_image_uris++ kb = tmp_path / "kb"+ (kb / "assets").mkdir(parents=True)+ _write_valid_png(kb / "assets" / "logo.png")+ (kb / "index.md").write_text("", encoding="utf-8")+ (kb / "guide.md").write_text("", encoding="utf-8")++ fake = FakeVikingFS()+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):+ result = await DirectoryParser().parse(str(kb))++ # Simulate finalize_from_temp + persist_temp_tree: the single doc dir+ # under the temp root is mirrored onto the final resource root.+ temp = result.temp_dir_path+ entries = await fake.ls(temp)+ doc_dirs = [e for e in entries if e["isDir"]]+ assert len(doc_dirs) == 1, entries+ root = f"viking://resources/{doc_dirs[0]['name']}"+ src_prefix = doc_dirs[0]["uri"].rstrip("/") + "/"+ for u in list(fake.files):+ if u.startswith(src_prefix):+ fake.files[f"{root}/{u[len(src_prefix):]}"] = fake.files[u]++ with self._patched(fake):+ stats = await rewrite_image_uris(root, lock_handle=None)++ assert stats["files_processed"] == 2, stats+ index_md = _decode(fake.files[f"{root}/index/index.md"])+ guide_md = _decode(fake.files[f"{root}/guide/guide.md"])+ assert f"" in index_md, index_md+ assert f"" in guide_md, guide_md+ assert not any(+ u.endswith(".image_mappings.json") and u.startswith(root)+ for u in fake.files+ ), fake.files