Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

47.3 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0

import re
from unittest.mock import AsyncMock, MagicMock

import pytest

from openviking.core.context import ContextLevel
from openviking.server.identity import RequestContext, Role
from openviking.storage.abstract_overview import (
    freshness_metadata,
    parse_abstract_overview,
    render_abstract_overview,
)
from openviking.storage.errors import LockAcquisitionError
from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor
from openviking.utils.content_hash import content_md5
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.session.user_id import UserIdentifier


class _FakeVikingFS:
    def __init__(self, tree, file_contents):
        self._tree = {self._norm(k): v for k, v in tree.items()}
        self._file_contents = {self._norm(k): v for k, v in file_contents.items()}
        self.writes = []
        self._async_agfs = self

    def _norm(self, path):
        if "://" not in path:
            return path
        scheme, rest = path.split("://", 1)
        rest = re.sub(r"/{2,}", "/", rest)
        return f"{scheme}://{rest}"

    async def ls(self, uri, node_limit=None, ctx=None):
        return self._tree.get(self._norm(uri), [])

    async def stat(self, uri, ctx=None, skip_count=False):
        content = self._file_contents.get(self._norm(uri), "")
        return {"size": len(content)}

    async def read_file(self, path, ctx=None):
        return self._file_contents.get(self._norm(path), "")

    async def read_file_bytes(self, path, ctx=None):
        content = self._file_contents.get(self._norm(path), b"")
        return content if isinstance(content, bytes) else content.encode()

    async def abstract(self, uri, ctx=None):
        return self._file_contents.get(
            self._norm(f"{uri}/.abstract.md"),
            f"# {uri} [Directory abstract is not ready]",
        )

    async def write_file(self, path, content, ctx=None, lease_ref=None):
        norm_path = self._norm(path)
        self._file_contents[norm_path] = content
        self.writes.append((norm_path, content))

    async def pathlock_acquire_exact_batch(self, paths):
        return {"paths": paths}

    async def pathlock_release(self, lease):
        return None

    def _uri_to_path(self, uri, ctx=None):
        return uri.replace("viking://", "/local/acc1/")


class _FakeProcessor:
    def __init__(self, viking_fs, transfer_summaries=None):
        self._fs = viking_fs
        self.transfer_summaries = transfer_summaries or {}
        self.transfer_summary_calls = []
        self.summarized_files = []
        self.sync_calls = []
        self.vectorized_files = []
        self.file_ingest_options = {}
        self.file_md5s = {}
        self.file_contents = {}
        self.directory_ingest_options = {}
        self.vectorized_dirs = []
        self.generated_overviews = []
        self.updated_file_vectors = []

    def _parse_overview_md(self, overview_content):
        results = {}
        for line in overview_content.splitlines():
            m = re.match(r"^-\s*(?P<name>[^:]+):\s*(?P<summary>.*)$", line.strip())
            if not m:
                continue
            results[m.group("name").strip()] = m.group("summary").strip()
        return results

    async def _generate_single_file_summary(
        self, file_path, llm_sem=None, ctx=None, file_content=None
    ):
        self.summarized_files.append(file_path)
        self.file_contents[("summary", file_path)] = file_content
        return {"name": file_path.split("/")[-1], "summary": "summary"}

    async def _generate_overview(self, dir_uri, file_summaries, children_abstracts, **kwargs):
        self.generated_overviews.append(dir_uri)
        lines = ["FILES:"]
        for item in file_summaries:
            name = item.get("name", "")
            summary = item.get("summary", "")
            lines.append(f"- {name}: {summary}")
        for item in children_abstracts:
            name = item.get("name", "")
            abstract = item.get("abstract", "")
            lines.append(f"- {name}/: {abstract}")
        return "\n".join(lines)

    async def _load_transfer_file_summaries(self, file_paths, ctx=None):
        self.transfer_summary_calls.append(list(file_paths))
        return {
            path: self.transfer_summaries[path]
            for path in file_paths
            if self.transfer_summaries.get(path)
        }

    def _normalize_overview_generation(self, overview):
        return overview, "abstract"

    async def _vectorize_single_file(
        self,
        parent_uri,
        context_type,
        file_path,
        summary_dict,
        ctx=None,
        use_summary=False,
        ingest_options=None,
        file_md5=None,
        file_content=None,
        materialize_content=False,
        scalar_override=None,
        action="merge",
        field_patch=None,
    ):
        self.vectorized_files.append(file_path)
        self.file_ingest_options[file_path] = ingest_options
        self.file_md5s[file_path] = file_md5
        self.file_contents[("vector", file_path)] = file_content
        self.file_contents[("materialize_content", file_path)] = materialize_content
        self.file_contents[("action", file_path)] = action
        self.file_contents[("scalar_override", file_path)] = scalar_override
        self.file_contents[("field_patch", file_path)] = field_patch
        return True

    async def _vectorize_directory(
        self,
        uri,
        context_type,
        abstract,
        overview,
        ctx=None,
        ingest_options=None,
        skill_source_path="",
        scalar_overrides=None,
        actions=None,
        field_patches=None,
        include_abstract=True,
        include_overview=True,
        md5s=None,
    ):
        self.directory_ingest_options[uri] = ingest_options
        self.vectorized_dirs.append(uri)
        self.file_contents[("actions", uri)] = actions
        self.file_contents[("scalar_overrides", uri)] = scalar_overrides
        self.file_contents[("field_patches", uri)] = field_patches
        self.file_contents[("md5s", uri)] = md5s
        return {
            level for level, included in ((0, include_abstract), (1, include_overview)) if included
        }

    async def _update_file_vector_fields(
        self,
        *,
        record_id,
        file_path,
        file_md5,
        file_content,
        ctx,
        scalar_fields=None,
    ):
        self.updated_file_vectors.append(
            {
                "record_id": record_id,
                "file_path": file_path,
                "file_md5": file_md5,
                "file_content": file_content,
                "ctx": ctx,
                "scalar_fields": scalar_fields,
            }
        )
        return True

    async def _sync_topdown_recursive(
        self, root_uri, target_uri, ctx=None, file_change_status=None, lock=None
    ):
        self.sync_calls.append((root_uri, target_uri))
        root_uri = self._fs._norm(root_uri)
        target_uri = self._fs._norm(target_uri)
        for path, content in list(self._fs._file_contents.items()):
            if path.startswith(root_uri + "/"):
                mapped = target_uri + path[len(root_uri) :]
                self._fs._file_contents[mapped] = content
        return MagicMock(
            added_files=[],
            deleted_files=[],
            updated_files=[],
            added_dirs=[],
            deleted_dirs=[],
        )


@pytest.mark.asyncio
async def test_file_root_semantic_plan_runs_file_node_without_listing_or_sidecars(monkeypatch):
    from openviking.storage.context_update_plan import (
        IndexSlot,
        SemanticPlan,
        SemanticTreeEntry,
        SemanticTreeSnapshot,
    )

    file_uri = "viking://resources/report.md"
    fake_fs = _FakeVikingFS(tree={}, file_contents={file_uri: b"report body"})
    fake_fs.ls = AsyncMock(side_effect=AssertionError("semantic plan must not list file root"))
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    plan = SemanticPlan(
        file_uri,
        "resource",
        SemanticTreeSnapshot(
            (
                SemanticTreeEntry(
                    "",
                    "file",
                    "modified",
                    "generate",
                    md5="fresh-md5",
                    index_slots=(IndexSlot(2, "report-l2", action="upsert"),),
                ),
            )
        ),
    )
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=1,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        semantic_plan=plan,
    )

    await executor.run(file_uri)

    assert processor.summarized_files == [file_uri]
    assert processor.vectorized_files == [file_uri]
    assert fake_fs.writes == []
    assert executor.get_stats().indexed_records == 1


@pytest.mark.asyncio
async def test_semantic_plan_repair_regenerates_and_reembeds_directory(monkeypatch):
    from openviking.storage.context_update_plan import (
        IndexSlot,
        SemanticPlan,
        SemanticTreeEntry,
        SemanticTreeSnapshot,
    )
    from openviking.utils.content_hash import content_md5

    root_uri = "viking://resources/root"
    fake_fs = _FakeVikingFS(tree={root_uri: []}, file_contents={})
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    plan = SemanticPlan(
        root_uri,
        "resource",
        SemanticTreeSnapshot(
            (
                SemanticTreeEntry(
                    "",
                    "directory",
                    "unchanged",
                    "aggregate",
                    index_slots=(
                        IndexSlot(0, "root-l0", {"abstract": "stale abstract"}, action="upsert"),
                        IndexSlot(1, "root-l1", {"abstract": "stale overview"}, action="upsert"),
                    ),
                    repair=True,
                ),
            )
        ),
    )
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=1,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        semantic_plan=plan,
        generation_trigger="reindex",
        source_contents={
            (root_uri, 0): "current abstract",
            (root_uri, 1): "current overview",
        },
    )

    await executor.run(root_uri)

    assert processor.generated_overviews == [root_uri]
    assert processor.vectorized_dirs == [root_uri]
    assert processor.file_contents[("md5s", root_uri)] == {
        0: content_md5(b"abstract"),
        1: content_md5(b"FILES:"),
    }


@pytest.mark.asyncio
async def test_skill_plan_repair_regenerates_and_reembeds_skill_root(monkeypatch):
    from openviking.storage.context_update_plan import (
        IndexSlot,
        SemanticPlan,
        SemanticTreeEntry,
        SemanticTreeSnapshot,
    )

    root_uri = "viking://user/alice/skills/demo"
    fake_fs = _FakeVikingFS(tree={root_uri: []}, file_contents={})
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    processor._skill_root_semantics = AsyncMock(return_value=("skill overview", "skill abstract"))
    plan = SemanticPlan(
        root_uri,
        "skill",
        SemanticTreeSnapshot(
            (
                SemanticTreeEntry(
                    "",
                    "directory",
                    "unchanged",
                    "aggregate",
                    index_slots=(
                        IndexSlot(0, "skill-l0", {"abstract": "old"}, action="upsert"),
                        IndexSlot(1, "skill-l1", {"abstract": "old"}, action="upsert"),
                    ),
                    repair=True,
                ),
            )
        ),
    )
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="skill",
        max_concurrent_llm=1,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        semantic_plan=plan,
        generation_trigger="content_write",
        source_contents={(f"{root_uri}/SKILL.md", 2): b"name: demo"},
    )

    await executor.run(root_uri)

    processor._skill_root_semantics.assert_awaited_once()
    assert processor._skill_root_semantics.await_args.kwargs["regenerate"] is True
    assert processor.vectorized_dirs == [root_uri]


@pytest.mark.asyncio
@pytest.mark.parametrize("sidecar_state", ["valid", "malformed_overview", "malformed_abstract"])
async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatch, sidecar_state):
    root_uri = "viking://resources/root"
    tree = {
        root_uri: [
            {"name": "a.txt", "isDir": False},
            {"name": "b.txt", "isDir": False},
        ],
    }

    fake_fs = _FakeVikingFS(
        tree=tree,
        file_contents={
            f"{root_uri}/a.txt": "new content",
            f"{root_uri}/b.txt": "unchanged",
            f"{root_uri}/.overview.md": render_abstract_overview(
                ContextLevel.OVERVIEW,
                root_uri,
                "FILES:\n- a.txt: old-a\n- b.txt: old-b",
                {
                    "generated_by": {
                        "component": "SemanticProcessor",
                        "trigger": "previous_refresh",
                    }
                },
            ),
            f"{root_uri}/.abstract.md": "old-abstract",
        },
    )
    if sidecar_state != "valid":
        filename = ".overview.md" if sidecar_state == "malformed_overview" else ".abstract.md"
        fake_fs._file_contents[f"{root_uri}/{filename}"] = "---\n"
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=ctx,
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [f"{root_uri}/a.txt"]} if sidecar_state == "valid" else {},
        file_md5s={f"{root_uri}/a.txt": "md5-a-new"},
    )

    await executor.run(root_uri)

    expected_files = {
        "valid": [f"{root_uri}/a.txt"],
        "malformed_overview": [f"{root_uri}/a.txt", f"{root_uri}/b.txt"],
        "malformed_abstract": [],
    }[sidecar_state]
    assert processor.summarized_files == expected_files
    assert processor.vectorized_files == expected_files
    assert processor.vectorized_dirs == [root_uri]
    assert processor.sync_calls == []
    # Resource ingestion uses the manifest md5 whenever the file is re-vectorized;
    # a skipped file produces no vector write.
    if f"{root_uri}/a.txt" in expected_files:
        assert processor.file_md5s[f"{root_uri}/a.txt"] == "md5-a-new"
        assert processor.file_contents[("action", f"{root_uri}/a.txt")] == "merge"
    else:
        assert f"{root_uri}/a.txt" not in processor.file_md5s
    overview = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"]).body
    expected_a = "old-a" if sidecar_state == "malformed_abstract" else "summary"
    expected_b = "summary" if sidecar_state == "malformed_overview" else "old-b"
    assert f"- a.txt: {expected_a}" in overview
    assert f"- b.txt: {expected_b}" in overview
    assert parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.abstract.md"]).body.strip()


@pytest.mark.asyncio
async def test_direct_incremental_created_file_forces_vector_upsert(monkeypatch):
    root_uri = "viking://resources/root"
    file_uri = f"{root_uri}/new.txt"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "new.txt", "isDir": False}]},
        file_contents={file_uri: "new body"},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=1,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"added": [file_uri]},
        file_md5s={file_uri: "new-md5"},
        file_vector_actions={file_uri: "upsert"},
    )

    await executor.run(root_uri)

    assert processor.file_contents[("action", file_uri)] == "upsert"


@pytest.mark.asyncio
async def test_modified_file_with_same_abstract_still_propagates(monkeypatch):
    root_uri = "viking://resources/root"
    child_uri = f"{root_uri}/nested"
    file_uri = f"{child_uri}/a.txt"
    old_root_overview = render_abstract_overview(
        ContextLevel.OVERVIEW, root_uri, "FILES:\n- nested/: old-root-child"
    )
    old_child_overview = render_abstract_overview(
        ContextLevel.OVERVIEW, child_uri, "FILES:\n- a.txt: summary"
    )
    fake_fs = _FakeVikingFS(
        tree={
            root_uri: [{"name": "nested", "isDir": True}],
            child_uri: [{"name": "a.txt", "isDir": False}],
        },
        file_contents={
            file_uri: "changed body",
            f"{root_uri}/.overview.md": old_root_overview,
            f"{root_uri}/.abstract.md": render_abstract_overview(
                ContextLevel.ABSTRACT, root_uri, "old-root-child"
            ),
            f"{child_uri}/.overview.md": old_child_overview,
            f"{child_uri}/.abstract.md": render_abstract_overview(
                ContextLevel.ABSTRACT, child_uri, "old-child"
            ),
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [file_uri]},
        file_abstracts={file_uri: "summary"},
        file_md5s={file_uri: "new-md5"},
    )

    await executor.run(root_uri)

    assert processor.summarized_files == [file_uri]
    assert processor.vectorized_files == [file_uri]
    assert processor.file_md5s[file_uri] == "new-md5"
    assert processor.generated_overviews == [child_uri, root_uri]
    assert processor.vectorized_dirs == [child_uri, root_uri]
    assert fake_fs._file_contents[f"{child_uri}/.overview.md"] != old_child_overview
    assert fake_fs._file_contents[f"{root_uri}/.overview.md"] != old_root_overview


@pytest.mark.asyncio
async def test_modified_file_with_different_abstract_still_propagates(monkeypatch):
    root_uri = "viking://resources/root"
    file_uri = f"{root_uri}/a.txt"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "a.txt", "isDir": False}]},
        file_contents={
            file_uri: "changed body",
            f"{root_uri}/.overview.md": render_abstract_overview(
                ContextLevel.OVERVIEW, root_uri, "FILES:\n- a.txt: old-summary"
            ),
            f"{root_uri}/.abstract.md": render_abstract_overview(
                ContextLevel.ABSTRACT, root_uri, "old-abstract"
            ),
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [file_uri]},
        file_abstracts={file_uri: "old-summary"},
    )

    await executor.run(root_uri)

    assert processor.generated_overviews == [root_uri]
    assert processor.vectorized_dirs == [root_uri]


@pytest.mark.asyncio
async def test_added_file_with_matching_stale_abstract_still_propagates(monkeypatch):
    root_uri = "viking://resources/root"
    child_uri = f"{root_uri}/nested"
    file_uri = f"{child_uri}/a.txt"
    fake_fs = _FakeVikingFS(
        tree={
            root_uri: [{"name": "nested", "isDir": True}],
            child_uri: [{"name": "a.txt", "isDir": False}],
        },
        file_contents={file_uri: "new body"},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"added": [file_uri]},
        file_abstracts={file_uri: "summary"},
    )

    await executor.run(root_uri)

    assert processor.generated_overviews == [child_uri, root_uri]
    assert processor.vectorized_dirs == [child_uri, root_uri]


@pytest.mark.asyncio
async def test_deleted_last_nested_file_still_refreshes_existing_ancestor(monkeypatch):
    root_uri = "viking://resources/root"
    deleted_uri = f"{root_uri}/removed/a.txt"
    fake_fs = _FakeVikingFS(
        tree={root_uri: []},
        file_contents={
            f"{root_uri}/.overview.md": render_abstract_overview(
                ContextLevel.OVERVIEW, root_uri, "FILES:\n- removed/: old-child"
            ),
            f"{root_uri}/.abstract.md": render_abstract_overview(
                ContextLevel.ABSTRACT, root_uri, "old-root"
            ),
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"deleted": [deleted_uri]},
    )

    await executor.run(root_uri)

    assert processor.generated_overviews == [root_uri]
    assert processor.vectorized_dirs == [root_uri]


@pytest.mark.asyncio
async def test_artifact_files_drive_tree_structure_when_target_listing_is_empty(
    tmp_path, monkeypatch
):
    root_uri = "viking://resources/root"
    # artifact_files supplies the committed tree structure so the executor does not
    # depend on a target ls; file bytes are always read from the (already
    # applied) target via viking_fs.
    fake_fs = _FakeVikingFS(
        tree={},
        file_contents={
            f"{root_uri}/a.txt": b"alpha",
            f"{root_uri}/src/b.txt": b"beta",
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=ctx,
        artifact_files=["a.txt", "src/b.txt"],
    )

    await executor.run(root_uri)

    assert sorted(processor.summarized_files) == [
        f"{root_uri}/a.txt",
        f"{root_uri}/src/b.txt",
    ]
    assert sorted(processor.vectorized_files) == [
        f"{root_uri}/a.txt",
        f"{root_uri}/src/b.txt",
    ]
    assert processor.file_contents[("summary", f"{root_uri}/a.txt")] == b"alpha"
    assert processor.file_contents[("vector", f"{root_uri}/src/b.txt")] == b"beta"


@pytest.mark.asyncio
async def test_direct_incremental_reuses_vector_abstract_without_overview(monkeypatch):
    root_uri = "viking://resources/root"
    file_uri = f"{root_uri}/a.txt"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "a.txt", "isDir": False}]},
        file_contents={},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={},
        artifact_files=["a.txt"],
        file_abstracts={file_uri: "summary from vector"},
    )

    await executor.run(root_uri)

    assert processor.summarized_files == []
    assert processor.vectorized_files == []


@pytest.mark.asyncio
async def test_missing_target_file_fails_instead_of_using_empty_summary(tmp_path, monkeypatch):
    root_uri = "viking://resources/root"

    class _MissingFileVikingFS(_FakeVikingFS):
        async def read_file_bytes(self, path, ctx=None):
            raise FileNotFoundError(path)

    fake_fs = _MissingFileVikingFS(tree={}, file_contents={})
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    executor = SemanticTreeExecutor(
        processor=_FakeProcessor(fake_fs),
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        artifact_files=["a.txt"],
    )

    with pytest.raises(FileNotFoundError):
        await executor.run(root_uri)


@pytest.mark.asyncio
async def test_content_write_tags_apply_only_to_changed_file(monkeypatch):
    root_uri = "viking://resources/root"
    changed_uri = f"{root_uri}/a.txt"
    fake_fs = _FakeVikingFS(
        tree={
            root_uri: [
                {"name": "a.txt", "isDir": False},
                {"name": "b.txt", "isDir": False},
            ],
        },
        file_contents={
            changed_uri: "new content",
            f"{root_uri}/b.txt": "unchanged",
            f"{root_uri}/.overview.md": render_abstract_overview(
                ContextLevel.OVERVIEW,
                root_uri,
                "FILES:\n- a.txt: old-a\n- b.txt: old-b",
                {},
            ),
            f"{root_uri}/.abstract.md": "old-abstract",
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
    tag_options = IngestOptions(search_tags=["team=search"])
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=ctx,
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [changed_uri]},
        ingest_options=tag_options,
        generation_trigger="content_write",
    )

    await executor.run(root_uri)

    assert processor.file_ingest_options == {changed_uri: tag_options}
    assert processor.directory_ingest_options[root_uri] == IngestOptions()


@pytest.mark.asyncio
async def test_content_write_uses_per_file_ingest_options(monkeypatch):
    root_uri = "viking://resources/root"
    first_uri = f"{root_uri}/a.txt"
    second_uri = f"{root_uri}/b.txt"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "a.txt", "isDir": False}, {"name": "b.txt", "isDir": False}]},
        file_contents={first_uri: "new a", second_uri: "new b"},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    first_options = IngestOptions(search_tags=["team=search"])
    second_options = IngestOptions(search_tags=[], search_tag_mode="clear")
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [first_uri, second_uri]},
        generation_trigger="content_write",
        file_ingest_options={first_uri: first_options, second_uri: second_options},
    )

    await executor.run(root_uri)

    assert processor.file_ingest_options == {
        first_uri: first_options,
        second_uri: second_options,
    }


@pytest.mark.asyncio
async def test_content_write_with_same_abstract_updates_file_and_parent(monkeypatch):
    root_uri = "viking://resources/root"
    changed_uri = f"{root_uri}/a.txt"
    old_overview = render_abstract_overview(
        ContextLevel.OVERVIEW, root_uri, "FILES:\n- a.txt: summary"
    )
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "a.txt", "isDir": False}]},
        file_contents={
            changed_uri: "new content",
            f"{root_uri}/.overview.md": old_overview,
            f"{root_uri}/.abstract.md": render_abstract_overview(
                ContextLevel.ABSTRACT, root_uri, "old-abstract"
            ),
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    tag_options = IngestOptions(search_tags=["team=search"])
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [changed_uri]},
        file_abstracts={changed_uri: "summary"},
        ingest_options=tag_options,
        generation_trigger="content_write",
    )

    await executor.run(root_uri)

    assert processor.file_ingest_options == {changed_uri: tag_options}
    assert processor.vectorized_files == [changed_uri]
    assert processor.generated_overviews == [root_uri]
    assert processor.vectorized_dirs == [root_uri]
    assert fake_fs._file_contents[f"{root_uri}/.overview.md"] != old_overview


@pytest.mark.asyncio
async def test_content_write_keeps_matching_upstream_md5(monkeypatch):
    root_uri = "viking://resources/root"
    changed_uri = f"{root_uri}/a.txt"
    latest_content = "newer content"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "a.txt", "isDir": False}]},
        file_contents={changed_uri: latest_content},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [changed_uri]},
        file_md5s={changed_uri: content_md5(latest_content.encode())},
        generation_trigger="content_write",
    )

    await executor.run(root_uri)

    assert processor.file_contents[("vector", changed_uri)] == latest_content.encode()
    assert processor.file_md5s[changed_uri] == content_md5(latest_content.encode())


@pytest.mark.asyncio
async def test_semantic_generate_hashes_read_content_when_manifest_md5_is_missing(monkeypatch):
    root_uri = "viking://resources/root"
    changed_uri = f"{root_uri}/a.txt"
    content = b"content read by semantic worker"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "a.txt", "isDir": False}]},
        file_contents={changed_uri: content},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        changes={"modified": [changed_uri]},
        generation_trigger="resource_ingest",
    )

    await executor.run(root_uri)

    assert processor.file_contents[("vector", changed_uri)] == content
    assert processor.file_md5s[changed_uri] == content_md5(content)


@pytest.mark.asyncio
async def test_pending_refresh_rebuilds_every_file_summary(monkeypatch):
    root_uri = "viking://resources/wide"
    file_names = [f"file-{idx:03}.txt" for idx in range(40)]
    file_paths = [f"{root_uri}/{name}" for name in file_names]
    changed_path = f"{root_uri}/file-020.txt"
    old_overview = "FILES:\n" + "\n".join(f"- {name}: old-summary" for name in file_names)
    metadata = {"freshness": freshness_metadata(40, 4, pending=4)}
    fake_fs = _FakeVikingFS(
        tree={
            root_uri: [{"name": name, "isDir": False} for name in file_names],
        },
        file_contents={
            **dict.fromkeys(file_paths, "content"),
            f"{root_uri}/.overview.md": render_abstract_overview(
                ContextLevel.OVERVIEW, root_uri, old_overview, metadata
            ),
            f"{root_uri}/.abstract.md": render_abstract_overview(
                ContextLevel.ABSTRACT, root_uri, "old abstract", metadata
            ),
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=ctx,
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"modified": [changed_path]},
        file_abstracts={changed_path: "summary"},
    )

    await executor.run(root_uri)

    assert set(processor.summarized_files) == set(file_paths)
    assert processor.vectorized_files == [changed_path]
    overview = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"])
    assert overview.metadata["freshness"]["pending_child_changes"] == 0


@pytest.mark.asyncio
async def test_directory_vectorization_retries_after_matching_sidecar_write(monkeypatch):
    root_uri = "viking://resources/root"
    file_path = f"{root_uri}/a.txt"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "a.txt", "isDir": False}]},
        file_contents={file_path: "new content"},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )

    processor = _FakeProcessor(fake_fs)
    vectorize_directory = AsyncMock(side_effect=[RuntimeError("temporary failure"), None])
    processor._vectorize_directory = vectorize_directory
    ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)

    def make_executor():
        return SemanticTreeExecutor(
            processor=processor,
            context_type="resource",
            max_concurrent_llm=2,
            ctx=ctx,
            incremental_update=True,
            target_uri=root_uri,
            recursive=False,
            changes={"modified": [file_path]},
        )

    acquire = fake_fs.pathlock_acquire_exact_batch
    fake_fs.pathlock_acquire_exact_batch = AsyncMock(
        side_effect=[{"paths": []}, LockAcquisitionError("parent sidecars are busy")]
    )
    await make_executor().run(root_uri)
    assert fake_fs.writes == []
    assert processor.vectorized_files == [file_path]
    vectorize_directory.assert_not_awaited()
    fake_fs.pathlock_acquire_exact_batch = acquire

    with pytest.raises(RuntimeError, match="temporary failure"):
        await make_executor().run(root_uri)
    await make_executor().run(root_uri)

    assert vectorize_directory.await_count == 2


@pytest.mark.asyncio
async def test_content_copy_rebuilds_target_overview_from_target_l2_summaries(monkeypatch):
    root_uri = "viking://resources/archive"
    copied_uri = f"{root_uri}/copied.jpg"
    fake_fs = _FakeVikingFS(
        tree={
            root_uri: [
                {"name": "copied.jpg", "isDir": False},
                {"name": "keep.txt", "isDir": False},
            ]
        },
        file_contents={
            copied_uri: "binary-placeholder",
            f"{root_uri}/keep.txt": "keep",
            f"{root_uri}/.overview.md": render_abstract_overview(
                ContextLevel.OVERVIEW,
                root_uri,
                "FILES:\n- keep.txt: existing summary",
            ),
            f"{root_uri}/.abstract.md": render_abstract_overview(
                ContextLevel.ABSTRACT,
                root_uri,
                "existing abstract",
            ),
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(
        fake_fs,
        transfer_summaries={
            copied_uri: "copied target L2 summary",
            f"{root_uri}/keep.txt": "kept target L2 summary",
        },
    )
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"added": [copied_uri]},
        skip_vectorization=False,
        generation_trigger="content_copy",
        copy_source_uri="viking://resources/source/original.jpg",
    )

    await executor.run(root_uri)

    assert processor.summarized_files == []
    assert processor.vectorized_files == []
    assert processor.generated_overviews == [root_uri]
    assert processor.vectorized_dirs == [root_uri]
    overview = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"]).body
    abstract = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.abstract.md"]).body
    assert "- copied.jpg: copied target L2 summary" in overview
    assert "- keep.txt: kept target L2 summary" in overview
    assert abstract.strip() == "abstract"


@pytest.mark.asyncio
async def test_transfer_rebuild_generates_file_summary_and_vector(monkeypatch):
    root_uri = "viking://resources/archive"
    copied_uri = f"{root_uri}/copied.md"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "copied.md", "isDir": False}]},
        file_contents={copied_uri: "copied body"},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"added": [copied_uri]},
        generation_trigger="content_transfer_rebuild",
    )

    await executor.run(root_uri)

    assert processor.summarized_files == [copied_uri]
    assert processor.vectorized_files == [copied_uri]
    assert processor.transfer_summary_calls == []


@pytest.mark.asyncio
async def test_recursive_transfer_rebuild_generates_subtree_vectors(monkeypatch):
    root_uri = "viking://resources/archive/copied"
    child_uri = f"{root_uri}/nested"
    root_file = f"{root_uri}/root.md"
    child_file = f"{child_uri}/child.md"
    fake_fs = _FakeVikingFS(
        tree={
            root_uri: [
                {"name": "nested", "isDir": True},
                {"name": "root.md", "isDir": False},
            ],
            child_uri: [{"name": "child.md", "isDir": False}],
        },
        file_contents={
            root_file: "root body",
            child_file: "child body",
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        recursive=True,
        generation_trigger="content_transfer_rebuild",
    )

    await executor.run(root_uri)

    assert set(processor.summarized_files) == {root_file, child_file}
    assert set(processor.vectorized_files) == {root_file, child_file}
    assert set(processor.vectorized_dirs) == {root_uri, child_uri}
    assert processor.transfer_summary_calls == []


@pytest.mark.asyncio
async def test_content_copy_loads_all_existing_summaries(monkeypatch):
    root_uri = "viking://resources/archive"
    file_paths = [f"{root_uri}/{name}.txt" for name in "abcde"]
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": path.rsplit("/", 1)[-1], "isDir": False} for path in file_paths]},
        file_contents={path: path for path in file_paths},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(
        fake_fs,
        transfer_summaries={path: f"summary-{path.rsplit('/', 1)[-1]}" for path in file_paths},
    )
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"added": [file_paths[2]]},
        generation_trigger="content_copy",
    )

    await executor.run(root_uri)

    assert processor.transfer_summary_calls == [file_paths]
    assert processor.summarized_files == []
    overview = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"]).body
    assert "a.txt" in overview
    assert "e.txt" in overview
    assert "c.txt" in overview


@pytest.mark.asyncio
async def test_content_copy_reports_missing_summary_among_all_children(monkeypatch):
    root_uri = "viking://resources/archive"
    file_paths = [f"{root_uri}/{name}.txt" for name in "abcde"]
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": path.rsplit("/", 1)[-1], "isDir": False} for path in file_paths]},
        file_contents={path: path for path in file_paths},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(
        fake_fs,
        transfer_summaries={path: f"summary-{path.rsplit('/', 1)[-1]}" for path in file_paths[1:]},
    )
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"added": [file_paths[2]]},
        generation_trigger="content_copy",
    )

    await executor.run(root_uri)

    assert processor.transfer_summary_calls == [file_paths]
    overview_doc = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"])
    assert "a.txt" not in overview_doc.body
    assert "b.txt" in overview_doc.body
    assert "e.txt" in overview_doc.body
    assert overview_doc.metadata["freshness"].get("missing_summary_entries") == 1


@pytest.mark.asyncio
async def test_content_copy_with_no_ready_summaries_preserves_existing_sidecars(monkeypatch):
    root_uri = "viking://resources/archive"
    file_path = f"{root_uri}/pending.txt"
    old_overview = render_abstract_overview(ContextLevel.OVERVIEW, root_uri, "old overview")
    old_abstract = render_abstract_overview(ContextLevel.ABSTRACT, root_uri, "old abstract")
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "pending.txt", "isDir": False}]},
        file_contents={
            file_path: "pending",
            f"{root_uri}/.overview.md": old_overview,
            f"{root_uri}/.abstract.md": old_abstract,
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"added": [file_path]},
        generation_trigger="content_copy",
    )

    await executor.run(root_uri)

    assert processor.generated_overviews == []
    assert processor.vectorized_dirs == []
    assert fake_fs._file_contents[f"{root_uri}/.overview.md"] == old_overview
    assert fake_fs._file_contents[f"{root_uri}/.abstract.md"] == old_abstract


@pytest.mark.asyncio
async def test_content_copy_rebuilds_semantics_when_move_leaves_source_directory_empty(monkeypatch):
    root_uri = "viking://resources/source"
    old_overview = render_abstract_overview(ContextLevel.OVERVIEW, root_uri, "old file summary")
    old_abstract = render_abstract_overview(ContextLevel.ABSTRACT, root_uri, "old abstract")
    fake_fs = _FakeVikingFS(
        tree={root_uri: []},
        file_contents={
            f"{root_uri}/.overview.md": old_overview,
            f"{root_uri}/.abstract.md": old_abstract,
        },
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"deleted": [f"{root_uri}/moved.txt"]},
        generation_trigger="content_copy",
    )

    await executor.run(root_uri)

    assert processor.generated_overviews == [root_uri]
    assert processor.vectorized_dirs == [root_uri]
    overview = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"]).body
    abstract = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.abstract.md"]).body
    assert overview.strip() == "FILES:"
    assert abstract.strip() == "abstract"


@pytest.mark.asyncio
async def test_content_copy_propagates_vector_summary_read_failure(monkeypatch):
    root_uri = "viking://resources/archive"
    file_path = f"{root_uri}/copied.txt"
    fake_fs = _FakeVikingFS(
        tree={root_uri: [{"name": "copied.txt", "isDir": False}]},
        file_contents={file_path: "copied"},
    )
    monkeypatch.setattr(
        "openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: fake_fs
    )
    processor = _FakeProcessor(fake_fs)

    async def fail_summary_read(file_paths, ctx=None):
        raise RuntimeError("vector backend unavailable")

    processor._load_transfer_file_summaries = fail_summary_read
    executor = SemanticTreeExecutor(
        processor=processor,
        context_type="resource",
        max_concurrent_llm=2,
        ctx=RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER),
        incremental_update=True,
        target_uri=root_uri,
        recursive=False,
        changes={"added": [file_path]},
        generation_trigger="content_copy",
    )

    with pytest.raises(RuntimeError, match="vector backend unavailable"):
        await executor.run(root_uri)


if __name__ == "__main__":
    pytest.main([__file__])