Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

24.9 KB
Raw Blame
"""One-KB moves retain bytes, assignments, and saved qualified references."""

from __future__ import annotations

from io import BytesIO
import json
from pathlib import Path
from types import SimpleNamespace

from fastapi import BackgroundTasks, UploadFile
import pytest

from deeptutor.multi_user.knowledge_access import (
    current_kb_manager,
    list_visible_knowledge_bases,
    resolve_kb,
)
from deeptutor.services.workspace import ContentWorkspaceService, WorkspaceError
from deeptutor.services.workspace.context import workspace_context
from deeptutor.services.workspace.kb_move import move_kb, preview_kb_move
from deeptutor.services.workspace.knowledge import qualified_kb_id


def _make_kb(name: str, marker: bytes) -> None:
    manager = current_kb_manager()
    folder = manager.base_dir / name
    (folder / "raw").mkdir(parents=True)
    (folder / "raw" / "source.pdf").write_bytes(marker)
    (folder / "version-1" / "index").parent.mkdir()
    (folder / "version-1" / "index").write_bytes(b"index:" + marker)
    manager.config = manager._load_config()
    manager.config.setdefault("knowledge_bases", {})[name] = {
        "path": name,
        "rag_provider": "llamaindex",
        "status": "ready",
        "description": "an atlas",
    }
    manager._save_config()


def test_move_one_kb_into_nonempty_workspace_preserves_bytes_and_references(as_user):
    with as_user("alice"):
        _make_kb("atlas", b"source-pixels")
        source_manager = current_kb_manager()
        source_manager.config = source_manager._load_config()
        source_manager.config["defaults"] = {"default_kb": "atlas"}
        source_manager._save_config()
        service = ContentWorkspaceService()
        destination = service.create_workspace("Research")
        destination_id = destination["workspace_id"]
        with workspace_context(destination_id):
            _make_kb("existing", b"keep-me")
        old_id = qualified_kb_id("atlas")
        new_id = qualified_kb_id("atlas", destination_id)
        consumer = service.create_workspace("Consumer", resources={"knowledge_bases": [old_id]})

        plan = preview_kb_move(old_id, destination_id)
        assert plan["blockers"] == []
        assert plan["files"] == 2
        assert [row["display_name"] for row in plan["assignments"]] == ["Consumer"]

        result = move_kb(old_id, destination_id)
        assert result["target_id"] == new_id
        with workspace_context(destination_id):
            manager = current_kb_manager()
            assert set(manager.list_knowledge_bases()) == {"atlas", "existing"}
            assert manager._load_config()["defaults"]["default_kb"] == "atlas"
            assert (
                manager.base_dir / "atlas" / "raw" / "source.pdf"
            ).read_bytes() == b"source-pixels"
            assert (
                manager.base_dir / "atlas" / "version-1" / "index"
            ).read_bytes() == b"index:source-pixels"
            assert (manager.base_dir / "existing" / "raw" / "source.pdf").read_bytes() == b"keep-me"
        assert "atlas" not in current_kb_manager().list_knowledge_bases()
        assert current_kb_manager()._load_config()["defaults"]["default_kb"] is None
        assert resolve_kb(old_id).id == new_id
        assert resolve_kb("user:kb:atlas").id == new_id
        assert resolve_kb("atlas").id == new_id
        moved_in_account = next(
            row for row in list_visible_knowledge_bases() if row["id"] == new_id
        )
        assert moved_in_account["provenance_label"] == "Research"
        with workspace_context(consumer["workspace_id"]):
            assert [row["id"] for row in list_visible_knowledge_bases()] == [new_id]
            assert resolve_kb(old_id).base_dir == resolve_kb(new_id).base_dir
        selected = next(
            row for row in service._catalog() if row["workspace_id"] == consumer["workspace_id"]
        )
        assert selected["resources"]["knowledge_bases"] == [new_id]


def test_move_reports_name_conflict_without_touching_either_catalog(as_user):
    with as_user("alice"):
        _make_kb("atlas", b"source")
        service = ContentWorkspaceService()
        destination = service.create_workspace("Research")["workspace_id"]
        with workspace_context(destination):
            _make_kb("atlas", b"destination")
        source_id = qualified_kb_id("atlas")
        assert "already contains" in preview_kb_move(source_id, destination)["blockers"][0]
        with pytest.raises(WorkspaceError, match="already contains"):
            move_kb(source_id, destination)
        assert (
            current_kb_manager().base_dir / "atlas" / "raw" / "source.pdf"
        ).read_bytes() == b"source"
        with workspace_context(destination):
            assert (
                current_kb_manager().base_dir / "atlas" / "raw" / "source.pdf"
            ).read_bytes() == b"destination"


def test_failed_publish_restores_source_destination_and_assignment(as_user, monkeypatch):
    from deeptutor.services.workspace import kb_move

    with as_user("alice"):
        _make_kb("atlas", b"source")
        service = ContentWorkspaceService()
        destination = service.create_workspace("Research")["workspace_id"]
        old_id = qualified_kb_id("atlas")
        consumer = service.create_workspace("Consumer", resources={"knowledge_bases": [old_id]})
        source_config = current_kb_manager().config_file
        original = kb_move.atomic_write_json
        failed = False

        def fail_once(path, payload):
            nonlocal failed
            if path == source_config and not failed:
                failed = True
                raise OSError("injected write failure")
            original(path, payload)

        monkeypatch.setattr(kb_move, "atomic_write_json", fail_once)
        with pytest.raises(OSError, match="injected"):
            move_kb(old_id, destination)
        assert "atlas" in json.loads(source_config.read_text())["knowledge_bases"]
        assert (source_config.parent / "atlas" / "raw" / "source.pdf").read_bytes() == b"source"
        with workspace_context(destination):
            assert "atlas" not in current_kb_manager().list_knowledge_bases()
            assert not (current_kb_manager().base_dir / "atlas").exists()
        row = next(
            row for row in service._catalog() if row["workspace_id"] == consumer["workspace_id"]
        )
        assert row["resources"]["knowledge_bases"] == [old_id]
        assert resolve_kb(old_id).base_dir == source_config.parent


def test_legacy_account_selection_redirect_is_private_to_owner(as_user):
    from deeptutor.core.errors import AccessError

    with as_user("alice"):
        _make_kb("atlas", b"source")
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        move_kb(qualified_kb_id("atlas"), destination)
        assert resolve_kb("user:kb:atlas").id == qualified_kb_id("atlas", destination)
    with as_user("bob"):
        with pytest.raises(AccessError):
            resolve_kb("user:kb:atlas")


def test_saved_general_chat_legacy_selection_is_rewritten_on_move(as_user):
    with as_user("alice"):
        _make_kb("atlas", b"source")
        service = ContentWorkspaceService()
        service.general_binding()
        general_id = service._builtin_id("general")
        # Older chat catalogs stored role-prefixed IDs before the qualified
        # account/workspace resource scheme existed.
        with service._catalog_connection() as conn:
            record = conn.execute(
                "SELECT payload FROM workspaces WHERE id = ?", (general_id,)
            ).fetchone()
            row = json.loads(record[0])
            row["resources"] = {"knowledge_bases": ["user:kb:atlas"]}
            conn.execute(
                "UPDATE workspaces SET payload = ? WHERE id = ?",
                (json.dumps(row), general_id),
            )
        destination = service.create_workspace("Research")["workspace_id"]
        plan = preview_kb_move(qualified_kb_id("atlas"), destination)
        assert any(item["workspace_id"] == general_id for item in plan["assignments"])
        move_kb(qualified_kb_id("atlas"), destination)
        target_id = qualified_kb_id("atlas", destination)
        with workspace_context():
            assert resolve_kb("user:kb:atlas").id == target_id
            assert [item["id"] for item in list_visible_knowledge_bases()] == [target_id]
        general = next(row for row in service._catalog() if row["workspace_id"] == general_id)
        assert general["resources"]["knowledge_bases"] == [target_id]


def test_legacy_admin_selection_redirects_only_in_admin_account(as_user):
    with as_user("operator", role="admin"):
        _make_kb("atlas", b"source")
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        move_kb(qualified_kb_id("atlas"), destination)
        assert resolve_kb("admin:kb:atlas").id == qualified_kb_id("atlas", destination)


def test_owner_can_move_kb_excluded_from_general_chat_selection(as_user):
    with as_user("alice"):
        _make_kb("selected", b"other")
        _make_kb("atlas", b"source")
        service = ContentWorkspaceService()
        service.update_workspace(
            service._builtin_id("general"),
            resources={"knowledge_bases": [qualified_kb_id("selected")]},
        )
        destination = service.create_workspace("Research")["workspace_id"]
        assert preview_kb_move(qualified_kb_id("atlas"), destination)["blockers"] == []
        move_kb(qualified_kb_id("atlas"), destination)
        with workspace_context(destination):
            assert "atlas" in current_kb_manager().list_knowledge_bases()


def test_moved_away_name_cannot_be_registered_by_connected_providers(as_user, tmp_path):
    with as_user("alice"):
        _make_kb("atlas", b"source")
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        move_kb(qualified_kb_id("atlas"), destination)
        manager = current_kb_manager()
        attempts = [
            lambda: manager.register_connected_entry("atlas", {"type": "obsidian"}),
            lambda: manager.register_obsidian_vault("atlas", str(tmp_path)),
            lambda: manager.register_linked_kb("atlas", str(tmp_path), "llamaindex"),
            lambda: manager.register_lightrag_server_kb("atlas", "http://localhost:9621"),
            lambda: manager.register_marginnote4_kb("atlas"),
            lambda: manager.register_ima_kb("atlas", "", "", "remote-id"),
            lambda: manager.register_weknora_kb("atlas", "http://localhost", "key", "remote-id"),
            lambda: manager.register_subagent_connection("atlas", "local"),
        ]
        for attempt in attempts:
            with pytest.raises(ValueError, match="reserved by an earlier move"):
                attempt()
        assert "atlas" not in manager._load_config()["knowledge_bases"]

        second_destination = ContentWorkspaceService().create_workspace("Archive")["workspace_id"]
        move_kb(qualified_kb_id("atlas", destination), second_destination)
        with workspace_context(destination):
            with pytest.raises(ValueError, match="reserved by an earlier move"):
                current_kb_manager().register_connected_entry("atlas", {"type": "obsidian"})
        assert resolve_kb("user:kb:atlas").id == qualified_kb_id("atlas", second_destination)


def test_marginnote_move_is_blocked_before_synced_database_is_lost(as_user):
    from deeptutor.capabilities.marginnote4.store import default_db_path

    with as_user("alice"):
        current_kb_manager().register_connected_entry("notes", {"type": "marginnote4"})
        database = default_db_path("notes")
        database.parent.mkdir(parents=True, exist_ok=True)
        database.write_bytes(b"synced-notes")
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        source_id = qualified_kb_id("notes")
        assert "MarginNote 4" in preview_kb_move(source_id, destination)["blockers"][0]
        with pytest.raises(WorkspaceError, match="MarginNote 4"):
            move_kb(source_id, destination)
        assert database.read_bytes() == b"synced-notes"
        assert "notes" in current_kb_manager().list_knowledge_bases()
        with workspace_context(destination):
            assert "notes" not in current_kb_manager().list_knowledge_bases()


def test_moved_llamaindex_retrieval_keeps_citations_and_images(as_user, monkeypatch):
    from llama_index.core import Settings, VectorStoreIndex
    from llama_index.core.embeddings import MockEmbedding
    from llama_index.core.schema import ImageNode, TextNode

    from deeptutor.services.path_service import get_path_service
    from deeptutor.services.rag.pipelines.llamaindex import storage
    from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline

    monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=8))
    with as_user("alice"):
        _make_kb("atlas", b"source text")
        source_root = current_kb_manager().base_dir / "atlas"
        source_file = source_root / "raw" / "source.pdf"
        cache_image = get_path_service().get_parse_cache_root() / "aa" / "image.png"
        cache_image.parent.mkdir(parents=True)
        cache_image.write_bytes(b"image bytes")
        text_node = TextNode(
            text="source alpha",
            id_="text-1",
            metadata={"file_name": "source.pdf", "file_path": str(source_file)},
            embedding=[0.5] * 8,
        )
        image_node = ImageNode(
            text="[Image] source.pdf",
            id_="image-1",
            image_path=str(cache_image),
            metadata={"file_name": "source.pdf", "file_path": str(source_file)},
            embedding=[0.5] * 8,
        )
        index = VectorStoreIndex(nodes=[text_node, image_node])
        index.storage_context.persist(persist_dir=str(source_root / "version-1"))
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]

        move_kb(qualified_kb_id("atlas"), destination)
        assert not source_root.exists()
        cache_image.unlink()

        with workspace_context(destination):
            target_root = current_kb_manager().base_dir / "atlas"
            nodes = storage.retrieve_nodes(target_root / "version-1", "alpha", top_k=2)
            assert {node.node.node_id for node in nodes} == {"text-1", "image-1"}
            result = LlamaIndexPipeline._nodes_to_result(None, "alpha", nodes)
            assert {source["source"] for source in result["sources"]} == {
                str(target_root / "raw" / "source.pdf")
            }
            assert Path(result["sources"][0]["source"]).is_file()
            moved_image = next(node.node for node in nodes if node.node.node_id == "image-1")
            assert Path(moved_image.image_path).is_file()
            assert Path(moved_image.image_path).read_bytes() == b"image bytes"
            assert Path(moved_image.image_path).is_relative_to(target_root)


@pytest.mark.asyncio
async def test_moved_lightrag_search_opens_published_native_workspace(as_user, monkeypatch):
    from deeptutor.services.rag.pipelines.lightrag import engine, pipeline, storage

    with as_user("alice"):
        _make_kb("atlas", b"source")
        source_root = current_kb_manager().base_dir / "atlas"
        source_version = source_root / "version-1"
        source_workspace = engine.workspace_for(source_version)
        native_store = source_version / source_workspace
        native_store.mkdir()
        (native_store / "kv_store_doc_status.json").write_text(
            json.dumps({"doc": {"status": "processed", "chunks_list": ["chunk"]}})
        )
        (native_store / "kv_store_text_chunks.json").write_text(
            json.dumps({"chunk": {"content": "grounded passage"}})
        )
        (source_version / "meta.json").write_text(
            json.dumps(
                {
                    "version": "version-1",
                    "signature": "lightrag",
                    "provider": "lightrag",
                    "state": "published",
                    "lightrag_adapter_schema": 2,
                    "parser_bridge_schema": 1,
                    "workspace": source_workspace,
                }
            )
        )
        manager = current_kb_manager()
        manager.config = manager._load_config()
        manager.config["knowledge_bases"]["atlas"]["rag_provider"] = "lightrag"
        manager._save_config()
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        move_kb(qualified_kb_id("atlas"), destination)

        with workspace_context(destination):
            target_root = current_kb_manager().base_dir / "atlas"
            target_version = target_root / "version-1"
            assert not source_root.exists()
            assert engine.workspace_for(target_version) == source_workspace
            assert storage.latest_published_root(target_root) == target_version

            async def in_worker(job):
                return await job(None)

            async def no_op(*_args, **_kwargs):
                return None

            async def fake_query(rag, *_args):
                payload = json.loads(
                    (rag.working_dir / rag.workspace / "kv_store_text_chunks.json").read_text()
                )
                return payload["chunk"]["content"], [{"file_path": "source.pdf"}]

            def fake_build_rag(working_dir, **_kwargs):
                return SimpleNamespace(
                    working_dir=Path(working_dir),
                    workspace=engine.workspace_for(working_dir),
                )

            monkeypatch.setattr(pipeline, "run_in_worker_loop", in_worker)
            monkeypatch.setattr(engine, "build_rag", fake_build_rag)
            monkeypatch.setattr(engine, "initialize", no_op)
            monkeypatch.setattr(engine, "finalize", no_op)
            monkeypatch.setattr(engine, "query_with_sources", fake_query)
            monkeypatch.setattr(storage, "require_compatible_embedding", lambda *_args: None)
            monkeypatch.setattr(
                "deeptutor.services.rag.pipelines.lightrag.roles.resolve_query_roles",
                lambda: {},
            )
            monkeypatch.setattr(
                "deeptutor.services.embedding.get_embedding_config", lambda: object()
            )
            rag_pipeline = pipeline.LightRagPipeline(kb_base_dir=str(target_root.parent))
            monkeypatch.setattr(rag_pipeline, "_ensure_available", lambda: None)
            result = await rag_pipeline.search("alpha", "atlas")
            assert result["content"] == "grounded passage"
            assert result["sources"] == [{"file_path": "source.pdf"}]


@pytest.mark.asyncio
async def test_create_destination_overrides_library_scope(as_user, monkeypatch):
    from deeptutor.services import config

    monkeypatch.setattr(config, "load_config_with_main", lambda *_args: {})
    from deeptutor.api.routers import knowledge
    from deeptutor.services.workspace.knowledge import library_request

    with as_user("alice"):
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        seen = []

        async def fake_create(*_args):
            seen.append(knowledge._current_kb_base_dir())
            return {"name": "new"}

        monkeypatch.setattr(knowledge, "_create_knowledge_base_owned", fake_create)
        token = library_request.set(True)
        try:
            result = await knowledge.create_knowledge_base(
                BackgroundTasks(),
                name="new",
                files=[],
                rag_provider="llamaindex",
                pageindex_mode="",
                search_mode="",
                rel_paths=[],
                indexing_llm="",
                embedding_model="",
                storage_workspace_id=destination,
            )
        finally:
            library_request.reset(token)
        with workspace_context(destination):
            assert seen == [current_kb_manager().base_dir]
        assert result["id"] == qualified_kb_id("new", destination)


@pytest.mark.asyncio
async def test_uploaded_create_background_task_keeps_destination_scope(as_user, monkeypatch):
    from deeptutor.services import config

    monkeypatch.setattr(config, "load_config_with_main", lambda *_args: {})
    from deeptutor.api.routers import knowledge
    from deeptutor.knowledge.initializer import KnowledgeBaseInitializer
    from deeptutor.services.config import get_kb_config_service
    from deeptutor.services.workspace.knowledge import library_request

    with as_user("alice"):
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        account_root = current_kb_manager().base_dir
        seen = []

        async def fake_process(self):
            seen.append(knowledge._current_kb_base_dir())
            get_kb_config_service().set_kb_config(self.kb_name, {"indexed_in_scope": True})

        monkeypatch.setattr(KnowledgeBaseInitializer, "process_documents", fake_process)
        monkeypatch.setattr(knowledge, "_assert_provider_ready", lambda *_args: None)
        background = BackgroundTasks()
        token = library_request.set(True)
        try:
            result = await knowledge.create_knowledge_base(
                background,
                name="atlas",
                files=[UploadFile(file=BytesIO(b"grounded text"), filename="source.txt")],
                rag_provider="llamaindex",
                pageindex_mode="",
                search_mode="",
                rel_paths=[],
                indexing_llm="",
                embedding_model="",
                storage_workspace_id=destination,
            )
        finally:
            library_request.reset(token)

        with workspace_context(destination):
            target_root = current_kb_manager().base_dir
            assert (
                current_kb_manager()._load_config()["knowledge_bases"]["atlas"]["status"]
                == "processing"
            )
        assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]
        assert result["task_id"]

        # Execute Starlette's real callback after the route's scope has gone.
        await background()

        assert seen == [target_root]
        with workspace_context(destination):
            entry = current_kb_manager()._load_config()["knowledge_bases"]["atlas"]
            assert entry["status"] == "ready"
            assert entry["indexed_in_scope"] is True
            assert (target_root / "atlas" / "raw" / "source.txt").read_bytes() == b"grounded text"
        assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]
        assert not (account_root / "atlas").exists()


@pytest.mark.asyncio
async def test_upload_to_moved_kb_background_task_uses_its_storage_scope(as_user, monkeypatch):
    from deeptutor.services import config

    monkeypatch.setattr(config, "load_config_with_main", lambda *_args: {})
    from deeptutor.api.routers import knowledge
    from deeptutor.services.config import get_kb_config_service
    from deeptutor.services.workspace.knowledge import library_request

    with as_user("alice"):
        _make_kb("atlas", b"original")
        destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
        move_kb(qualified_kb_id("atlas"), destination)
        account_root = current_kb_manager().base_dir
        seen = []

        class FakeAdder:
            def __init__(self, **_kwargs):
                pass

            def add_documents(self, paths, **_kwargs):
                return paths

            async def process_new_documents(self, paths):
                return SimpleNamespace(processed_count=len(paths), has_failures=False)

            def update_metadata(self, _count):
                seen.append(knowledge._current_kb_base_dir())
                get_kb_config_service().set_kb_config("atlas", {"uploaded_in_scope": True})

        monkeypatch.setattr(knowledge, "DocumentAdder", FakeAdder)
        monkeypatch.setattr(knowledge, "_assert_provider_ready", lambda *_args: None)
        monkeypatch.setattr(knowledge, "_freeze_append_indexing", lambda *_args: None)
        background = BackgroundTasks()
        token = library_request.set(True)
        try:
            result = await knowledge.upload_files(
                qualified_kb_id("atlas", destination),
                background,
                files=[UploadFile(file=BytesIO(b"second source"), filename="new.txt")],
                rag_provider=None,
                rel_paths=[],
                dest_subdir=None,
                image_analysis=None,
            )
        finally:
            library_request.reset(token)
        assert result["task_id"]
        await background()

        with workspace_context(destination):
            target_root = current_kb_manager().base_dir
            assert seen == [target_root]
            assert (
                current_kb_manager()._load_config()["knowledge_bases"]["atlas"]["uploaded_in_scope"]
                is True
            )
            assert (target_root / "atlas" / "raw" / "new.txt").read_bytes() == b"second source"
        assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]
        assert not (account_root / "atlas").exists()