Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(knowledge): preserve moved KB retrieval and scoped jobs

Bingxi Zhao (Frank)a824e4b01f54
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index f12be562f..367fc10f5 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -99,6 +99,7 @@ from deeptutor.services.rag.pipelines.ima.config import (
)
from deeptutor.services.rag.visual_assets import VisualAssetStore
from deeptutor.services.web_source.scheduler import get_web_source_sync_scheduler
+from deeptutor.services.workspace.knowledge import workspace_id_for_kb_base_dir
from deeptutor.utils.document_extractor import (
MAX_EXTRACTED_CHARS_PER_DOC,
DocumentExtractionError,
@@ -970,16 +971,37 @@ def _matching_index_is_valid(kb_name: str, matching_version: dict | None) -> boo
return False
-async def run_initialization_task(initializer: KnowledgeBaseInitializer, task_id: str):
+async def run_initialization_task(
+ initializer: KnowledgeBaseInitializer,
+ task_id: str,
+ *,
+ storage_workspace_id: str | None = None,
+):
"""Background task for knowledge base initialization"""
owner = getattr(initializer, "owner", None)
if owner is not None and get_current_user_or_none() != owner:
token = set_current_user(owner)
try:
- return await run_initialization_task(initializer, task_id)
+ return await run_initialization_task(
+ initializer, task_id, storage_workspace_id=storage_workspace_id
+ )
finally:
reset_current_user(token)
+ # FastAPI runs BackgroundTasks after the create route has left its
+ # workspace_context. Keep the selected storage scope for the parser,
+ # indexer, config service, and final status write, including on failure.
+ if storage_workspace_id is not None:
+ from deeptutor.services.workspace.context import workspace_context
+ from deeptutor.services.workspace.knowledge import library_request
+
+ with workspace_context(storage_workspace_id):
+ token = library_request.set(False)
+ try:
+ return await run_initialization_task(initializer, task_id)
+ finally:
+ library_request.reset(token)
+
task_manager = TaskIDManager.get_instance()
task_stream_manager = get_task_stream_manager()
task_stream_manager.ensure_task(task_id)
@@ -1098,6 +1120,7 @@ async def run_upload_processing_task(
folder_root: str = None,
owner=None,
accepted_indexing_snapshot=None,
+ storage_workspace_id: str | None = None,
):
"""Background task for processing uploaded files.
@@ -1123,10 +1146,31 @@ async def run_upload_processing_task(
folder_id=folder_id,
folder_root=folder_root,
accepted_indexing_snapshot=accepted_indexing_snapshot,
+ storage_workspace_id=storage_workspace_id,
)
finally:
reset_current_user(token)
+ if storage_workspace_id is not None:
+ from deeptutor.services.workspace.context import workspace_context
+ from deeptutor.services.workspace.knowledge import library_request
+
+ with workspace_context(storage_workspace_id):
+ token = library_request.set(False)
+ try:
+ return await run_upload_processing_task(
+ kb_name=kb_name,
+ base_dir=base_dir,
+ uploaded_file_paths=uploaded_file_paths,
+ task_id=task_id,
+ rag_provider=rag_provider,
+ folder_id=folder_id,
+ folder_root=folder_root,
+ accepted_indexing_snapshot=accepted_indexing_snapshot,
+ )
+ finally:
+ library_request.reset(token)
+
task_manager = TaskIDManager.get_instance()
task_stream_manager = get_task_stream_manager()
task_stream_manager.ensure_task(task_id)
@@ -3329,6 +3373,7 @@ async def upload_files(
task_id=task_id,
rag_provider=kb_provider,
owner=get_current_user(),
+ storage_workspace_id=workspace_id_for_kb_base_dir(kb_base_dir),
)
return {
@@ -3611,7 +3656,14 @@ async def _create_knowledge_base_owned(
total=len(uploaded_files),
)
- background_tasks.add_task(run_initialization_task, initializer, task_id)
+ from deeptutor.services.workspace.context import current_workspace_id
+
+ background_tasks.add_task(
+ run_initialization_task,
+ initializer,
+ task_id,
+ storage_workspace_id=current_workspace_id(),
+ )
logger.info(f"KB '{name}' created, processing {len(uploaded_files)} files in background")
@@ -3639,6 +3691,7 @@ async def run_reindex_task(
owner=None,
embedding_selection=None,
embedding_config=None,
+ storage_workspace_id: str | None = None,
) -> None:
"""Re-index a KB's raw documents with its selected embedding configuration.
@@ -3657,10 +3710,30 @@ async def run_reindex_task(
indexing_snapshot=indexing_snapshot,
embedding_selection=embedding_selection,
embedding_config=embedding_config,
+ storage_workspace_id=storage_workspace_id,
)
finally:
reset_current_user(token)
+ if storage_workspace_id is not None:
+ from deeptutor.services.workspace.context import workspace_context
+ from deeptutor.services.workspace.knowledge import library_request
+
+ with workspace_context(storage_workspace_id):
+ token = library_request.set(False)
+ try:
+ return await run_reindex_task(
+ kb_name=kb_name,
+ base_dir=base_dir,
+ task_id=task_id,
+ signature_hash=signature_hash,
+ indexing_snapshot=indexing_snapshot,
+ embedding_selection=embedding_selection,
+ embedding_config=embedding_config,
+ )
+ finally:
+ library_request.reset(token)
+
task_manager = TaskIDManager.get_instance()
task_stream_manager = get_task_stream_manager()
task_stream_manager.ensure_task(task_id)
@@ -4044,6 +4117,7 @@ async def reindex_knowledge_base(
signature_hash=signature_hash,
indexing_snapshot=indexing_snapshot,
owner=get_current_user(),
+ storage_workspace_id=workspace_id_for_kb_base_dir(kb_base_dir),
**(
{"embedding_selection": embedding_selection, "embedding_config": embedding_config}
if embedding_selection
@@ -4563,6 +4637,7 @@ async def sync_folder(kb_name: str, folder_id: str, background_tasks: Background
folder_id=folder_id, # Pass folder_id to update state on success
folder_root=folder_path, # Preserve each file's path relative to this root
owner=get_current_user(),
+ storage_workspace_id=workspace_id_for_kb_base_dir(kb_base_dir),
)
return SyncFolderResponse(
diff --git a/deeptutor/knowledge/manager.py b/deeptutor/knowledge/manager.py
index b1467dbe9..58a698661 100644
--- a/deeptutor/knowledge/manager.py
+++ b/deeptutor/knowledge/manager.py
@@ -56,6 +56,23 @@ from deeptutor.services.web_source.crawler import MAX_CRAWL_DEPTH, MAX_CRAWL_PAG
logger = logging.getLogger(__name__)
+def _assert_move_id_available(base_dir: Path, name: str) -> None:
+ """Prevent a new registration from shadowing a saved moved-KB redirect."""
+ from deeptutor.services.workspace.knowledge import (
+ canonical_kb_id,
+ qualified_kb_id,
+ workspace_id_for_kb_base_dir,
+ )
+
+ workspace_id = workspace_id_for_kb_base_dir(base_dir)
+ if workspace_id is None:
+ # A manager targeting a partner's separate directory has no catalog ID.
+ return
+ resource_id = qualified_kb_id(name, workspace_id)
+ if canonical_kb_id(resource_id) != resource_id:
+ raise ValueError("This knowledge base ID is reserved by an earlier move.")
+
+
# How long an entry can be missing its KB directory before ``list_knowledge_bases``
# treats it as a stale orphan. The KB create flow writes the "initializing"
# config entry before the on-disk folder is created, so a too-short grace would
@@ -739,6 +756,7 @@ class KnowledgeBaseManager:
def register_knowledge_base(self, name: str, description: str = "", set_default: bool = False):
"""Register a knowledge base"""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
kb_dir = self.base_dir / name
if not kb_dir.exists():
raise ValueError(f"Knowledge base directory does not exist: {kb_dir}")
@@ -771,6 +789,7 @@ class KnowledgeBaseManager:
raise ValueError("Knowledge base name is required.")
if not is_connected_kb(entry):
raise ValueError(f"Not a connected knowledge base entry: {name}")
+ _assert_move_id_available(self.base_dir, name)
self.config = self._load_config()
knowledge_bases = self.config.setdefault("knowledge_bases", {})
@@ -789,6 +808,7 @@ class KnowledgeBaseManager:
live. Raises ``ValueError`` on a missing/invalid path or a name clash.
"""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
vault = Path(vault_path).expanduser()
if not vault.is_dir():
raise ValueError(f"Vault path is not a directory: {vault_path}")
@@ -832,6 +852,7 @@ class KnowledgeBaseManager:
Raises ``ValueError`` on a missing/invalid path or a name clash.
"""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
provider = normalize_provider_name(provider)
folder = Path(external_path).expanduser()
if not folder.is_dir():
@@ -874,6 +895,7 @@ class KnowledgeBaseManager:
) -> dict:
"""Register a local or remote agent connection without creating an index."""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
agent_kind = (agent_kind or "").strip()
if agent_kind == "partner":
raise ValueError("Select partners directly through Ask partner instead.")
@@ -926,6 +948,7 @@ class KnowledgeBaseManager:
name clash.
"""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
server_url = (server_url or "").strip().rstrip("/")
if not server_url:
raise ValueError("LightRAG server URL is required.")
@@ -973,6 +996,7 @@ class KnowledgeBaseManager:
claimed by another library.
"""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
self.config = self._load_config()
knowledge_bases = self.config.setdefault("knowledge_bases", {})
@@ -1052,6 +1076,7 @@ class KnowledgeBaseManager:
or a name clash.
"""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
client_id = (client_id or "").strip()
api_key = (api_key or "").strip()
knowledge_base_id = (knowledge_base_id or "").strip()
@@ -1095,6 +1120,7 @@ class KnowledgeBaseManager:
) -> dict:
"""Register a self-hosted WeKnora knowledge base as a pointer KB."""
name = validate_knowledge_base_name(name)
+ _assert_move_id_available(self.base_dir, name)
server_url = (server_url or "").strip().rstrip("/")
api_key = (api_key or "").strip()
knowledge_base_id = (knowledge_base_id or "").strip()
diff --git a/deeptutor/multi_user/knowledge_access.py b/deeptutor/multi_user/knowledge_access.py
index 777d46402..9bdb34b7b 100644
--- a/deeptutor/multi_user/knowledge_access.py
+++ b/deeptutor/multi_user/knowledge_access.py
@@ -94,7 +94,7 @@ def resolve_kb(kb_ref: str, *, require_write: bool = False) -> KnowledgeResource
selected = None if library_request.get() else current_resources().knowledge_bases
if selected is not None:
return resolve_selected(kb_ref, selected, require_write=require_write)
- if parse_kb_id(kb_ref) is not None:
+ if parse_kb_id(kb_ref) is not None or canonical_kb_id(kb_ref) != kb_ref:
return resolve_qualified(kb_ref, require_write=require_write)
if kb_ref and not kb_ref.startswith((ADMIN_PREFIX, USER_PREFIX)):
previous_id = qualified_kb_id(kb_ref, current_workspace_id())
diff --git a/deeptutor/services/rag/pipelines/lightrag/engine.py b/deeptutor/services/rag/pipelines/lightrag/engine.py
index 7a543ca64..1022246ef 100644
--- a/deeptutor/services/rag/pipelines/lightrag/engine.py
+++ b/deeptutor/services/rag/pipelines/lightrag/engine.py
@@ -7,7 +7,9 @@ from collections.abc import Awaitable, Callable, Iterable, Mapping
import hashlib
from importlib.metadata import PackageNotFoundError, version
import inspect
+import json
from pathlib import Path
+import re
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@@ -181,7 +183,22 @@ def _register_parser() -> None:
def workspace_for(working_dir: Path) -> str:
- identity = str(Path(working_dir).resolve()).encode("utf-8")
+ root = Path(working_dir)
+ # A published version keeps its native LightRAG workspace name when its
+ # containing KB is moved. The name is a hash of the original absolute
+ # version path; recomputing it after a move would open an empty store.
+ try:
+ meta = json.loads((root / "meta.json").read_text(encoding="utf-8"))
+ published = str(meta.get("workspace") or "") if isinstance(meta, dict) else ""
+ if (
+ isinstance(meta, dict)
+ and meta.get("provider") == "lightrag"
+ and re.fullmatch(r"deeptutor_[0-9a-f]{16}", published)
+ ):
+ return published
+ except (OSError, ValueError):
+ pass
+ identity = str(root.resolve()).encode("utf-8")
return f"deeptutor_{hashlib.sha256(identity).hexdigest()[:16]}"
diff --git a/deeptutor/services/workspace/kb_move.py b/deeptutor/services/workspace/kb_move.py
index aa01f258a..966969035 100644
--- a/deeptutor/services/workspace/kb_move.py
+++ b/deeptutor/services/workspace/kb_move.py
@@ -7,15 +7,15 @@ until the destination, catalog entries, and redirects have been published.
from __future__ import annotations
from copy import deepcopy
+import hashlib
import json
import os
from pathlib import Path
import shutil
import uuid
-from deeptutor.knowledge.kb_types import is_connected_kb
+from deeptutor.knowledge.kb_types import MARGINNOTE4_KB_TYPE, is_connected_kb
from deeptutor.multi_user.context import get_current_user
-from deeptutor.multi_user.knowledge_access import resolve_kb
from deeptutor.services.file_io import atomic_write_json
from deeptutor.services.path_service import get_path_service
from deeptutor.services.rag.factory import has_ready_provider_index
@@ -32,6 +32,7 @@ from deeptutor.services.workspace.knowledge import (
move_aliases,
parse_kb_id,
qualified_kb_id,
+ resolve_qualified,
)
from deeptutor.services.workspace.models import WorkspaceError
@@ -73,6 +74,14 @@ def _shared_grants(name: str) -> list[str]:
]
+def _legacy_account_id(name: str) -> str:
+ # Historical general-chat selections used a role-specific prefix. Keep
+ # that alias only for the owning account; custom workspaces can have
+ # same-name KBs, so their legacy IDs cannot be redirected globally.
+ role = "admin" if get_current_user().is_admin else "user"
+ return f"{role}:kb:{name}"
+
+
def preview_kb_move(source_id: str, target_workspace_id: str) -> dict:
"""Return a collision and assignment preview without changing either store."""
from deeptutor.services.workspace import get_content_workspace_service
@@ -83,8 +92,9 @@ def preview_kb_move(source_id: str, target_workspace_id: str) -> dict:
raise WorkspaceError("Choose a knowledge base from the library.")
source_workspace_id, name = parsed
target_workspace_id = str(target_workspace_id or "")
- # The qualified resolver checks account ownership and write permission.
- resolve_kb(source_id, require_write=True)
+ # Move is a library operation. Chat selection is a read ceiling for turns,
+ # not an ownership check for managing a different stored KB.
+ resolve_qualified(source_id, require_write=True)
source_root, source_archived = _root(source_workspace_id)
target_root, target_archived = _root(target_workspace_id)
source_config = _config(source_root)
@@ -99,6 +109,11 @@ def preview_kb_move(source_id: str, target_workspace_id: str) -> dict:
blockers.append("Restore the workspace before moving its knowledge base.")
if not isinstance(entry, dict):
blockers.append("The source knowledge base is missing from its configuration.")
+ elif entry.get("type") == MARGINNOTE4_KB_TYPE:
+ blockers.append(
+ "MarginNote 4 stores its synced database outside the knowledge-base folder. "
+ "Move is unavailable until that database can be transferred safely."
+ )
elif not is_connected_kb(entry):
if entry.get("path", name) != name:
blockers.append(
@@ -129,6 +144,8 @@ def preview_kb_move(source_id: str, target_workspace_id: str) -> dict:
file_count = byte_count = 0
aliases = move_aliases()
old_ids = {source_id, *(key for key in aliases if canonical_kb_id(key) == source_id)}
+ if not source_workspace_id:
+ old_ids.add(_legacy_account_id(name))
assignments = [
{"workspace_id": row["workspace_id"], "display_name": row["display_name"]}
for row in get_content_workspace_service()._catalog()
@@ -207,6 +224,102 @@ def _copy_kb(source: Path, destination: Path) -> None:
(destination / candidate.relative_to(source)).mkdir(parents=True, exist_ok=True)
+def _parse_cache_root(workspace_id: str) -> Path:
+ with workspace_context(workspace_id):
+ return get_path_service().get_parse_cache_root()
+
+
+def _rebase_llamaindex_paths(
+ stage: Path, source: Path, target: Path, source_cache_root: Path
+) -> None:
+ """Rebase persisted citations and keep indexed parse-cache images with the KB.
+
+ FAISS vector files are binary and contain no paths. LlamaIndex stores node
+ paths in docstore JSON and, for older simple indexes, vector-store JSON.
+ BM25 sidecars can contain serialized nodes, so discard them; retrieval
+ rebuilds BM25 from the corrected docstore when needed.
+ """
+ copied_images: dict[str, str] = {}
+ source_prefix = str(source) + os.sep
+ cache_root = source_cache_root.resolve()
+
+ def rewrite_path(value: str, key: str) -> str:
+ if value.startswith(source_prefix):
+ relative = Path(value).relative_to(source)
+ if not (stage / relative).is_file():
+ raise WorkspaceError(f"Indexed source is missing: {relative}")
+ return str(target / relative)
+ if key != "image_path" or not Path(value).is_absolute():
+ return value
+ if value in copied_images:
+ return copied_images[value]
+ image = Path(value)
+ try:
+ resolved = image.resolve(strict=True)
+ relative = resolved.relative_to(cache_root)
+ except (OSError, ValueError) as exc:
+ raise WorkspaceError(
+ "An indexed image is outside the movable knowledge base and parse cache."
+ ) from exc
+ path_parts = [image, *list(image.parents)[: len(image.parts) - len(cache_root.parts)]]
+ if resolved != image or not image.is_file() or any(
+ part.is_symlink() for part in path_parts
+ ):
+ raise WorkspaceError("An indexed image cannot be copied safely.")
+ digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
+ asset = Path(".index_assets") / digest / image.name
+ _snapshot(image, stage / asset.parent)
+ copied_images[value] = str(target / asset)
+ return copied_images[value]
+
+ def rewrite(value: object, key: str = "") -> tuple[object, bool]:
+ if isinstance(value, dict):
+ changed = False
+ result = {}
+ for child_key, child_value in value.items():
+ result[child_key], child_changed = rewrite(child_value, child_key)
+ changed |= child_changed
+ return result, changed
+ if isinstance(value, list):
+ children = [rewrite(child) for child in value]
+ return [child for child, _ in children], any(changed for _, changed in children)
+ if isinstance(value, str):
+ if value.startswith(source_prefix) or key == "image_path":
+ updated = rewrite_path(value, key)
+ return updated, updated != value
+ # Some LlamaIndex stores embed node objects as serialized JSON.
+ if key == "__data__" and value.startswith("{") and source_prefix in value:
+ parsed = json.loads(value)
+ updated, changed = rewrite(parsed)
+ if changed:
+ return json.dumps(updated, ensure_ascii=False), True
+ return value, False
+
+ for docstore in list(stage.rglob("docstore.json")):
+ relative = docstore.relative_to(stage)
+ if relative.parts[0] != "llamaindex_storage" and not (
+ relative.parts[0].startswith("version-") or relative.parts[0] == "index_versions"
+ ):
+ continue
+ storage = docstore.parent
+ if not (storage / "index_store.json").is_file():
+ continue
+ for path in (docstore, *storage.glob("*vector_store.json")):
+ # The FAISS store keeps binary bytes under a .json filename.
+ if path != docstore:
+ with path.open("rb") as handle:
+ if handle.read(1) != b"{":
+ continue
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ updated, changed = rewrite(payload)
+ except (ValueError, UnicodeError) as exc:
+ raise WorkspaceError(f"Cannot rebase LlamaIndex paths in {path.name}.") from exc
+ if changed:
+ atomic_write_json(path, updated)
+ shutil.rmtree(storage / "bm25_retriever", ignore_errors=True)
+
+
def move_kb(source_id: str, target_workspace_id: str) -> dict:
"""Publish one verified KB copy and preserve qualified saved references."""
with data_activity(exclusive=True):
@@ -232,6 +345,8 @@ def move_kb(source_id: str, target_workspace_id: str) -> dict:
plan["source_id"],
*(key for key in aliases_before if canonical_kb_id(key) == plan["source_id"]),
}
+ if not plan["source_workspace_id"]:
+ old_ids.add(_legacy_account_id(name))
stage = target_root / f".kb-move-{uuid.uuid4().hex}"
published = False
source_hidden = None
@@ -250,6 +365,13 @@ def move_kb(source_id: str, target_workspace_id: str) -> dict:
_copy_kb(source_path, stage)
entry = source_before["knowledge_bases"][name]
provider = entry.get("rag_provider")
+ if provider in {None, "", "llamaindex"}:
+ _rebase_llamaindex_paths(
+ stage,
+ source_path,
+ target_path,
+ _parse_cache_root(plan["source_workspace_id"]),
+ )
if (
not is_connected_kb(entry)
and has_ready_provider_index(source_path, provider)
@@ -282,6 +404,8 @@ def move_kb(source_id: str, target_workspace_id: str) -> dict:
for key, value in aliases_before.items()
}
aliases_next[plan["source_id"]] = plan["target_id"]
+ if not plan["source_workspace_id"]:
+ aliases_next[_legacy_account_id(name)] = plan["target_id"]
atomic_write_json(aliases_path, aliases_next)
atomic_write_json(source_config_path, source_next)
if source_path.is_dir():
diff --git a/deeptutor/services/workspace/knowledge.py b/deeptutor/services/workspace/knowledge.py
index c6a234111..7a39d6a1f 100644
--- a/deeptutor/services/workspace/knowledge.py
+++ b/deeptutor/services/workspace/knowledge.py
@@ -6,6 +6,7 @@ from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import replace
import json
+from pathlib import Path
from fastapi import HTTPException
@@ -16,6 +17,25 @@ def qualified_kb_id(name: str, workspace_id: str = "") -> str:
return f"workspace:{workspace_id}:kb:{name}" if workspace_id else f"account:kb:{name}"
+def workspace_id_for_kb_base_dir(base_dir: str | Path) -> str | None:
+ """Find the owned storage scope for a KB directory used by a background job."""
+ from deeptutor.multi_user.paths import get_account_path_service
+ from deeptutor.services.path_service import get_path_service
+ from deeptutor.services.workspace import get_content_workspace_service
+ from deeptutor.services.workspace.context import workspace_context
+
+ root = Path(base_dir).resolve()
+ if root == get_account_path_service().get_knowledge_bases_root().resolve():
+ return ""
+ for row in get_content_workspace_service()._catalog():
+ if row.get("kind") != "workspace":
+ continue
+ with workspace_context(row["workspace_id"]):
+ if root == get_path_service().get_knowledge_bases_root().resolve():
+ return str(row["workspace_id"])
+ return None
+
+
def parse_kb_id(value: str) -> tuple[str, str] | None:
if value.startswith("account:kb:"):
return "", value[len("account:kb:") :]
diff --git a/tests/multi_user/test_kb_move.py b/tests/multi_user/test_kb_move.py
index a684353ff..fef8dee75 100644
--- a/tests/multi_user/test_kb_move.py
+++ b/tests/multi_user/test_kb_move.py
@@ -2,9 +2,12 @@
from __future__ import annotations
+from io import BytesIO
import json
+from pathlib import Path
+from types import SimpleNamespace
-from fastapi import BackgroundTasks
+from fastapi import BackgroundTasks, UploadFile
import pytest
from deeptutor.multi_user.knowledge_access import (
@@ -72,8 +75,11 @@ def test_move_one_kb_into_nonempty_workspace_preserves_bytes_and_references(as_u
assert "atlas" not in current_kb_manager().list_knowledge_bases()
assert current_kb_manager()._load_config()["defaults"]["default_kb"] is None
assert resolve_kb(old_id).id == new_id
+ assert resolve_kb("user:kb:atlas").id == new_id
assert resolve_kb("atlas").id == new_id
- moved_in_account = next(row for row in list_visible_knowledge_bases() if row["id"] == new_id)
+ moved_in_account = next(
+ row for row in list_visible_knowledge_bases() if row["id"] == new_id
+ )
assert moved_in_account["provenance_label"] == "Research"
with workspace_context(consumer["workspace_id"]):
assert [row["id"] for row in list_visible_knowledge_bases()] == [new_id]
@@ -139,6 +145,219 @@ def test_failed_publish_restores_source_destination_and_assignment(as_user, monk
assert resolve_kb(old_id).base_dir == source_config.parent
+def test_legacy_account_selection_redirect_is_private_to_owner(as_user):
+ from fastapi import HTTPException
+
+ with as_user("alice"):
+ _make_kb("atlas", b"source")
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+ move_kb(qualified_kb_id("atlas"), destination)
+ assert resolve_kb("user:kb:atlas").id == qualified_kb_id("atlas", destination)
+ with as_user("bob"):
+ with pytest.raises(HTTPException):
+ resolve_kb("user:kb:atlas")
+
+
+def test_legacy_admin_selection_redirects_only_in_admin_account(as_user):
+ with as_user("operator", role="admin"):
+ _make_kb("atlas", b"source")
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+ move_kb(qualified_kb_id("atlas"), destination)
+ assert resolve_kb("admin:kb:atlas").id == qualified_kb_id("atlas", destination)
+
+
+def test_owner_can_move_kb_excluded_from_general_chat_selection(as_user):
+ with as_user("alice"):
+ _make_kb("selected", b"other")
+ _make_kb("atlas", b"source")
+ service = ContentWorkspaceService()
+ service.update_workspace(
+ service._builtin_id("general"),
+ resources={"knowledge_bases": [qualified_kb_id("selected")]},
+ )
+ destination = service.create_workspace("Research")["workspace_id"]
+ assert preview_kb_move(qualified_kb_id("atlas"), destination)["blockers"] == []
+ move_kb(qualified_kb_id("atlas"), destination)
+ with workspace_context(destination):
+ assert "atlas" in current_kb_manager().list_knowledge_bases()
+
+
+def test_moved_away_name_cannot_be_registered_by_connected_providers(as_user, tmp_path):
+ with as_user("alice"):
+ _make_kb("atlas", b"source")
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+ move_kb(qualified_kb_id("atlas"), destination)
+ manager = current_kb_manager()
+ attempts = [
+ lambda: manager.register_connected_entry("atlas", {"type": "obsidian"}),
+ lambda: manager.register_obsidian_vault("atlas", str(tmp_path)),
+ lambda: manager.register_linked_kb("atlas", str(tmp_path), "llamaindex"),
+ lambda: manager.register_lightrag_server_kb("atlas", "http://localhost:9621"),
+ lambda: manager.register_marginnote4_kb("atlas"),
+ lambda: manager.register_ima_kb("atlas", "", "", "remote-id"),
+ lambda: manager.register_weknora_kb("atlas", "http://localhost", "key", "remote-id"),
+ lambda: manager.register_subagent_connection("atlas", "local"),
+ ]
+ for attempt in attempts:
+ with pytest.raises(ValueError, match="reserved by an earlier move"):
+ attempt()
+ assert "atlas" not in manager._load_config()["knowledge_bases"]
+
+ second_destination = ContentWorkspaceService().create_workspace("Archive")[
+ "workspace_id"
+ ]
+ move_kb(qualified_kb_id("atlas", destination), second_destination)
+ with workspace_context(destination):
+ with pytest.raises(ValueError, match="reserved by an earlier move"):
+ current_kb_manager().register_connected_entry("atlas", {"type": "obsidian"})
+ assert resolve_kb("user:kb:atlas").id == qualified_kb_id("atlas", second_destination)
+
+
+def test_marginnote_move_is_blocked_before_synced_database_is_lost(as_user):
+ from deeptutor.capabilities.marginnote4.store import default_db_path
+
+ with as_user("alice"):
+ current_kb_manager().register_connected_entry("notes", {"type": "marginnote4"})
+ database = default_db_path("notes")
+ database.parent.mkdir(parents=True, exist_ok=True)
+ database.write_bytes(b"synced-notes")
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+ source_id = qualified_kb_id("notes")
+ assert "MarginNote 4" in preview_kb_move(source_id, destination)["blockers"][0]
+ with pytest.raises(WorkspaceError, match="MarginNote 4"):
+ move_kb(source_id, destination)
+ assert database.read_bytes() == b"synced-notes"
+ assert "notes" in current_kb_manager().list_knowledge_bases()
+ with workspace_context(destination):
+ assert "notes" not in current_kb_manager().list_knowledge_bases()
+
+
+def test_moved_llamaindex_retrieval_keeps_citations_and_images(as_user, monkeypatch):
+ from llama_index.core import Settings, VectorStoreIndex
+ from llama_index.core.embeddings import MockEmbedding
+ from llama_index.core.schema import ImageNode, TextNode
+
+ from deeptutor.services.path_service import get_path_service
+ from deeptutor.services.rag.pipelines.llamaindex import storage
+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline
+
+ monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=8))
+ with as_user("alice"):
+ _make_kb("atlas", b"source text")
+ source_root = current_kb_manager().base_dir / "atlas"
+ source_file = source_root / "raw" / "source.pdf"
+ cache_image = get_path_service().get_parse_cache_root() / "aa" / "image.png"
+ cache_image.parent.mkdir(parents=True)
+ cache_image.write_bytes(b"image bytes")
+ text_node = TextNode(
+ text="source alpha", id_="text-1",
+ metadata={"file_name": "source.pdf", "file_path": str(source_file)},
+ embedding=[0.5] * 8,
+ )
+ image_node = ImageNode(
+ text="[Image] source.pdf", id_="image-1", image_path=str(cache_image),
+ metadata={"file_name": "source.pdf", "file_path": str(source_file)},
+ embedding=[0.5] * 8,
+ )
+ index = VectorStoreIndex(nodes=[text_node, image_node])
+ index.storage_context.persist(persist_dir=str(source_root / "version-1"))
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+
+ move_kb(qualified_kb_id("atlas"), destination)
+ assert not source_root.exists()
+ cache_image.unlink()
+
+ with workspace_context(destination):
+ target_root = current_kb_manager().base_dir / "atlas"
+ nodes = storage.retrieve_nodes(target_root / "version-1", "alpha", top_k=2)
+ assert {node.node.node_id for node in nodes} == {"text-1", "image-1"}
+ result = LlamaIndexPipeline._nodes_to_result(None, "alpha", nodes)
+ assert {source["source"] for source in result["sources"]} == {
+ str(target_root / "raw" / "source.pdf")
+ }
+ assert Path(result["sources"][0]["source"]).is_file()
+ moved_image = next(node.node for node in nodes if node.node.node_id == "image-1")
+ assert Path(moved_image.image_path).is_file()
+ assert Path(moved_image.image_path).read_bytes() == b"image bytes"
+ assert Path(moved_image.image_path).is_relative_to(target_root)
+
+
+async def test_moved_lightrag_search_opens_published_native_workspace(as_user, monkeypatch):
+ from deeptutor.services.rag.pipelines.lightrag import engine, pipeline, storage
+
+ with as_user("alice"):
+ _make_kb("atlas", b"source")
+ source_root = current_kb_manager().base_dir / "atlas"
+ source_version = source_root / "version-1"
+ source_workspace = engine.workspace_for(source_version)
+ native_store = source_version / source_workspace
+ native_store.mkdir()
+ (native_store / "kv_store_doc_status.json").write_text(
+ json.dumps({"doc": {"status": "processed", "chunks_list": ["chunk"]}})
+ )
+ (native_store / "kv_store_text_chunks.json").write_text(
+ json.dumps({"chunk": {"content": "grounded passage"}})
+ )
+ (source_version / "meta.json").write_text(
+ json.dumps({
+ "version": "version-1", "signature": "lightrag", "provider": "lightrag",
+ "state": "published", "lightrag_adapter_schema": 2,
+ "parser_bridge_schema": 1, "workspace": source_workspace,
+ })
+ )
+ manager = current_kb_manager()
+ manager.config = manager._load_config()
+ manager.config["knowledge_bases"]["atlas"]["rag_provider"] = "lightrag"
+ manager._save_config()
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+ move_kb(qualified_kb_id("atlas"), destination)
+
+ with workspace_context(destination):
+ target_root = current_kb_manager().base_dir / "atlas"
+ target_version = target_root / "version-1"
+ assert not source_root.exists()
+ assert engine.workspace_for(target_version) == source_workspace
+ assert storage.latest_published_root(target_root) == target_version
+
+ async def in_worker(job):
+ return await job(None)
+
+ async def no_op(*_args, **_kwargs):
+ return None
+
+ async def fake_query(rag, *_args):
+ payload = json.loads(
+ (rag.working_dir / rag.workspace / "kv_store_text_chunks.json").read_text()
+ )
+ return payload["chunk"]["content"], [{"file_path": "source.pdf"}]
+
+ def fake_build_rag(working_dir, **_kwargs):
+ return SimpleNamespace(
+ working_dir=Path(working_dir),
+ workspace=engine.workspace_for(working_dir),
+ )
+
+ monkeypatch.setattr(pipeline, "run_in_worker_loop", in_worker)
+ monkeypatch.setattr(engine, "build_rag", fake_build_rag)
+ monkeypatch.setattr(engine, "initialize", no_op)
+ monkeypatch.setattr(engine, "finalize", no_op)
+ monkeypatch.setattr(engine, "query_with_sources", fake_query)
+ monkeypatch.setattr(storage, "require_compatible_embedding", lambda *_args: None)
+ monkeypatch.setattr(
+ "deeptutor.services.rag.pipelines.lightrag.roles.resolve_query_roles",
+ lambda: {},
+ )
+ monkeypatch.setattr(
+ "deeptutor.services.embedding.get_embedding_config", lambda: object()
+ )
+ rag_pipeline = pipeline.LightRagPipeline(kb_base_dir=str(target_root.parent))
+ monkeypatch.setattr(rag_pipeline, "_ensure_available", lambda: None)
+ result = await rag_pipeline.search("alpha", "atlas")
+ assert result["content"] == "grounded passage"
+ assert result["sources"] == [{"file_path": "source.pdf"}]
+
+
@pytest.mark.asyncio
async def test_create_destination_overrides_library_scope(as_user, monkeypatch):
from deeptutor.services import config
@@ -175,3 +394,121 @@ async def test_create_destination_overrides_library_scope(as_user, monkeypatch):
with workspace_context(destination):
assert seen == [current_kb_manager().base_dir]
assert result["id"] == qualified_kb_id("new", destination)
+
+
+async def test_uploaded_create_background_task_keeps_destination_scope(as_user, monkeypatch):
+ from deeptutor.services import config
+
+ monkeypatch.setattr(config, "load_config_with_main", lambda *_args: {})
+ from deeptutor.api.routers import knowledge
+ from deeptutor.knowledge.initializer import KnowledgeBaseInitializer
+ from deeptutor.services.config import get_kb_config_service
+ from deeptutor.services.workspace.knowledge import library_request
+
+ with as_user("alice"):
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+ account_root = current_kb_manager().base_dir
+ seen = []
+
+ async def fake_process(self):
+ seen.append(knowledge._current_kb_base_dir())
+ get_kb_config_service().set_kb_config(self.kb_name, {"indexed_in_scope": True})
+
+ monkeypatch.setattr(KnowledgeBaseInitializer, "process_documents", fake_process)
+ monkeypatch.setattr(knowledge, "_assert_provider_ready", lambda *_args: None)
+ background = BackgroundTasks()
+ token = library_request.set(True)
+ try:
+ result = await knowledge.create_knowledge_base(
+ background,
+ name="atlas",
+ files=[UploadFile(file=BytesIO(b"grounded text"), filename="source.txt")],
+ rag_provider="llamaindex",
+ pageindex_mode="",
+ search_mode="",
+ rel_paths=[],
+ indexing_llm="",
+ embedding_model="",
+ storage_workspace_id=destination,
+ )
+ finally:
+ library_request.reset(token)
+
+ with workspace_context(destination):
+ target_root = current_kb_manager().base_dir
+ assert (
+ current_kb_manager()._load_config()["knowledge_bases"]["atlas"]["status"]
+ == "processing"
+ )
+ assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]
+ assert result["task_id"]
+
+ # Execute Starlette's real callback after the route's scope has gone.
+ await background()
+
+ assert seen == [target_root]
+ with workspace_context(destination):
+ entry = current_kb_manager()._load_config()["knowledge_bases"]["atlas"]
+ assert entry["status"] == "ready"
+ assert entry["indexed_in_scope"] is True
+ assert (target_root / "atlas" / "raw" / "source.txt").read_bytes() == b"grounded text"
+ assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]
+ assert not (account_root / "atlas").exists()
+
+
+async def test_upload_to_moved_kb_background_task_uses_its_storage_scope(as_user, monkeypatch):
+ from deeptutor.services import config
+
+ monkeypatch.setattr(config, "load_config_with_main", lambda *_args: {})
+ from deeptutor.api.routers import knowledge
+ from deeptutor.services.config import get_kb_config_service
+ from deeptutor.services.workspace.knowledge import library_request
+
+ with as_user("alice"):
+ _make_kb("atlas", b"original")
+ destination = ContentWorkspaceService().create_workspace("Research")["workspace_id"]
+ move_kb(qualified_kb_id("atlas"), destination)
+ account_root = current_kb_manager().base_dir
+ seen = []
+
+ class FakeAdder:
+ def __init__(self, **_kwargs):
+ pass
+
+ def add_documents(self, paths, **_kwargs):
+ return paths
+
+ async def process_new_documents(self, paths):
+ return SimpleNamespace(processed_count=len(paths), has_failures=False)
+
+ def update_metadata(self, _count):
+ seen.append(knowledge._current_kb_base_dir())
+ get_kb_config_service().set_kb_config("atlas", {"uploaded_in_scope": True})
+
+ monkeypatch.setattr(knowledge, "DocumentAdder", FakeAdder)
+ monkeypatch.setattr(knowledge, "_assert_provider_ready", lambda *_args: None)
+ monkeypatch.setattr(knowledge, "_freeze_append_indexing", lambda *_args: None)
+ background = BackgroundTasks()
+ token = library_request.set(True)
+ try:
+ result = await knowledge.upload_files(
+ qualified_kb_id("atlas", destination), background,
+ files=[UploadFile(file=BytesIO(b"second source"), filename="new.txt")],
+ rag_provider=None, rel_paths=[], dest_subdir=None, image_analysis=None,
+ )
+ finally:
+ library_request.reset(token)
+ assert result["task_id"]
+ await background()
+
+ with workspace_context(destination):
+ target_root = current_kb_manager().base_dir
+ assert seen == [target_root]
+ assert current_kb_manager()._load_config()["knowledge_bases"]["atlas"][
+ "uploaded_in_scope"
+ ] is True
+ assert (target_root / "atlas" / "raw" / "new.txt").read_bytes() == b"second source"
+ assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]
+ assert not (account_root / "atlas").exists()