Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Preserve linked-folder sync snapshots and empty KB bootstrap

Bingxi Zhao (Frank)777babc382c5
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index c2e4d6b17..44ed98f47 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -1130,6 +1130,17 @@ async def run_upload_processing_task(
with capture_task_logs(task_id):
try:
+ # Snapshot before staging: changes made while indexing must still
+ # appear as modified on the next linked-folder sync.
+ source_mtimes = {}
+ if folder_id:
+ for source_path in uploaded_file_paths:
+ try:
+ source_mtimes[source_path] = datetime.fromtimestamp(
+ Path(source_path).stat().st_mtime
+ ).isoformat()
+ except OSError:
+ pass
_task_log(task_id, f"Processing {len(uploaded_file_paths)} file(s) for KB '{kb_name}'")
progress_tracker.update(
ProgressStage.PROCESSING_DOCUMENTS,
@@ -1172,7 +1183,9 @@ async def run_upload_processing_task(
if folder_id:
try:
manager = get_kb_manager()
- manager.update_folder_sync_state(kb_name, folder_id, uploaded_file_paths)
+ manager.update_folder_sync_state(
+ kb_name, folder_id, uploaded_file_paths, source_mtimes
+ )
_task_log(task_id, f"Updated folder sync state: {folder_id}")
except Exception as sync_err:
_task_log(
@@ -1248,7 +1261,9 @@ async def run_upload_processing_task(
# Folder change detection keys state by the original
# source paths, so persist the complete successful input
# batch instead of the internal staging paths.
- manager.update_folder_sync_state(kb_name, folder_id, uploaded_file_paths)
+ manager.update_folder_sync_state(
+ kb_name, folder_id, uploaded_file_paths, source_mtimes
+ )
_task_log(task_id, f"Updated folder sync state: {folder_id}")
except Exception as sync_err:
_task_log(
diff --git a/deeptutor/knowledge/add_documents.py b/deeptutor/knowledge/add_documents.py
index ddc846715..ac2feb280 100644
--- a/deeptutor/knowledge/add_documents.py
+++ b/deeptutor/knowledge/add_documents.py
@@ -204,11 +204,22 @@ class DocumentAdder:
# Both pipelines create their first index on add; existing broken versions
# still require reindex instead of being silently replaced (#1458).
- allows_bootstrap = self.rag_provider in {
- DEFAULT_PROVIDER,
- LIGHTRAG_PROVIDER,
- } and not list_kb_versions(self.kb_dir)
+ versions = list_kb_versions(self.kb_dir)
+ allows_bootstrap = (
+ self.rag_provider
+ in {
+ DEFAULT_PROVIDER,
+ LIGHTRAG_PROVIDER,
+ }
+ and not versions
+ )
if not has_provider_index and not allows_bootstrap:
+ if versions:
+ summary = provider_failure_summary(self.kb_dir, self.rag_provider)
+ raise ValueError(
+ f"Knowledge base has no ready {self.rag_provider} index; reindex required: "
+ f"{summary or 'stored index version is incomplete'}"
+ )
raise ValueError(f"Knowledge base not initialized ({self.rag_provider}): {kb_name}")
self.accepted_indexing_snapshot = accepted_indexing_snapshot
diff --git a/deeptutor/knowledge/manager.py b/deeptutor/knowledge/manager.py
index 97cacc84b..e2d8f124d 100644
--- a/deeptutor/knowledge/manager.py
+++ b/deeptutor/knowledge/manager.py
@@ -2003,7 +2003,13 @@ class KnowledgeBaseManager:
"modified_count": len(modified_files),
}
- def update_folder_sync_state(self, kb_name: str, folder_id: str, synced_files: list[str]):
+ def update_folder_sync_state(
+ self,
+ kb_name: str,
+ folder_id: str,
+ synced_files: list[str],
+ source_mtimes: dict[str, str] | None = None,
+ ):
"""
Update the sync state for a linked folder after successful sync.
@@ -2014,6 +2020,7 @@ class KnowledgeBaseManager:
kb_name: Knowledge base name
folder_id: Folder ID
synced_files: List of file paths that were successfully synced
+ source_mtimes: Modification times captured before source staging.
"""
if kb_name not in self.list_knowledge_bases():
raise ValueError(f"Knowledge base not found: {kb_name}")
@@ -2041,10 +2048,14 @@ class KnowledgeBaseManager:
file_states = folder.get("synced_files", {})
for file_path in synced_files:
try:
- p = Path(file_path)
- if p.exists():
- mtime = datetime.fromtimestamp(p.stat().st_mtime)
- file_states[file_path] = mtime.isoformat()
+ if source_mtimes is not None:
+ if file_path in source_mtimes:
+ file_states[file_path] = source_mtimes[file_path]
+ else:
+ p = Path(file_path)
+ if p.exists():
+ mtime = datetime.fromtimestamp(p.stat().st_mtime)
+ file_states[file_path] = mtime.isoformat()
except Exception:
pass
diff --git a/deeptutor/services/rag/index_probe.py b/deeptutor/services/rag/index_probe.py
index c4c860279..c55458e4b 100644
--- a/deeptutor/services/rag/index_probe.py
+++ b/deeptutor/services/rag/index_probe.py
@@ -121,12 +121,8 @@ def provider_failure_summary(
bulk callers (e.g. ``KnowledgeBaseManager.get_info``) do not rescan and
re-parse every index version just to collect failure text.
- When no version entry exists at all (empty-KB create never wrote an index),
- return an explicit diagnostic instead of an empty string.
"""
entries = versions if versions is not None else inspect_kb_versions(kb_dir, provider)
- if not entries:
- return "No index version exists yet."
failures: list[str] = []
for entry in entries:
summary = str(entry.get("failure_summary") or "").strip()
diff --git a/deeptutor/services/setup/data_volume.py b/deeptutor/services/setup/data_volume.py
index 49b718f5e..edf1d5328 100644
--- a/deeptutor/services/setup/data_volume.py
+++ b/deeptutor/services/setup/data_volume.py
@@ -107,13 +107,6 @@ def ensure_data_volume_writable(
target = Path(path)
probe_uid = os.geteuid() if uid is None else uid
probe_gid = os.getegid() if gid is None else gid
- try:
- target.mkdir(parents=True, exist_ok=True)
- except OSError as exc:
- raise DataVolumePermissionError(
- format_data_volume_permission_error(target, uid=probe_uid, gid=probe_gid, cause=exc)
- ) from exc
-
drop_privs = (
os.geteuid() == 0
and probe_uid != 0
@@ -141,10 +134,16 @@ def check_container_data_volume(data_root: Path | str | None = None) -> None:
def _try_write_or_raise(path: Path, uid: int, gid: int) -> None:
probe = path / f"{_PROBE_PREFIX}-{os.getpid()}"
try:
+ # Create as the process that will write the data. A root parent can
+ # receive EACCES on a root-squashed NAS mount even when PUID can write.
+ path.mkdir(parents=True, exist_ok=True)
probe.write_text("ok", encoding="utf-8")
probe.unlink(missing_ok=True)
except OSError as exc:
- probe.unlink(missing_ok=True)
+ try:
+ probe.unlink(missing_ok=True)
+ except OSError:
+ pass
raise DataVolumePermissionError(
format_data_volume_permission_error(path, uid=uid, gid=gid, cause=exc)
) from exc
@@ -163,6 +162,4 @@ def _ensure_writable_as(path: Path, uid: int, gid: int) -> None:
os._exit(1)
_, status = os.waitpid(pid, 0)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
- raise DataVolumePermissionError(
- format_data_volume_permission_error(path, uid=uid, gid=gid)
- )
+ raise DataVolumePermissionError(format_data_volume_permission_error(path, uid=uid, gid=gid))
diff --git a/tests/api/test_linked_folder_routes.py b/tests/api/test_linked_folder_routes.py
index d34b396d8..60703fd14 100644
--- a/tests/api/test_linked_folder_routes.py
+++ b/tests/api/test_linked_folder_routes.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+from datetime import datetime
from pathlib import Path
from types import SimpleNamespace
@@ -49,6 +50,7 @@ class _FolderManager:
"last_sync": None,
}
self.update_calls: list[tuple[str, str, list[str]]] = []
+ self.mtime_calls: list[dict[str, str] | None] = []
self.detect_result = {
"new_files": [],
"modified_files": [],
@@ -77,8 +79,15 @@ class _FolderManager:
def detect_folder_changes(self, _kb_name: str, _folder_id: str) -> dict:
return self.detect_result
- def update_folder_sync_state(self, kb_name: str, folder_id: str, files: list[str]) -> None:
+ def update_folder_sync_state(
+ self,
+ kb_name: str,
+ folder_id: str,
+ files: list[str],
+ source_mtimes: dict[str, str] | None = None,
+ ) -> None:
self.update_calls.append((kb_name, folder_id, files))
+ self.mtime_calls.append(source_mtimes)
def update_kb_status(self, name: str, status: str, progress: dict | None = None) -> None:
self.config["knowledge_bases"][name]["status"] = status
@@ -223,6 +232,8 @@ def test_completed_folder_task_records_source_paths(
_patch_manager(monkeypatch, manager)
source_root = tmp_path / "notes"
source_path = source_root / "nested" / "note.md"
+ source_path.parent.mkdir(parents=True)
+ source_path.write_text("note", encoding="utf-8")
staged_path = tmp_path / "kb" / "raw" / "nested" / "note.md"
class _SuccessfulAdder:
@@ -257,6 +268,9 @@ def test_completed_folder_task_records_source_paths(
)
assert manager.update_calls == [("kb", "folder-1", [str(source_path)])]
+ assert manager.mtime_calls == [
+ {str(source_path): datetime.fromtimestamp(source_path.stat().st_mtime).isoformat()}
+ ]
def test_empty_completed_folder_task_advances_sync_state(
@@ -265,6 +279,8 @@ def test_empty_completed_folder_task_advances_sync_state(
manager = _FolderManager(tmp_path)
_patch_manager(monkeypatch, manager)
source_path = tmp_path / "notes" / "note.md"
+ source_path.parent.mkdir(parents=True)
+ source_path.write_text("note", encoding="utf-8")
class _EmptyAdder:
def __init__(self, *_args, **_kwargs) -> None:
@@ -291,6 +307,9 @@ def test_empty_completed_folder_task_advances_sync_state(
)
assert manager.update_calls == [("kb", "folder-1", [str(source_path)])]
+ assert manager.mtime_calls == [
+ {str(source_path): datetime.fromtimestamp(source_path.stat().st_mtime).isoformat()}
+ ]
def test_failed_folder_task_does_not_advance_sync_state(
diff --git a/tests/knowledge/test_document_adder_provider.py b/tests/knowledge/test_document_adder_provider.py
index 642e9dd20..041753c39 100644
--- a/tests/knowledge/test_document_adder_provider.py
+++ b/tests/knowledge/test_document_adder_provider.py
@@ -9,6 +9,7 @@ import pytest
from deeptutor.knowledge.add_documents import (
DocumentAdder,
RawDocumentRemoval,
+ add_documents,
remove_raw_document,
)
@@ -135,7 +136,7 @@ def test_document_adder_rejects_unready_existing_version(tmp_path: Path) -> None
encoding="utf-8",
)
- with pytest.raises(ValueError, match="not initialized"):
+ with pytest.raises(ValueError, match="reindex required"):
DocumentAdder(kb_name="kb", base_dir=str(tmp_path), rag_provider="llamaindex")
@@ -161,6 +162,43 @@ def test_document_adder_allows_empty_llamaindex_kb_to_bootstrap(tmp_path: Path)
assert adder.raw_dir.is_dir()
[email protected]("provider", ["graphrag", "pageindex", "pageindex-oss"])
+def test_empty_provider_kb_bootstraps_on_first_upload(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str
+) -> None:
+ from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+ kb_dir = tmp_path / "empty"
+ (kb_dir / "raw").mkdir(parents=True)
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+ manager.register_knowledge_base("empty")
+ manager.config["knowledge_bases"]["empty"]["rag_provider"] = provider
+ manager._save_config()
+ manager.update_kb_status(name="empty", status="ready")
+ assert manager.get_info("empty")["status"] == "ready"
+
+ doc = tmp_path / "lesson.md"
+ doc.write_text("Lesson", encoding="utf-8")
+ initialized: list[tuple[str, list[str]]] = []
+
+ class FakeRagService:
+ def __init__(self, *, kb_base_dir: str) -> None:
+ assert kb_base_dir == str(tmp_path)
+
+ async def initialize(self, *, kb_name: str, file_paths: list[str]) -> bool:
+ initialized.append((kb_name, file_paths))
+ return True
+
+ def _resolve_provider(self, _kb_name: str) -> str:
+ return provider
+
+ monkeypatch.setattr("deeptutor.knowledge.add_documents.RAGService", FakeRagService)
+
+ assert asyncio.run(add_documents("empty", [str(doc)], base_dir=str(tmp_path))) == 1
+ assert initialized == [("empty", [str(doc)])]
+ assert KnowledgeBaseManager(base_dir=str(tmp_path)).get_info("empty")["status"] == "ready"
+
+
def test_document_adder_unready_index_reports_probe_summary(tmp_path: Path) -> None:
kb_dir = tmp_path / "kb"
(kb_dir / "raw").mkdir(parents=True)
diff --git a/tests/knowledge/test_linked_folder_sync.py b/tests/knowledge/test_linked_folder_sync.py
index 09f8a55f4..14221b004 100644
--- a/tests/knowledge/test_linked_folder_sync.py
+++ b/tests/knowledge/test_linked_folder_sync.py
@@ -64,6 +64,15 @@ def test_update_folder_sync_state_unknown_folder_writes_nothing(tmp_path: Path)
assert metadata_file.read_bytes() == before
+def test_sync_snapshot_preserves_change_made_during_indexing(tmp_path: Path) -> None:
+ manager, _metadata_file, folder_id, doc = _manager_with_linked_folder(tmp_path)
+ staged_mtime = "2026-09-01T10:00:00"
+ manager.update_folder_sync_state("kb", folder_id, [str(doc)], {str(doc): staged_mtime})
+
+ changes = manager.detect_folder_changes("kb", folder_id)
+ assert changes["modified_files"] == [str(doc)]
+
+
def test_empty_successful_sync_records_last_sync_without_changing_file_state(
tmp_path: Path,
) -> None:
diff --git a/tests/knowledge/test_manager_get_info_status.py b/tests/knowledge/test_manager_get_info_status.py
index a4cdc9244..2abd62975 100644
--- a/tests/knowledge/test_manager_get_info_status.py
+++ b/tests/knowledge/test_manager_get_info_status.py
@@ -95,6 +95,16 @@ def test_processing_with_ready_index_promotes_to_ready(
assert info["statistics"]["rag_initialized"] is True
+def test_fresh_empty_kb_stays_ready_without_an_index_version(tmp_path: Path) -> None:
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+ manager.update_kb_status(name="empty", status="ready", progress=None)
+
+ info = manager.get_info("empty")
+
+ assert info["status"] == "ready"
+ assert info["statistics"]["rag_initialized"] is False
+
+
def test_processing_with_completed_progress_and_ready_index_promotes(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
diff --git a/tests/services/rag/test_index_probe.py b/tests/services/rag/test_index_probe.py
index bebcab4d8..ce711c7e1 100644
--- a/tests/services/rag/test_index_probe.py
+++ b/tests/services/rag/test_index_probe.py
@@ -52,15 +52,13 @@ def test_llamaindex_requires_real_storage_files(tmp_path: Path) -> None:
assert probe.doc_count == 1
-def test_empty_kb_reports_no_index_version_exists(tmp_path: Path) -> None:
+def test_empty_kb_has_no_failure_summary(tmp_path: Path) -> None:
(tmp_path / "raw").mkdir()
assert inspect_kb_versions(tmp_path, "llamaindex") == []
assert has_ready_provider_index(tmp_path, "llamaindex") is False
- assert provider_failure_summary(tmp_path, "llamaindex") == "No index version exists yet."
- assert provider_failure_summary(tmp_path, "llamaindex", versions=[]) == (
- "No index version exists yet."
- )
+ assert provider_failure_summary(tmp_path, "llamaindex") == ""
+ assert provider_failure_summary(tmp_path, "llamaindex", versions=[]) == ""
def test_kb_versions_overrule_fake_llamaindex_ready_marker(tmp_path: Path) -> None:
diff --git a/tests/services/setup/test_data_volume.py b/tests/services/setup/test_data_volume.py
index b0a40974f..2201ac3b3 100644
--- a/tests/services/setup/test_data_volume.py
+++ b/tests/services/setup/test_data_volume.py
@@ -57,6 +57,24 @@ def test_ensure_writable_succeeds_on_owned_directory(tmp_path: Path) -> None:
assert not any(target.glob(".deeptutor-write-probe-*"))
+def test_root_probe_defers_mkdir_until_after_identity_drop(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ target = tmp_path / "mounted" / "knowledge_bases"
+ monkeypatch.setattr("deeptutor.services.setup.data_volume.os.geteuid", lambda: 0)
+
+ def create_as_runtime_user(path: Path, uid: int, gid: int) -> None:
+ assert (uid, gid) == (1234, 1234)
+ assert not path.exists()
+ path.mkdir(parents=True)
+
+ monkeypatch.setattr(
+ "deeptutor.services.setup.data_volume._ensure_writable_as", create_as_runtime_user
+ )
+ ensure_data_volume_writable(target, uid=1234, gid=1234)
+ assert target.is_dir()
+
+
def test_ensure_unwritable_directory_fail_fasts(tmp_path: Path) -> None:
if hasattr(os, "geteuid") and os.geteuid() == 0:
pytest.skip("root bypasses directory mode bits")
@@ -70,9 +88,7 @@ def test_ensure_unwritable_directory_fail_fasts(tmp_path: Path) -> None:
target.chmod(0o755)
-def test_check_container_data_volume_probes_knowledge_bases(
- tmp_path: Path, monkeypatch
-) -> None:
+def test_check_container_data_volume_probes_knowledge_bases(tmp_path: Path, monkeypatch) -> None:
data_root = tmp_path / "app-data"
monkeypatch.setenv("PUID", "1000")
monkeypatch.setenv("PGID", "1000")