DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
Preserve linked-folder sync snapshots and empty KB bootstrap
Bingxi Zhao (Frank)777babc382c5
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex c2e4d6b17..44ed98f47 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -1130,6 +1130,17 @@ async def run_upload_processing_task(with capture_task_logs(task_id):try:+ # Snapshot before staging: changes made while indexing must still+ # appear as modified on the next linked-folder sync.+ source_mtimes = {}+ if folder_id:+ for source_path in uploaded_file_paths:+ try:+ source_mtimes[source_path] = datetime.fromtimestamp(+ Path(source_path).stat().st_mtime+ ).isoformat()+ except OSError:+ pass_task_log(task_id, f"Processing {len(uploaded_file_paths)} file(s) for KB '{kb_name}'")progress_tracker.update(ProgressStage.PROCESSING_DOCUMENTS,@@ -1172,7 +1183,9 @@ async def run_upload_processing_task(if folder_id:try:manager = get_kb_manager()- manager.update_folder_sync_state(kb_name, folder_id, uploaded_file_paths)+ manager.update_folder_sync_state(+ kb_name, folder_id, uploaded_file_paths, source_mtimes+ )_task_log(task_id, f"Updated folder sync state: {folder_id}")except Exception as sync_err:_task_log(@@ -1248,7 +1261,9 @@ async def run_upload_processing_task(# Folder change detection keys state by the original# source paths, so persist the complete successful input# batch instead of the internal staging paths.- manager.update_folder_sync_state(kb_name, folder_id, uploaded_file_paths)+ manager.update_folder_sync_state(+ kb_name, folder_id, uploaded_file_paths, source_mtimes+ )_task_log(task_id, f"Updated folder sync state: {folder_id}")except Exception as sync_err:_task_log(diff --git a/deeptutor/knowledge/add_documents.py b/deeptutor/knowledge/add_documents.pyindex ddc846715..ac2feb280 100644--- a/deeptutor/knowledge/add_documents.py+++ b/deeptutor/knowledge/add_documents.py@@ -204,11 +204,22 @@ class DocumentAdder:# Both pipelines create their first index on add; existing broken versions# still require reindex instead of being silently replaced (#1458).- allows_bootstrap = self.rag_provider in {- DEFAULT_PROVIDER,- LIGHTRAG_PROVIDER,- } and not list_kb_versions(self.kb_dir)+ versions = list_kb_versions(self.kb_dir)+ allows_bootstrap = (+ self.rag_provider+ in {+ DEFAULT_PROVIDER,+ LIGHTRAG_PROVIDER,+ }+ and not versions+ )if not has_provider_index and not allows_bootstrap:+ if versions:+ summary = provider_failure_summary(self.kb_dir, self.rag_provider)+ raise ValueError(+ f"Knowledge base has no ready {self.rag_provider} index; reindex required: "+ f"{summary or 'stored index version is incomplete'}"+ )raise ValueError(f"Knowledge base not initialized ({self.rag_provider}): {kb_name}")self.accepted_indexing_snapshot = accepted_indexing_snapshotdiff --git a/deeptutor/knowledge/manager.py b/deeptutor/knowledge/manager.pyindex 97cacc84b..e2d8f124d 100644--- a/deeptutor/knowledge/manager.py+++ b/deeptutor/knowledge/manager.py@@ -2003,7 +2003,13 @@ class KnowledgeBaseManager:"modified_count": len(modified_files),}- def update_folder_sync_state(self, kb_name: str, folder_id: str, synced_files: list[str]):+ def update_folder_sync_state(+ self,+ kb_name: str,+ folder_id: str,+ synced_files: list[str],+ source_mtimes: dict[str, str] | None = None,+ ):"""Update the sync state for a linked folder after successful sync.@@ -2014,6 +2020,7 @@ class KnowledgeBaseManager:kb_name: Knowledge base namefolder_id: Folder IDsynced_files: List of file paths that were successfully synced+ source_mtimes: Modification times captured before source staging."""if kb_name not in self.list_knowledge_bases():raise ValueError(f"Knowledge base not found: {kb_name}")@@ -2041,10 +2048,14 @@ class KnowledgeBaseManager:file_states = folder.get("synced_files", {})for file_path in synced_files:try:- p = Path(file_path)- if p.exists():- mtime = datetime.fromtimestamp(p.stat().st_mtime)- file_states[file_path] = mtime.isoformat()+ if source_mtimes is not None:+ if file_path in source_mtimes:+ file_states[file_path] = source_mtimes[file_path]+ else:+ p = Path(file_path)+ if p.exists():+ mtime = datetime.fromtimestamp(p.stat().st_mtime)+ file_states[file_path] = mtime.isoformat()except Exception:passdiff --git a/deeptutor/services/rag/index_probe.py b/deeptutor/services/rag/index_probe.pyindex c4c860279..c55458e4b 100644--- a/deeptutor/services/rag/index_probe.py+++ b/deeptutor/services/rag/index_probe.py@@ -121,12 +121,8 @@ def provider_failure_summary(bulk callers (e.g. ``KnowledgeBaseManager.get_info``) do not rescan andre-parse every index version just to collect failure text.- When no version entry exists at all (empty-KB create never wrote an index),- return an explicit diagnostic instead of an empty string."""entries = versions if versions is not None else inspect_kb_versions(kb_dir, provider)- if not entries:- return "No index version exists yet."failures: list[str] = []for entry in entries:summary = str(entry.get("failure_summary") or "").strip()diff --git a/deeptutor/services/setup/data_volume.py b/deeptutor/services/setup/data_volume.pyindex 49b718f5e..edf1d5328 100644--- a/deeptutor/services/setup/data_volume.py+++ b/deeptutor/services/setup/data_volume.py@@ -107,13 +107,6 @@ def ensure_data_volume_writable(target = Path(path)probe_uid = os.geteuid() if uid is None else uidprobe_gid = os.getegid() if gid is None else gid- try:- target.mkdir(parents=True, exist_ok=True)- except OSError as exc:- raise DataVolumePermissionError(- format_data_volume_permission_error(target, uid=probe_uid, gid=probe_gid, cause=exc)- ) from exc-drop_privs = (os.geteuid() == 0and probe_uid != 0@@ -141,10 +134,16 @@ def check_container_data_volume(data_root: Path | str | None = None) -> None:def _try_write_or_raise(path: Path, uid: int, gid: int) -> None:probe = path / f"{_PROBE_PREFIX}-{os.getpid()}"try:+ # Create as the process that will write the data. A root parent can+ # receive EACCES on a root-squashed NAS mount even when PUID can write.+ path.mkdir(parents=True, exist_ok=True)probe.write_text("ok", encoding="utf-8")probe.unlink(missing_ok=True)except OSError as exc:- probe.unlink(missing_ok=True)+ try:+ probe.unlink(missing_ok=True)+ except OSError:+ passraise DataVolumePermissionError(format_data_volume_permission_error(path, uid=uid, gid=gid, cause=exc)) from exc@@ -163,6 +162,4 @@ def _ensure_writable_as(path: Path, uid: int, gid: int) -> None:os._exit(1)_, status = os.waitpid(pid, 0)if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:- raise DataVolumePermissionError(- format_data_volume_permission_error(path, uid=uid, gid=gid)- )+ raise DataVolumePermissionError(format_data_volume_permission_error(path, uid=uid, gid=gid))diff --git a/tests/api/test_linked_folder_routes.py b/tests/api/test_linked_folder_routes.pyindex d34b396d8..60703fd14 100644--- a/tests/api/test_linked_folder_routes.py+++ b/tests/api/test_linked_folder_routes.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+from datetime import datetimefrom pathlib import Pathfrom types import SimpleNamespace@@ -49,6 +50,7 @@ class _FolderManager:"last_sync": None,}self.update_calls: list[tuple[str, str, list[str]]] = []+ self.mtime_calls: list[dict[str, str] | None] = []self.detect_result = {"new_files": [],"modified_files": [],@@ -77,8 +79,15 @@ class _FolderManager:def detect_folder_changes(self, _kb_name: str, _folder_id: str) -> dict:return self.detect_result- def update_folder_sync_state(self, kb_name: str, folder_id: str, files: list[str]) -> None:+ def update_folder_sync_state(+ self,+ kb_name: str,+ folder_id: str,+ files: list[str],+ source_mtimes: dict[str, str] | None = None,+ ) -> None:self.update_calls.append((kb_name, folder_id, files))+ self.mtime_calls.append(source_mtimes)def update_kb_status(self, name: str, status: str, progress: dict | None = None) -> None:self.config["knowledge_bases"][name]["status"] = status@@ -223,6 +232,8 @@ def test_completed_folder_task_records_source_paths(_patch_manager(monkeypatch, manager)source_root = tmp_path / "notes"source_path = source_root / "nested" / "note.md"+ source_path.parent.mkdir(parents=True)+ source_path.write_text("note", encoding="utf-8")staged_path = tmp_path / "kb" / "raw" / "nested" / "note.md"class _SuccessfulAdder:@@ -257,6 +268,9 @@ def test_completed_folder_task_records_source_paths()assert manager.update_calls == [("kb", "folder-1", [str(source_path)])]+ assert manager.mtime_calls == [+ {str(source_path): datetime.fromtimestamp(source_path.stat().st_mtime).isoformat()}+ ]def test_empty_completed_folder_task_advances_sync_state(@@ -265,6 +279,8 @@ def test_empty_completed_folder_task_advances_sync_state(manager = _FolderManager(tmp_path)_patch_manager(monkeypatch, manager)source_path = tmp_path / "notes" / "note.md"+ source_path.parent.mkdir(parents=True)+ source_path.write_text("note", encoding="utf-8")class _EmptyAdder:def __init__(self, *_args, **_kwargs) -> None:@@ -291,6 +307,9 @@ def test_empty_completed_folder_task_advances_sync_state()assert manager.update_calls == [("kb", "folder-1", [str(source_path)])]+ assert manager.mtime_calls == [+ {str(source_path): datetime.fromtimestamp(source_path.stat().st_mtime).isoformat()}+ ]def test_failed_folder_task_does_not_advance_sync_state(diff --git a/tests/knowledge/test_document_adder_provider.py b/tests/knowledge/test_document_adder_provider.pyindex 642e9dd20..041753c39 100644--- a/tests/knowledge/test_document_adder_provider.py+++ b/tests/knowledge/test_document_adder_provider.py@@ -9,6 +9,7 @@ import pytestfrom deeptutor.knowledge.add_documents import (DocumentAdder,RawDocumentRemoval,+ add_documents,remove_raw_document,)@@ -135,7 +136,7 @@ def test_document_adder_rejects_unready_existing_version(tmp_path: Path) -> Noneencoding="utf-8",)- with pytest.raises(ValueError, match="not initialized"):+ with pytest.raises(ValueError, match="reindex required"):DocumentAdder(kb_name="kb", base_dir=str(tmp_path), rag_provider="llamaindex")@@ -161,6 +162,43 @@ def test_document_adder_allows_empty_llamaindex_kb_to_bootstrap(tmp_path: Path)assert adder.raw_dir.is_dir()[email protected]("provider", ["graphrag", "pageindex", "pageindex-oss"])+def test_empty_provider_kb_bootstraps_on_first_upload(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str+) -> None:+ from deeptutor.knowledge.manager import KnowledgeBaseManager++ kb_dir = tmp_path / "empty"+ (kb_dir / "raw").mkdir(parents=True)+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))+ manager.register_knowledge_base("empty")+ manager.config["knowledge_bases"]["empty"]["rag_provider"] = provider+ manager._save_config()+ manager.update_kb_status(name="empty", status="ready")+ assert manager.get_info("empty")["status"] == "ready"++ doc = tmp_path / "lesson.md"+ doc.write_text("Lesson", encoding="utf-8")+ initialized: list[tuple[str, list[str]]] = []++ class FakeRagService:+ def __init__(self, *, kb_base_dir: str) -> None:+ assert kb_base_dir == str(tmp_path)++ async def initialize(self, *, kb_name: str, file_paths: list[str]) -> bool:+ initialized.append((kb_name, file_paths))+ return True++ def _resolve_provider(self, _kb_name: str) -> str:+ return provider++ monkeypatch.setattr("deeptutor.knowledge.add_documents.RAGService", FakeRagService)++ assert asyncio.run(add_documents("empty", [str(doc)], base_dir=str(tmp_path))) == 1+ assert initialized == [("empty", [str(doc)])]+ assert KnowledgeBaseManager(base_dir=str(tmp_path)).get_info("empty")["status"] == "ready"++def test_document_adder_unready_index_reports_probe_summary(tmp_path: Path) -> None:kb_dir = tmp_path / "kb"(kb_dir / "raw").mkdir(parents=True)diff --git a/tests/knowledge/test_linked_folder_sync.py b/tests/knowledge/test_linked_folder_sync.pyindex 09f8a55f4..14221b004 100644--- a/tests/knowledge/test_linked_folder_sync.py+++ b/tests/knowledge/test_linked_folder_sync.py@@ -64,6 +64,15 @@ def test_update_folder_sync_state_unknown_folder_writes_nothing(tmp_path: Path)assert metadata_file.read_bytes() == before+def test_sync_snapshot_preserves_change_made_during_indexing(tmp_path: Path) -> None:+ manager, _metadata_file, folder_id, doc = _manager_with_linked_folder(tmp_path)+ staged_mtime = "2026-09-01T10:00:00"+ manager.update_folder_sync_state("kb", folder_id, [str(doc)], {str(doc): staged_mtime})++ changes = manager.detect_folder_changes("kb", folder_id)+ assert changes["modified_files"] == [str(doc)]++def test_empty_successful_sync_records_last_sync_without_changing_file_state(tmp_path: Path,) -> None:diff --git a/tests/knowledge/test_manager_get_info_status.py b/tests/knowledge/test_manager_get_info_status.pyindex a4cdc9244..2abd62975 100644--- a/tests/knowledge/test_manager_get_info_status.py+++ b/tests/knowledge/test_manager_get_info_status.py@@ -95,6 +95,16 @@ def test_processing_with_ready_index_promotes_to_ready(assert info["statistics"]["rag_initialized"] is True+def test_fresh_empty_kb_stays_ready_without_an_index_version(tmp_path: Path) -> None:+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))+ manager.update_kb_status(name="empty", status="ready", progress=None)++ info = manager.get_info("empty")++ assert info["status"] == "ready"+ assert info["statistics"]["rag_initialized"] is False++def test_processing_with_completed_progress_and_ready_index_promotes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/tests/services/rag/test_index_probe.py b/tests/services/rag/test_index_probe.pyindex bebcab4d8..ce711c7e1 100644--- a/tests/services/rag/test_index_probe.py+++ b/tests/services/rag/test_index_probe.py@@ -52,15 +52,13 @@ def test_llamaindex_requires_real_storage_files(tmp_path: Path) -> None:assert probe.doc_count == 1-def test_empty_kb_reports_no_index_version_exists(tmp_path: Path) -> None:+def test_empty_kb_has_no_failure_summary(tmp_path: Path) -> None:(tmp_path / "raw").mkdir()assert inspect_kb_versions(tmp_path, "llamaindex") == []assert has_ready_provider_index(tmp_path, "llamaindex") is False- assert provider_failure_summary(tmp_path, "llamaindex") == "No index version exists yet."- assert provider_failure_summary(tmp_path, "llamaindex", versions=[]) == (- "No index version exists yet."- )+ assert provider_failure_summary(tmp_path, "llamaindex") == ""+ assert provider_failure_summary(tmp_path, "llamaindex", versions=[]) == ""def test_kb_versions_overrule_fake_llamaindex_ready_marker(tmp_path: Path) -> None:diff --git a/tests/services/setup/test_data_volume.py b/tests/services/setup/test_data_volume.pyindex b0a40974f..2201ac3b3 100644--- a/tests/services/setup/test_data_volume.py+++ b/tests/services/setup/test_data_volume.py@@ -57,6 +57,24 @@ def test_ensure_writable_succeeds_on_owned_directory(tmp_path: Path) -> None:assert not any(target.glob(".deeptutor-write-probe-*"))+def test_root_probe_defers_mkdir_until_after_identity_drop(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ target = tmp_path / "mounted" / "knowledge_bases"+ monkeypatch.setattr("deeptutor.services.setup.data_volume.os.geteuid", lambda: 0)++ def create_as_runtime_user(path: Path, uid: int, gid: int) -> None:+ assert (uid, gid) == (1234, 1234)+ assert not path.exists()+ path.mkdir(parents=True)++ monkeypatch.setattr(+ "deeptutor.services.setup.data_volume._ensure_writable_as", create_as_runtime_user+ )+ ensure_data_volume_writable(target, uid=1234, gid=1234)+ assert target.is_dir()++def test_ensure_unwritable_directory_fail_fasts(tmp_path: Path) -> None:if hasattr(os, "geteuid") and os.geteuid() == 0:pytest.skip("root bypasses directory mode bits")@@ -70,9 +88,7 @@ def test_ensure_unwritable_directory_fail_fasts(tmp_path: Path) -> None:target.chmod(0o755)-def test_check_container_data_volume_probes_knowledge_bases(- tmp_path: Path, monkeypatch-) -> None:+def test_check_container_data_volume_probes_knowledge_bases(tmp_path: Path, monkeypatch) -> None:data_root = tmp_path / "app-data"monkeypatch.setenv("PUID", "1000")monkeypatch.setenv("PGID", "1000")