DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(knowledge): reuse catalog probes without blocking the API (#1711)
Bingxi Zhao (Frank)4d7b080fbeca
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex c92a3e9f8..5510f9134 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -1376,17 +1376,27 @@ async def run_upload_processing_task(@router.get("/knowledge-bases/health")async def health_check():- """Health check endpoint"""+ """Count registered KBs without constructing/probing the catalog (#1711)."""+ return await asyncio.to_thread(_knowledge_health)+++def _knowledge_health():try:- manager = get_kb_manager()- config_exists = manager.config_file.exists()- kb_count = len(manager.list_knowledge_bases())+ base_dir = current_kb_base_dir()+ config_file = base_dir / "kb_config.json"+ config_exists = config_file.exists()+ config = (+ json.loads(config_file.read_text(encoding="utf-8").strip() or "{}")+ if config_exists+ else {}+ )+ kb_count = len(config.get("knowledge_bases", {}))return {"status": "ok",- "config_file": str(manager.config_file),+ "config_file": str(config_file),"config_exists": config_exists,- "base_dir": str(manager.base_dir),- "base_dir_exists": manager.base_dir.exists(),+ "base_dir": str(base_dir),+ "base_dir_exists": base_dir.exists(),"knowledge_bases_count": kb_count,}except Exception as e:@@ -2765,6 +2775,11 @@ def _resource_knowledge_bases() -> list[KnowledgeBaseInfo]:@router.get("/knowledge-bases", response_model=list[KnowledgeBaseInfo])async def list_knowledge_bases():+ """Disk probes must not block the async worker or its other requests (#1711)."""+ return await asyncio.to_thread(_list_knowledge_bases)+++def _list_knowledge_bases():"""List all available knowledge bases with their details."""from deeptutor.services.workspace.context import current_workspace_idfrom deeptutor.services.workspace.knowledge import library_requestdiff --git a/deeptutor/knowledge/manager.py b/deeptutor/knowledge/manager.pyindex 58a698661..677512b64 100644--- a/deeptutor/knowledge/manager.py+++ b/deeptutor/knowledge/manager.py@@ -6,6 +6,7 @@ Manages multiple knowledge bases and provides utilities for accessing them."""from contextlib import contextmanager+from copy import deepcopyfrom datetime import datetime, timedeltaimport hashlibimport json@@ -15,6 +16,8 @@ from pathlib import Pathimport shutilimport statimport sys+from threading import RLock+import timefrom typing import Anyfrom urllib.parse import urlparse@@ -80,6 +83,10 @@ def _assert_move_id_available(base_dir: Path, name: str) -> None:# than the create handshake while still keeping multi-day zombies out._ORPHAN_PRUNE_GRACE_SECONDS = 60+# External/manual index changes are re-probed within this interval (#1711).+# Normal indexing writes kb_config.json, which invalidates the snapshot at once.+_CATALOG_CACHE_SECONDS = 2.0+def _entry_updated_after(kb_entry: dict | None, cutoff: datetime) -> bool:"""Return True when the entry's ``updated_at`` is strictly after ``cutoff``.@@ -324,6 +331,9 @@ class KnowledgeBaseManager:# Config file to track knowledge basesself.config_file = self.base_dir / "kb_config.json"+ self._catalog_lock = RLock()+ self._config_cache = None+ self._info_cache: dict[tuple, tuple] = {}self.config = self._load_config()# PocketBase sync — enabled when integrations.pocketbase_url is set.@@ -333,7 +343,42 @@ class KnowledgeBaseManager:self._pb_enabled = is_pocketbase_enabled()+ def _catalog_revision(self) -> tuple:+ from deeptutor.multi_user.context import get_current_user_or_none+ from deeptutor.services.rag.embedding_signature import signature_from_embedding_config+ from deeptutor.services.workspace.context import current_workspace_id++ try:+ stat_result = self.config_file.stat()+ revision = (stat_result.st_ino, stat_result.st_mtime_ns, stat_result.st_size)+ except FileNotFoundError:+ revision = None+ signature = signature_from_embedding_config()+ user = get_current_user_or_none()+ return (+ revision,+ signature.hash() if signature is not None else None,+ user.id if user is not None else None,+ current_workspace_id(),+ )+def _load_config(self) -> dict:+ # The registry is atomic; one stat detects writes by other processes.+ # Single-flight the expensive reconciliation on slow volumes (#1711).+ with self._catalog_lock:+ revision = self._catalog_revision()+ cached = self._config_cache+ if cached and cached[0] == revision and time.monotonic() < cached[1]:+ return deepcopy(cached[2])+ config = self._read_and_reconcile_config()+ self._config_cache = (+ self._catalog_revision(),+ time.monotonic() + _CATALOG_CACHE_SECONDS,+ deepcopy(config),+ )+ return config++ def _read_and_reconcile_config(self) -> dict:"""Load knowledge base configuration from the canonical kb_config.json file."""if self.config_file.exists():try:@@ -421,7 +466,10 @@ class KnowledgeBaseManager:ever see the previous or the new file — ``open(..., "w")`` used totruncate the config before the lock was even acquired."""- atomic_write_json(self.config_file, self.config)+ with self._catalog_lock:+ atomic_write_json(self.config_file, self.config)+ self._config_cache = None+ self._info_cache.clear()def _sync_kb_to_pb(self, name: str, kb_entry: dict) -> None:"""@@ -595,6 +643,10 @@ class KnowledgeBaseManager:}def list_knowledge_bases(self) -> list[str]:+ with self._catalog_lock:+ return self._list_knowledge_bases()++ def _list_knowledge_bases(self) -> list[str]:"""List all available knowledge bases.This method:@@ -1374,6 +1426,39 @@ class KnowledgeBaseManager:*,refresh_config: bool = True,default_name: str | None = None,+ ) -> dict:+ if refresh_config:+ return self._get_info(name, default_name=default_name)+ with self._catalog_lock:+ key = (name, default_name)+ revision = (+ self._catalog_revision(),+ json.dumps(+ self.config.get("knowledge_bases", {}).get(name or default_name, {}),+ sort_keys=True,+ default=str,+ ),+ )+ cached = self._info_cache.get(key)+ if cached and cached[0] == revision and time.monotonic() < cached[1]:+ return deepcopy(cached[2])+ info = self._get_info(name, refresh_config=False, default_name=default_name)+ # Keep only the current catalog's entries, including connected KBs.+ if len(self._info_cache) >= max(1, len(self.config.get("knowledge_bases", {})) * 2):+ self._info_cache.clear()+ self._info_cache[key] = (+ revision,+ time.monotonic() + _CATALOG_CACHE_SECONDS,+ deepcopy(info),+ )+ return info++ def _get_info(+ self,+ name: str | None = None,+ *,+ refresh_config: bool = True,+ default_name: str | None = None,) -> dict:"""Get detailed information about a knowledge base.diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.pyindex ec0c32963..f46cc58df 100644--- a/tests/api/test_knowledge_router.py+++ b/tests/api/test_knowledge_router.py@@ -3207,3 +3207,48 @@ def test_create_empty_llamaindex_kb_then_upload_and_retry(assert len(calls) == (3 if first_upload_fails else 2)asyncio.run(workflow())+++async def test_slow_catalog_read_leaves_event_loop_responsive(monkeypatch):+ """The single-worker API can serve other work during disk probes (#1711)."""+ import asyncio+ from contextvars import ContextVar+ from threading import Event++ started, release = Event(), Event()+ request_scope = ContextVar("test_catalog_scope", default="missing")++ def slow_list():+ assert request_scope.get() == "workspace-user"+ started.set()+ assert release.wait(timeout=2)+ return []++ monkeypatch.setattr(knowledge_router_module, "_list_knowledge_bases", slow_list)+ token = request_scope.set("workspace-user")+ task = asyncio.create_task(knowledge_router_module.list_knowledge_bases())+ try:+ await asyncio.wait_for(asyncio.to_thread(started.wait), timeout=1)+ # This await must run while the filesystem worker is still blocked.+ await asyncio.sleep(0)+ assert not task.done()+ finally:+ release.set()+ request_scope.reset(token)+ assert await task == []+++async def test_health_counts_registry_without_constructing_index_manager(tmp_path, monkeypatch):+ config = tmp_path / "kb_config.json"+ config.write_text(json.dumps({"knowledge_bases": {"one": {}, "two": {}}}))+ monkeypatch.setattr(knowledge_router_module, "current_kb_base_dir", lambda: tmp_path)++ def unexpected_manager():+ raise AssertionError("health must not initialize or probe indexes")++ monkeypatch.setattr(knowledge_router_module, "get_kb_manager", unexpected_manager)+ result = await knowledge_router_module.health_check()+ assert result["status"] == "ok"+ assert result["knowledge_bases_count"] == 2diff --git a/tests/knowledge/test_manager_list.py b/tests/knowledge/test_manager_list.pyindex 7425f2643..06fb4b8a9 100644--- a/tests/knowledge/test_manager_list.py+++ b/tests/knowledge/test_manager_list.py@@ -97,3 +97,54 @@ def test_auto_register_legacy_storage_marks_needs_reindex(tmp_path: Path) -> Nonentry = _read_config(manager.config_file)["knowledge_bases"]["legacy"]assert entry["status"] == "needs_reindex"assert entry["needs_reindex"] is True+++def test_repeated_catalog_reads_reuse_probes_and_return_independent_results(tmp_path, monkeypatch):+ """A slow index is probed once per snapshot, not per list request (#1711)."""+ from deeptutor.knowledge import manager as module++ manager = KnowledgeBaseManager(base_dir=str(tmp_path))+ _seed_kb(manager, "book")+ original = module.inspect_kb_versions+ probes = []++ def inspect(*args, **kwargs):+ probes.append(args)+ return original(*args, **kwargs)++ monkeypatch.setattr(module, "inspect_kb_versions", inspect)+ names = manager.list_knowledge_bases()+ info = manager.get_info("book", refresh_config=False, default_name=names[0])+ initial = len(probes)+ assert initial > 0+ info["statistics"]["index_versions"].clear()+ for _ in range(6):+ names = manager.list_knowledge_bases()+ fresh = manager.get_info("book", refresh_config=False, default_name=names[0])+ assert fresh["statistics"]["index_versions"]+ assert len(probes) == initial++ # A background process publishes status in the atomic registry.+ config = _read_config(manager.config_file)+ config["knowledge_bases"]["book"]["description"] = "updated elsewhere"+ manager.config_file.write_text(json.dumps(config), encoding="utf-8")+ manager.list_knowledge_bases()+ fresh = manager.get_info("book", refresh_config=False, default_name="book")+ assert fresh["metadata"]["description"] == "updated elsewhere"+ assert len(probes) > initial+++def test_catalog_rechecks_manual_index_changes_after_snapshot_expires(tmp_path, monkeypatch):+ from deeptutor.knowledge import manager as module++ now = [100.0]+ monkeypatch.setattr(module.time, "monotonic", lambda: now[0])+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))+ kb_dir = _seed_kb(manager, "book")+ manager.list_knowledge_bases()+ manager.get_info("book", refresh_config=False, default_name="book")+ shutil.rmtree(kb_dir / "version-1")+ now[0] += module._CATALOG_CACHE_SECONDS + 1+ manager.list_knowledge_bases()+ fresh = manager.get_info("book", refresh_config=False, default_name="book")+ assert fresh["statistics"]["index_versions"] == []