DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
style: format knowledge source integrations
Bingxi Zhao (Frank)176257b1f8f7
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex 367fc10f5..10907877e 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -2575,10 +2575,7 @@ async def list_kiwix_catalog(except KiwixError as exc:raise HTTPException(status_code=400, detail=str(exc)) from excreturn {- "archives": [- {"zim_name": archive.zim_name, "title": archive.title}- for archive in archives- ]+ "archives": [{"zim_name": archive.zim_name, "title": archive.title} for archive in archives]}diff --git a/deeptutor/api/routers/reading.py b/deeptutor/api/routers/reading.pyindex 5a1f12ca5..b19380eb4 100644--- a/deeptutor/api/routers/reading.py+++ b/deeptutor/api/routers/reading.py@@ -608,9 +608,8 @@ async def import_zim_article(payload: ZimArticleImportRequest) -> dict[str, Any]material_id = hashlib.sha256(f"{resource.id}\0{path}".encode()).hexdigest()[:16]catalog = _catalog()store = ReadingStore(catalog.root)- source_url = (- f"{client.base_url}/content/{client.zim_name}/"- + "/".join(quote(segment, safe="") for segment in path.split("/"))+ source_url = f"{client.base_url}/content/{client.zim_name}/" + "/".join(+ quote(segment, safe="") for segment in path.split("/"))store.ingest_units(material_id,diff --git a/deeptutor/knowledge/kb_types.py b/deeptutor/knowledge/kb_types.pyindex be13893e4..e318606c9 100644--- a/deeptutor/knowledge/kb_types.py+++ b/deeptutor/knowledge/kb_types.py@@ -154,8 +154,8 @@ def supports_local_raw_files(entry: Any) -> bool:def external_root_of(entry: Any) -> str | None:"""Absolute path a connected KB points at, or ``None`` for ordinary KBs.- ``linked`` KBs store it under ``external_path``; ``obsidian`` vaults under-the older ``vault_path`` field. One accessor so callers don't care which.+ ``linked`` KBs store it under ``external_path``; ``obsidian`` vaults under+ the older ``vault_path`` field. One accessor so callers don't care which."""if not isinstance(entry, dict):return Nonediff --git a/deeptutor/services/rag/pipelines/kiwix/client.py b/deeptutor/services/rag/pipelines/kiwix/client.pyindex e714068e1..38460eea5 100644--- a/deeptutor/services/rag/pipelines/kiwix/client.py+++ b/deeptutor/services/rag/pipelines/kiwix/client.py@@ -22,7 +22,9 @@ MAX_ARTICLE_BYTES = 2_000_000MAX_ARTICLE_CHARS = 12_000MAX_TITLE_BYTES = 32_000_ZIM_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")-_BLOCKS = frozenset({"article", "br", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "tr"})+_BLOCKS = frozenset(+ {"article", "br", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "tr"}+)_SKIP = frozenset({"script", "style", "nav", "footer", "header", "noscript"})@@ -49,7 +51,9 @@ def normalize_base_url(value: str) -> str:or parsed.fragmentor any(part in {".", ".."} for part in unquote(parsed.path).split("/"))):- raise KiwixError("Enter a kiwix-serve HTTP(S) base URL without credentials or query parameters.")+ raise KiwixError(+ "Enter a kiwix-serve HTTP(S) base URL without credentials or query parameters."+ )return parsed.geturl().rstrip("/")@@ -100,7 +104,9 @@ def html_to_text(raw: str, *, max_chars: int = MAX_ARTICLE_CHARS) -> str:parser = _PlainText()parser.feed(raw)text = unescape("".join(parser.parts))- return re.sub(r"\n{3,}", "\n\n", "\n".join(" ".join(line.split()) for line in text.splitlines())).strip()[:max_chars]+ return re.sub(+ r"\n{3,}", "\n\n", "\n".join(" ".join(line.split()) for line in text.splitlines())+ ).strip()[:max_chars]@dataclass(frozen=True)@@ -152,7 +158,7 @@ def parse_catalog_xml(data: bytes, *, base_url: str) -> list[KiwixArchive]:continueif url.query or not url.path.startswith(prefix):continue- name = unquote(url.path[len(prefix):].split("/", 1)[0])+ name = unquote(url.path[len(prefix) :].split("/", 1)[0])breakif not name:name = (fields.get("name").text or "").strip() if fields.get("name") is not None else ""@@ -163,7 +169,11 @@ def parse_catalog_xml(data: bytes, *, base_url: str) -> list[KiwixArchive]:if name in seen:continueseen.add(name)- title = " ".join((fields.get("title").text or "").split()) if fields.get("title") is not None else ""+ title = (+ " ".join((fields.get("title").text or "").split())+ if fields.get("title") is not None+ else ""+ )archives.append(KiwixArchive(zim_name=name, title=title[:300] or name))if len(archives) >= 50:break@@ -189,18 +199,26 @@ def parse_search_xml(data: bytes, *, base_url: str, zim_name: str) -> list[Kiwixif parsed.query or not parsed.path.startswith(prefix):continuetry:- article_path = validate_article_path(parsed.path[len(prefix):])+ article_path = validate_article_path(parsed.path[len(prefix) :])except KiwixError:continuetitle = " ".join((item.findtext("title") or "").split())[:300]description = item.find("description")- snippet = " ".join("".join(description.itertext()).split())[:1000] if description is not None else ""- hits.append(KiwixHit(title=title or article_path, article_path=article_path, snippet=snippet))+ snippet = (+ " ".join("".join(description.itertext()).split())[:1000]+ if description is not None+ else ""+ )+ hits.append(+ KiwixHit(title=title or article_path, article_path=article_path, snippet=snippet)+ )return hitsclass KiwixClient:- def __init__(self, base_url: str, zim_name: str, *, transport: httpx.AsyncBaseTransport | None = None) -> None:+ def __init__(+ self, base_url: str, zim_name: str, *, transport: httpx.AsyncBaseTransport | None = None+ ) -> None:self.base_url = normalize_base_url(base_url)self.zim_name = validate_zim_name(zim_name)self.transport = transport@@ -223,10 +241,19 @@ class KiwixClient:if query.strip():params["q"] = query.strip()[:100]async with client._client() as http:- raw, _ = await client._get(http, "catalog/search", params=params, limit=MAX_CATALOG_BYTES)+ raw, _ = await client._get(+ http, "catalog/search", params=params, limit=MAX_CATALOG_BYTES+ )return parse_catalog_xml(raw, base_url=client.base_url)- async def _get(self, client: httpx.AsyncClient, path: str, *, params: dict[str, str | int] | None = None, limit: int) -> tuple[bytes, str]:+ async def _get(+ self,+ client: httpx.AsyncClient,+ path: str,+ *,+ params: dict[str, str | int] | None = None,+ limit: int,+ ) -> tuple[bytes, str]:url = f"{self.base_url}/{path.lstrip('/')}"try:async with client.stream("GET", url, params=params) as response:@@ -254,7 +281,12 @@ class KiwixClient:xml, _ = await self._get(client,"search",- params={"books.name": self.zim_name, "pattern": "a", "format": "xml", "pageLength": 1},+ params={+ "books.name": self.zim_name,+ "pattern": "a",+ "format": "xml",+ "pageLength": 1,+ },limit=MAX_SEARCH_BYTES,)parse_search_xml(xml, base_url=self.base_url, zim_name=self.zim_name)@@ -274,7 +306,12 @@ class KiwixClient:xml, _ = await self._get(client,"search",- params={"books.name": self.zim_name, "pattern": query[:500], "format": "xml", "pageLength": count},+ params={+ "books.name": self.zim_name,+ "pattern": query[:500],+ "format": "xml",+ "pageLength": count,+ },limit=MAX_SEARCH_BYTES,)hits = parse_search_xml(xml, base_url=self.base_url, zim_name=self.zim_name)[:count]@@ -318,7 +355,11 @@ class KiwixClient:if mime not in {"text/html", "application/xhtml+xml", "text/plain"}:raise KiwixError("The selected ZIM entry is not a readable article.")decoded = raw.decode("utf-8", errors="replace")- return html_to_text(decoded, max_chars=max_chars) if mime != "text/plain" else decoded.strip()[:max_chars]+ return (+ html_to_text(decoded, max_chars=max_chars)+ if mime != "text/plain"+ else decoded.strip()[:max_chars]+ )async def read_article(self, article_path: str) -> str:async with self._client() as client:diff --git a/deeptutor/services/rag/pipelines/kiwix/pipeline.py b/deeptutor/services/rag/pipelines/kiwix/pipeline.pyindex af09a7400..89307e715 100644--- a/deeptutor/services/rag/pipelines/kiwix/pipeline.py+++ b/deeptutor/services/rag/pipelines/kiwix/pipeline.py@@ -20,7 +20,9 @@ class KiwixPipeline:async def search(self, query: str, kb_name: str, **kwargs: Any) -> dict[str, Any]:entry = load_kb_config_entry(self.kb_base_dir, kb_name)if entry.get("type") != PROVIDER:- return self._error(query, "This knowledge base is not connected to Kiwix.", "not_configured")+ return self._error(+ query, "This knowledge base is not connected to Kiwix.", "not_configured"+ )try:client = self._client_factory(entry["server_url"], entry["zim_name"])hits = await client.search(query, top_k=kwargs.get("top_k") or 5)diff --git a/deeptutor/services/workspace/kb_move.py b/deeptutor/services/workspace/kb_move.pyindex 966969035..f5d33ec3d 100644--- a/deeptutor/services/workspace/kb_move.py+++ b/deeptutor/services/workspace/kb_move.py@@ -262,8 +262,10 @@ def _rebase_llamaindex_paths("An indexed image is outside the movable knowledge base and parse cache.") from excpath_parts = [image, *list(image.parents)[: len(image.parts) - len(cache_root.parts)]]- if resolved != image or not image.is_file() or any(- part.is_symlink() for part in path_parts+ if (+ resolved != image+ or not image.is_file()+ or any(part.is_symlink() for part in path_parts)):raise WorkspaceError("An indexed image cannot be copied safely.")digest = hashlib.sha256(value.encode("utf-8")).hexdigest()diff --git a/deeptutor_cli/kb.py b/deeptutor_cli/kb.pyindex 913e6b772..835da0e59 100644--- a/deeptutor_cli/kb.py+++ b/deeptutor_cli/kb.py@@ -194,7 +194,9 @@ def register(app: typer.Typer) -> None:try:client = KiwixClient(server_url, zim_name)title = asyncio.run(client.probe())- _get_kb_manager().register_kiwix_kb(name, client.base_url, client.zim_name, zim_title=title)+ _get_kb_manager().register_kiwix_kb(+ name, client.base_url, client.zim_name, zim_title=title+ )except Exception as exc:console.print(f"[red]Could not connect Kiwix archive: {exc}[/]")raise typer.Exit(code=1) from excdiff --git a/tests/multi_user/test_kb_move.py b/tests/multi_user/test_kb_move.pyindex fd1335630..9e20115dc 100644--- a/tests/multi_user/test_kb_move.py+++ b/tests/multi_user/test_kb_move.py@@ -233,9 +233,7 @@ def test_moved_away_name_cannot_be_registered_by_connected_providers(as_user, tmattempt()assert "atlas" not in manager._load_config()["knowledge_bases"]- second_destination = ContentWorkspaceService().create_workspace("Archive")[- "workspace_id"- ]+ second_destination = ContentWorkspaceService().create_workspace("Archive")["workspace_id"]move_kb(qualified_kb_id("atlas", destination), second_destination)with workspace_context(destination):with pytest.raises(ValueError, match="reserved by an earlier move"):@@ -280,12 +278,15 @@ def test_moved_llamaindex_retrieval_keeps_citations_and_images(as_user, monkeypacache_image.parent.mkdir(parents=True)cache_image.write_bytes(b"image bytes")text_node = TextNode(- text="source alpha", id_="text-1",+ text="source alpha",+ id_="text-1",metadata={"file_name": "source.pdf", "file_path": str(source_file)},embedding=[0.5] * 8,)image_node = ImageNode(- text="[Image] source.pdf", id_="image-1", image_path=str(cache_image),+ text="[Image] source.pdf",+ id_="image-1",+ image_path=str(cache_image),metadata={"file_name": "source.pdf", "file_path": str(source_file)},embedding=[0.5] * 8,)@@ -330,11 +331,17 @@ async def test_moved_lightrag_search_opens_published_native_workspace(as_user, mjson.dumps({"chunk": {"content": "grounded passage"}}))(source_version / "meta.json").write_text(- json.dumps({- "version": "version-1", "signature": "lightrag", "provider": "lightrag",- "state": "published", "lightrag_adapter_schema": 2,- "parser_bridge_schema": 1, "workspace": source_workspace,- })+ json.dumps(+ {+ "version": "version-1",+ "signature": "lightrag",+ "provider": "lightrag",+ "state": "published",+ "lightrag_adapter_schema": 2,+ "parser_bridge_schema": 1,+ "workspace": source_workspace,+ }+ ))manager = current_kb_manager()manager.config = manager._load_config()@@ -524,9 +531,13 @@ async def test_upload_to_moved_kb_background_task_uses_its_storage_scope(as_usertoken = library_request.set(True)try:result = await knowledge.upload_files(- qualified_kb_id("atlas", destination), background,+ qualified_kb_id("atlas", destination),+ background,files=[UploadFile(file=BytesIO(b"second source"), filename="new.txt")],- rag_provider=None, rel_paths=[], dest_subdir=None, image_analysis=None,+ rag_provider=None,+ rel_paths=[],+ dest_subdir=None,+ image_analysis=None,)finally:library_request.reset(token)@@ -536,9 +547,10 @@ async def test_upload_to_moved_kb_background_task_uses_its_storage_scope(as_userwith workspace_context(destination):target_root = current_kb_manager().base_dirassert seen == [target_root]- assert current_kb_manager()._load_config()["knowledge_bases"]["atlas"][- "uploaded_in_scope"- ] is True+ assert (+ current_kb_manager()._load_config()["knowledge_bases"]["atlas"]["uploaded_in_scope"]+ is True+ )assert (target_root / "atlas" / "raw" / "new.txt").read_bytes() == b"second source"assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]assert not (account_root / "atlas").exists()diff --git a/tests/services/rag/test_kiwix_pipeline.py b/tests/services/rag/test_kiwix_pipeline.pyindex be60a0d81..6d3594078 100644--- a/tests/services/rag/test_kiwix_pipeline.py+++ b/tests/services/rag/test_kiwix_pipeline.py@@ -188,15 +188,23 @@ def test_connection_rejects_invalid_names_and_urls() -> None:@pytest.mark.asyncioasync def test_article_fetch_rejects_redirects_and_unbounded_bodies() -> None:redirect = KiwixClient(- "http://kiwix.test", "music",- transport=httpx.MockTransport(lambda _: httpx.Response(302, headers={"location": "http://other.test/"})),+ "http://kiwix.test",+ "music",+ transport=httpx.MockTransport(+ lambda _: httpx.Response(302, headers={"location": "http://other.test/"})+ ),)with pytest.raises(KiwixError, match="HTTP 302"):await redirect.read_article("A/Jazz")oversized = KiwixClient(- "http://kiwix.test", "music",- transport=httpx.MockTransport(lambda _: httpx.Response(200, content=b"x" * 2_000_001, headers={"content-type": "text/plain"})),+ "http://kiwix.test",+ "music",+ transport=httpx.MockTransport(+ lambda _: httpx.Response(+ 200, content=b"x" * 2_000_001, headers={"content-type": "text/plain"}+ )+ ),)with pytest.raises(KiwixError, match="size limit"):await oversized.read_article("A/Jazz")@@ -208,13 +216,21 @@ async def test_book_and_mastery_use_same_access_checked_rag_search(tmp_path, monfrom deeptutor.tools.rag_tool import rag_searchmanager = KnowledgeBaseManager(base_dir=str(tmp_path))- manager.register_kiwix_kb("offline-music", "http://kiwix.test", "music", zim_title="Music archive")+ manager.register_kiwix_kb(+ "offline-music", "http://kiwix.test", "music", zim_title="Music archive"+ )resource = SimpleNamespace(base_dir=tmp_path, name="offline-music", assigned=False)monkeypatch.setattr(access, "resolve_for_rag", lambda _ref: resource)monkeypatch.setattr(access, "resolve_kb", lambda _ref, **_kwargs: resource)- monkeypatch.setattr(access, "resolve_kb_metadata", lambda _ref: manager.get_metadata("offline-music"))+ monkeypatch.setattr(+ access, "resolve_kb_metadata", lambda _ref: manager.get_metadata("offline-music")+ )monkeypatch.setattr(access, "resolve_kb_manifest", lambda *_args, **_kwargs: None)- monkeypatch.setattr(KiwixClient, "_client", lambda _self: httpx.AsyncClient(transport=httpx.MockTransport(_transport)))+ monkeypatch.setattr(+ KiwixClient,+ "_client",+ lambda _self: httpx.AsyncClient(transport=httpx.MockTransport(_transport)),+ )result = await rag_search("jazz", "offline-music")assert result["provider"] == "kiwix"@@ -231,8 +247,10 @@ async def test_book_and_mastery_use_same_access_checked_rag_search(tmp_path, monassert explorer.skipped_knowledge_bases == []source = TopicSource(- id="topic-source-1", kind=TopicSourceKind.KNOWLEDGE_BASE,- source_id="offline-music", label="Music archive",+ id="topic-source-1",+ kind=TopicSourceKind.KNOWLEDGE_BASE,+ source_id="offline-music",+ label="Music archive",)grounded = await _ground_knowledge_base_source(source, query="jazz")assert grounded.available@@ -249,7 +267,9 @@ async def test_book_and_mastery_use_same_access_checked_rag_search(tmp_path, mon@pytest.mark.asyncio-async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_path, monkeypatch) -> None:+async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(+ tmp_path, monkeypatch+) -> None:reading = importlib.import_module("deeptutor.api.routers.reading")access = importlib.import_module("deeptutor.multi_user.knowledge_access")client_module = importlib.import_module("deeptutor.services.rag.pipelines.kiwix.client")@@ -258,6 +278,7 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_from deeptutor.reading import ReadingCatalogStore, ReadingStorecatalog = ReadingCatalogStore(tmp_path / "reading")+def scoped_catalog():assert not workspace_knowledge.library_request.get()return catalog@@ -265,7 +286,9 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_monkeypatch.setattr(reading, "_catalog", scoped_catalog)monkeypatch.setattr(reading, "assert_learning_material", lambda *_args, **_kwargs: None)resource = SimpleNamespace(id="account:kb:offline-music", name="offline-music")- monkeypatch.setattr(workspace_resources, "current_resources", lambda: SimpleNamespace(knowledge_bases=[]))+ monkeypatch.setattr(+ workspace_resources, "current_resources", lambda: SimpleNamespace(knowledge_bases=[])+ )def library_resource(ref, **_kwargs):assert ref == "account:kb:offline-music"@@ -273,9 +296,17 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_return resourcemonkeypatch.setattr(workspace_knowledge, "resolve_qualified", library_resource)- monkeypatch.setattr(access, "manager_for_resource", lambda _resource: SimpleNamespace(get_metadata=lambda _name: {- "type": "kiwix", "server_url": "http://kiwix.test", "zim_name": "music"- }))+ monkeypatch.setattr(+ access,+ "manager_for_resource",+ lambda _resource: SimpleNamespace(+ get_metadata=lambda _name: {+ "type": "kiwix",+ "server_url": "http://kiwix.test",+ "zim_name": "music",+ }+ ),+ )class FakeClient:base_url = "http://kiwix.test"@@ -289,7 +320,9 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_return "Jazz\n\nImprovised music."monkeypatch.setattr(client_module, "KiwixClient", FakeClient)- payload = reading.ZimArticleImportRequest(kb_ref="account:kb:offline-music", article_path="A/Jazz", title="Jazz")+ payload = reading.ZimArticleImportRequest(+ kb_ref="account:kb:offline-music", article_path="A/Jazz", title="Jazz"+ )result = await reading.import_zim_article(payload)material_id = result["material"]["material_id"]store = ReadingStore(catalog.root)