Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

style: format knowledge source integrations

Bingxi Zhao (Frank)176257b1f8f7
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index 367fc10f5..10907877e 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -2575,10 +2575,7 @@ async def list_kiwix_catalog(
except KiwixError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return {
- "archives": [
- {"zim_name": archive.zim_name, "title": archive.title}
- for archive in archives
- ]
+ "archives": [{"zim_name": archive.zim_name, "title": archive.title} for archive in archives]
}
diff --git a/deeptutor/api/routers/reading.py b/deeptutor/api/routers/reading.py
index 5a1f12ca5..b19380eb4 100644
--- a/deeptutor/api/routers/reading.py
+++ b/deeptutor/api/routers/reading.py
@@ -608,9 +608,8 @@ async def import_zim_article(payload: ZimArticleImportRequest) -> dict[str, Any]
material_id = hashlib.sha256(f"{resource.id}\0{path}".encode()).hexdigest()[:16]
catalog = _catalog()
store = ReadingStore(catalog.root)
- source_url = (
- f"{client.base_url}/content/{client.zim_name}/"
- + "/".join(quote(segment, safe="") for segment in path.split("/"))
+ source_url = f"{client.base_url}/content/{client.zim_name}/" + "/".join(
+ quote(segment, safe="") for segment in path.split("/")
)
store.ingest_units(
material_id,
diff --git a/deeptutor/knowledge/kb_types.py b/deeptutor/knowledge/kb_types.py
index be13893e4..e318606c9 100644
--- a/deeptutor/knowledge/kb_types.py
+++ b/deeptutor/knowledge/kb_types.py
@@ -154,8 +154,8 @@ def supports_local_raw_files(entry: Any) -> bool:
def external_root_of(entry: Any) -> str | None:
"""Absolute path a connected KB points at, or ``None`` for ordinary KBs.
- ``linked`` KBs store it under ``external_path``; ``obsidian`` vaults under
-the older ``vault_path`` field. One accessor so callers don't care which.
+ ``linked`` KBs store it under ``external_path``; ``obsidian`` vaults under
+ the older ``vault_path`` field. One accessor so callers don't care which.
"""
if not isinstance(entry, dict):
return None
diff --git a/deeptutor/services/rag/pipelines/kiwix/client.py b/deeptutor/services/rag/pipelines/kiwix/client.py
index e714068e1..38460eea5 100644
--- a/deeptutor/services/rag/pipelines/kiwix/client.py
+++ b/deeptutor/services/rag/pipelines/kiwix/client.py
@@ -22,7 +22,9 @@ MAX_ARTICLE_BYTES = 2_000_000
MAX_ARTICLE_CHARS = 12_000
MAX_TITLE_BYTES = 32_000
_ZIM_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
-_BLOCKS = frozenset({"article", "br", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "tr"})
+_BLOCKS = frozenset(
+ {"article", "br", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "tr"}
+)
_SKIP = frozenset({"script", "style", "nav", "footer", "header", "noscript"})
@@ -49,7 +51,9 @@ def normalize_base_url(value: str) -> str:
or parsed.fragment
or any(part in {".", ".."} for part in unquote(parsed.path).split("/"))
):
- raise KiwixError("Enter a kiwix-serve HTTP(S) base URL without credentials or query parameters.")
+ raise KiwixError(
+ "Enter a kiwix-serve HTTP(S) base URL without credentials or query parameters."
+ )
return parsed.geturl().rstrip("/")
@@ -100,7 +104,9 @@ def html_to_text(raw: str, *, max_chars: int = MAX_ARTICLE_CHARS) -> str:
parser = _PlainText()
parser.feed(raw)
text = unescape("".join(parser.parts))
- return re.sub(r"\n{3,}", "\n\n", "\n".join(" ".join(line.split()) for line in text.splitlines())).strip()[:max_chars]
+ return re.sub(
+ r"\n{3,}", "\n\n", "\n".join(" ".join(line.split()) for line in text.splitlines())
+ ).strip()[:max_chars]
@dataclass(frozen=True)
@@ -152,7 +158,7 @@ def parse_catalog_xml(data: bytes, *, base_url: str) -> list[KiwixArchive]:
continue
if url.query or not url.path.startswith(prefix):
continue
- name = unquote(url.path[len(prefix):].split("/", 1)[0])
+ name = unquote(url.path[len(prefix) :].split("/", 1)[0])
break
if not name:
name = (fields.get("name").text or "").strip() if fields.get("name") is not None else ""
@@ -163,7 +169,11 @@ def parse_catalog_xml(data: bytes, *, base_url: str) -> list[KiwixArchive]:
if name in seen:
continue
seen.add(name)
- title = " ".join((fields.get("title").text or "").split()) if fields.get("title") is not None else ""
+ title = (
+ " ".join((fields.get("title").text or "").split())
+ if fields.get("title") is not None
+ else ""
+ )
archives.append(KiwixArchive(zim_name=name, title=title[:300] or name))
if len(archives) >= 50:
break
@@ -189,18 +199,26 @@ def parse_search_xml(data: bytes, *, base_url: str, zim_name: str) -> list[Kiwix
if parsed.query or not parsed.path.startswith(prefix):
continue
try:
- article_path = validate_article_path(parsed.path[len(prefix):])
+ article_path = validate_article_path(parsed.path[len(prefix) :])
except KiwixError:
continue
title = " ".join((item.findtext("title") or "").split())[:300]
description = item.find("description")
- snippet = " ".join("".join(description.itertext()).split())[:1000] if description is not None else ""
- hits.append(KiwixHit(title=title or article_path, article_path=article_path, snippet=snippet))
+ snippet = (
+ " ".join("".join(description.itertext()).split())[:1000]
+ if description is not None
+ else ""
+ )
+ hits.append(
+ KiwixHit(title=title or article_path, article_path=article_path, snippet=snippet)
+ )
return hits
class KiwixClient:
- def __init__(self, base_url: str, zim_name: str, *, transport: httpx.AsyncBaseTransport | None = None) -> None:
+ def __init__(
+ self, base_url: str, zim_name: str, *, transport: httpx.AsyncBaseTransport | None = None
+ ) -> None:
self.base_url = normalize_base_url(base_url)
self.zim_name = validate_zim_name(zim_name)
self.transport = transport
@@ -223,10 +241,19 @@ class KiwixClient:
if query.strip():
params["q"] = query.strip()[:100]
async with client._client() as http:
- raw, _ = await client._get(http, "catalog/search", params=params, limit=MAX_CATALOG_BYTES)
+ raw, _ = await client._get(
+ http, "catalog/search", params=params, limit=MAX_CATALOG_BYTES
+ )
return parse_catalog_xml(raw, base_url=client.base_url)
- async def _get(self, client: httpx.AsyncClient, path: str, *, params: dict[str, str | int] | None = None, limit: int) -> tuple[bytes, str]:
+ async def _get(
+ self,
+ client: httpx.AsyncClient,
+ path: str,
+ *,
+ params: dict[str, str | int] | None = None,
+ limit: int,
+ ) -> tuple[bytes, str]:
url = f"{self.base_url}/{path.lstrip('/')}"
try:
async with client.stream("GET", url, params=params) as response:
@@ -254,7 +281,12 @@ class KiwixClient:
xml, _ = await self._get(
client,
"search",
- params={"books.name": self.zim_name, "pattern": "a", "format": "xml", "pageLength": 1},
+ params={
+ "books.name": self.zim_name,
+ "pattern": "a",
+ "format": "xml",
+ "pageLength": 1,
+ },
limit=MAX_SEARCH_BYTES,
)
parse_search_xml(xml, base_url=self.base_url, zim_name=self.zim_name)
@@ -274,7 +306,12 @@ class KiwixClient:
xml, _ = await self._get(
client,
"search",
- params={"books.name": self.zim_name, "pattern": query[:500], "format": "xml", "pageLength": count},
+ params={
+ "books.name": self.zim_name,
+ "pattern": query[:500],
+ "format": "xml",
+ "pageLength": count,
+ },
limit=MAX_SEARCH_BYTES,
)
hits = parse_search_xml(xml, base_url=self.base_url, zim_name=self.zim_name)[:count]
@@ -318,7 +355,11 @@ class KiwixClient:
if mime not in {"text/html", "application/xhtml+xml", "text/plain"}:
raise KiwixError("The selected ZIM entry is not a readable article.")
decoded = raw.decode("utf-8", errors="replace")
- return html_to_text(decoded, max_chars=max_chars) if mime != "text/plain" else decoded.strip()[:max_chars]
+ return (
+ html_to_text(decoded, max_chars=max_chars)
+ if mime != "text/plain"
+ else decoded.strip()[:max_chars]
+ )
async def read_article(self, article_path: str) -> str:
async with self._client() as client:
diff --git a/deeptutor/services/rag/pipelines/kiwix/pipeline.py b/deeptutor/services/rag/pipelines/kiwix/pipeline.py
index af09a7400..89307e715 100644
--- a/deeptutor/services/rag/pipelines/kiwix/pipeline.py
+++ b/deeptutor/services/rag/pipelines/kiwix/pipeline.py
@@ -20,7 +20,9 @@ class KiwixPipeline:
async def search(self, query: str, kb_name: str, **kwargs: Any) -> dict[str, Any]:
entry = load_kb_config_entry(self.kb_base_dir, kb_name)
if entry.get("type") != PROVIDER:
- return self._error(query, "This knowledge base is not connected to Kiwix.", "not_configured")
+ return self._error(
+ query, "This knowledge base is not connected to Kiwix.", "not_configured"
+ )
try:
client = self._client_factory(entry["server_url"], entry["zim_name"])
hits = await client.search(query, top_k=kwargs.get("top_k") or 5)
diff --git a/deeptutor/services/workspace/kb_move.py b/deeptutor/services/workspace/kb_move.py
index 966969035..f5d33ec3d 100644
--- a/deeptutor/services/workspace/kb_move.py
+++ b/deeptutor/services/workspace/kb_move.py
@@ -262,8 +262,10 @@ def _rebase_llamaindex_paths(
"An indexed image is outside the movable knowledge base and parse cache."
) from exc
path_parts = [image, *list(image.parents)[: len(image.parts) - len(cache_root.parts)]]
- if resolved != image or not image.is_file() or any(
- part.is_symlink() for part in path_parts
+ if (
+ resolved != image
+ or not image.is_file()
+ or any(part.is_symlink() for part in path_parts)
):
raise WorkspaceError("An indexed image cannot be copied safely.")
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
diff --git a/deeptutor_cli/kb.py b/deeptutor_cli/kb.py
index 913e6b772..835da0e59 100644
--- a/deeptutor_cli/kb.py
+++ b/deeptutor_cli/kb.py
@@ -194,7 +194,9 @@ def register(app: typer.Typer) -> None:
try:
client = KiwixClient(server_url, zim_name)
title = asyncio.run(client.probe())
- _get_kb_manager().register_kiwix_kb(name, client.base_url, client.zim_name, zim_title=title)
+ _get_kb_manager().register_kiwix_kb(
+ name, client.base_url, client.zim_name, zim_title=title
+ )
except Exception as exc:
console.print(f"[red]Could not connect Kiwix archive: {exc}[/]")
raise typer.Exit(code=1) from exc
diff --git a/tests/multi_user/test_kb_move.py b/tests/multi_user/test_kb_move.py
index fd1335630..9e20115dc 100644
--- a/tests/multi_user/test_kb_move.py
+++ b/tests/multi_user/test_kb_move.py
@@ -233,9 +233,7 @@ def test_moved_away_name_cannot_be_registered_by_connected_providers(as_user, tm
attempt()
assert "atlas" not in manager._load_config()["knowledge_bases"]
- second_destination = ContentWorkspaceService().create_workspace("Archive")[
- "workspace_id"
- ]
+ second_destination = ContentWorkspaceService().create_workspace("Archive")["workspace_id"]
move_kb(qualified_kb_id("atlas", destination), second_destination)
with workspace_context(destination):
with pytest.raises(ValueError, match="reserved by an earlier move"):
@@ -280,12 +278,15 @@ def test_moved_llamaindex_retrieval_keeps_citations_and_images(as_user, monkeypa
cache_image.parent.mkdir(parents=True)
cache_image.write_bytes(b"image bytes")
text_node = TextNode(
- text="source alpha", id_="text-1",
+ text="source alpha",
+ id_="text-1",
metadata={"file_name": "source.pdf", "file_path": str(source_file)},
embedding=[0.5] * 8,
)
image_node = ImageNode(
- text="[Image] source.pdf", id_="image-1", image_path=str(cache_image),
+ text="[Image] source.pdf",
+ id_="image-1",
+ image_path=str(cache_image),
metadata={"file_name": "source.pdf", "file_path": str(source_file)},
embedding=[0.5] * 8,
)
@@ -330,11 +331,17 @@ async def test_moved_lightrag_search_opens_published_native_workspace(as_user, m
json.dumps({"chunk": {"content": "grounded passage"}})
)
(source_version / "meta.json").write_text(
- json.dumps({
- "version": "version-1", "signature": "lightrag", "provider": "lightrag",
- "state": "published", "lightrag_adapter_schema": 2,
- "parser_bridge_schema": 1, "workspace": source_workspace,
- })
+ json.dumps(
+ {
+ "version": "version-1",
+ "signature": "lightrag",
+ "provider": "lightrag",
+ "state": "published",
+ "lightrag_adapter_schema": 2,
+ "parser_bridge_schema": 1,
+ "workspace": source_workspace,
+ }
+ )
)
manager = current_kb_manager()
manager.config = manager._load_config()
@@ -524,9 +531,13 @@ async def test_upload_to_moved_kb_background_task_uses_its_storage_scope(as_user
token = library_request.set(True)
try:
result = await knowledge.upload_files(
- qualified_kb_id("atlas", destination), background,
+ qualified_kb_id("atlas", destination),
+ background,
files=[UploadFile(file=BytesIO(b"second source"), filename="new.txt")],
- rag_provider=None, rel_paths=[], dest_subdir=None, image_analysis=None,
+ rag_provider=None,
+ rel_paths=[],
+ dest_subdir=None,
+ image_analysis=None,
)
finally:
library_request.reset(token)
@@ -536,9 +547,10 @@ async def test_upload_to_moved_kb_background_task_uses_its_storage_scope(as_user
with workspace_context(destination):
target_root = current_kb_manager().base_dir
assert seen == [target_root]
- assert current_kb_manager()._load_config()["knowledge_bases"]["atlas"][
- "uploaded_in_scope"
- ] is True
+ assert (
+ current_kb_manager()._load_config()["knowledge_bases"]["atlas"]["uploaded_in_scope"]
+ is True
+ )
assert (target_root / "atlas" / "raw" / "new.txt").read_bytes() == b"second source"
assert "atlas" not in current_kb_manager()._load_config()["knowledge_bases"]
assert not (account_root / "atlas").exists()
diff --git a/tests/services/rag/test_kiwix_pipeline.py b/tests/services/rag/test_kiwix_pipeline.py
index be60a0d81..6d3594078 100644
--- a/tests/services/rag/test_kiwix_pipeline.py
+++ b/tests/services/rag/test_kiwix_pipeline.py
@@ -188,15 +188,23 @@ def test_connection_rejects_invalid_names_and_urls() -> None:
@pytest.mark.asyncio
async def test_article_fetch_rejects_redirects_and_unbounded_bodies() -> None:
redirect = KiwixClient(
- "http://kiwix.test", "music",
- transport=httpx.MockTransport(lambda _: httpx.Response(302, headers={"location": "http://other.test/"})),
+ "http://kiwix.test",
+ "music",
+ transport=httpx.MockTransport(
+ lambda _: httpx.Response(302, headers={"location": "http://other.test/"})
+ ),
)
with pytest.raises(KiwixError, match="HTTP 302"):
await redirect.read_article("A/Jazz")
oversized = KiwixClient(
- "http://kiwix.test", "music",
- transport=httpx.MockTransport(lambda _: httpx.Response(200, content=b"x" * 2_000_001, headers={"content-type": "text/plain"})),
+ "http://kiwix.test",
+ "music",
+ transport=httpx.MockTransport(
+ lambda _: httpx.Response(
+ 200, content=b"x" * 2_000_001, headers={"content-type": "text/plain"}
+ )
+ ),
)
with pytest.raises(KiwixError, match="size limit"):
await oversized.read_article("A/Jazz")
@@ -208,13 +216,21 @@ async def test_book_and_mastery_use_same_access_checked_rag_search(tmp_path, mon
from deeptutor.tools.rag_tool import rag_search
manager = KnowledgeBaseManager(base_dir=str(tmp_path))
- manager.register_kiwix_kb("offline-music", "http://kiwix.test", "music", zim_title="Music archive")
+ manager.register_kiwix_kb(
+ "offline-music", "http://kiwix.test", "music", zim_title="Music archive"
+ )
resource = SimpleNamespace(base_dir=tmp_path, name="offline-music", assigned=False)
monkeypatch.setattr(access, "resolve_for_rag", lambda _ref: resource)
monkeypatch.setattr(access, "resolve_kb", lambda _ref, **_kwargs: resource)
- monkeypatch.setattr(access, "resolve_kb_metadata", lambda _ref: manager.get_metadata("offline-music"))
+ monkeypatch.setattr(
+ access, "resolve_kb_metadata", lambda _ref: manager.get_metadata("offline-music")
+ )
monkeypatch.setattr(access, "resolve_kb_manifest", lambda *_args, **_kwargs: None)
- monkeypatch.setattr(KiwixClient, "_client", lambda _self: httpx.AsyncClient(transport=httpx.MockTransport(_transport)))
+ monkeypatch.setattr(
+ KiwixClient,
+ "_client",
+ lambda _self: httpx.AsyncClient(transport=httpx.MockTransport(_transport)),
+ )
result = await rag_search("jazz", "offline-music")
assert result["provider"] == "kiwix"
@@ -231,8 +247,10 @@ async def test_book_and_mastery_use_same_access_checked_rag_search(tmp_path, mon
assert explorer.skipped_knowledge_bases == []
source = TopicSource(
- id="topic-source-1", kind=TopicSourceKind.KNOWLEDGE_BASE,
- source_id="offline-music", label="Music archive",
+ id="topic-source-1",
+ kind=TopicSourceKind.KNOWLEDGE_BASE,
+ source_id="offline-music",
+ label="Music archive",
)
grounded = await _ground_knowledge_base_source(source, query="jazz")
assert grounded.available
@@ -249,7 +267,9 @@ async def test_book_and_mastery_use_same_access_checked_rag_search(tmp_path, mon
@pytest.mark.asyncio
-async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_path, monkeypatch) -> None:
+async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(
+ tmp_path, monkeypatch
+) -> None:
reading = importlib.import_module("deeptutor.api.routers.reading")
access = importlib.import_module("deeptutor.multi_user.knowledge_access")
client_module = importlib.import_module("deeptutor.services.rag.pipelines.kiwix.client")
@@ -258,6 +278,7 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_
from deeptutor.reading import ReadingCatalogStore, ReadingStore
catalog = ReadingCatalogStore(tmp_path / "reading")
+
def scoped_catalog():
assert not workspace_knowledge.library_request.get()
return catalog
@@ -265,7 +286,9 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_
monkeypatch.setattr(reading, "_catalog", scoped_catalog)
monkeypatch.setattr(reading, "assert_learning_material", lambda *_args, **_kwargs: None)
resource = SimpleNamespace(id="account:kb:offline-music", name="offline-music")
- monkeypatch.setattr(workspace_resources, "current_resources", lambda: SimpleNamespace(knowledge_bases=[]))
+ monkeypatch.setattr(
+ workspace_resources, "current_resources", lambda: SimpleNamespace(knowledge_bases=[])
+ )
def library_resource(ref, **_kwargs):
assert ref == "account:kb:offline-music"
@@ -273,9 +296,17 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_
return resource
monkeypatch.setattr(workspace_knowledge, "resolve_qualified", library_resource)
- monkeypatch.setattr(access, "manager_for_resource", lambda _resource: SimpleNamespace(get_metadata=lambda _name: {
- "type": "kiwix", "server_url": "http://kiwix.test", "zim_name": "music"
- }))
+ monkeypatch.setattr(
+ access,
+ "manager_for_resource",
+ lambda _resource: SimpleNamespace(
+ get_metadata=lambda _name: {
+ "type": "kiwix",
+ "server_url": "http://kiwix.test",
+ "zim_name": "music",
+ }
+ ),
+ )
class FakeClient:
base_url = "http://kiwix.test"
@@ -289,7 +320,9 @@ async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_
return "Jazz\n\nImprovised music."
monkeypatch.setattr(client_module, "KiwixClient", FakeClient)
- payload = reading.ZimArticleImportRequest(kb_ref="account:kb:offline-music", article_path="A/Jazz", title="Jazz")
+ payload = reading.ZimArticleImportRequest(
+ kb_ref="account:kb:offline-music", article_path="A/Jazz", title="Jazz"
+ )
result = await reading.import_zim_article(payload)
material_id = result["material"]["material_id"]
store = ReadingStore(catalog.root)