Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Add connected Kiwix ZIM knowledge sources

Bingxi Zhao (Frank)64c2316093a3
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index c92014a0c..0e7cf2427 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -29,6 +29,7 @@ from fastapi import (
File,
Form,
HTTPException,
+ Query,
UploadFile,
WebSocket,
WebSocketDisconnect,
@@ -2508,6 +2509,102 @@ class ConnectImaRequest(BaseModel):
knowledge_base_id: str
+class KiwixConnectionRequest(BaseModel):
+ server_url: str
+ zim_name: str
+
+
+class ConnectKiwixRequest(KiwixConnectionRequest):
+ name: str
+
+
[email protected]("/knowledge-bases/kiwix-catalog", dependencies=[Depends(require_admin)])
+async def list_kiwix_catalog(
+ server_url: str = Query(min_length=1, max_length=2048),
+ q: str = Query(default="", max_length=100),
+) -> dict[str, Any]:
+ """List exact ZIM names from legacy OPDS; v2 can omit loaded archives."""
+ from deeptutor.services.rag.pipelines.kiwix.client import KiwixClient, KiwixError
+
+ try:
+ archives = await KiwixClient.list_archives(server_url, q)
+ except KiwixError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ return {
+ "archives": [
+ {"zim_name": archive.zim_name, "title": archive.title}
+ for archive in archives
+ ]
+ }
+
+
[email protected]("/knowledge-bases/probe-kiwix", dependencies=[Depends(require_admin)])
+async def probe_kiwix_route(payload: KiwixConnectionRequest) -> dict[str, Any]:
+ """Verify that one archive is readable without copying its ZIM file."""
+ from deeptutor.services.rag.pipelines.kiwix.client import KiwixClient, KiwixError
+
+ try:
+ client = KiwixClient(payload.server_url, payload.zim_name)
+ title = await client.probe()
+ except KiwixError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ return {"ok": True, "zim_name": client.zim_name, "title": title}
+
+
[email protected]("/knowledge-bases/connect-kiwix", dependencies=[Depends(require_admin)])
+async def connect_kiwix_route(payload: ConnectKiwixRequest) -> dict[str, Any]:
+ """Bind a KB to a kiwix-serve archive; no ingest or index is created."""
+ from deeptutor.services.rag.pipelines.kiwix.client import KiwixClient, KiwixError
+
+ try:
+ client = KiwixClient(payload.server_url, payload.zim_name)
+ title = await client.probe()
+ manager = get_kb_manager()
+ entry = manager.register_kiwix_kb(
+ payload.name,
+ client.base_url,
+ client.zim_name,
+ zim_title=title,
+ )
+ except (KiwixError, ValueError) as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ return {
+ "status": "connected",
+ "name": payload.name,
+ "zim_name": entry["zim_name"],
+ "title": entry["zim_title"],
+ "rag_provider": entry["rag_provider"],
+ }
+
+
[email protected]("/knowledge-bases/kiwix-articles")
+async def search_kiwix_articles(
+ kb_ref: str = Query(min_length=1, max_length=300),
+ q: str = Query(min_length=1, max_length=500),
+) -> dict[str, Any]:
+ """List bounded article matches for Knowledge Center and Reading import."""
+ from deeptutor.multi_user.knowledge_access import resolve_kb_metadata
+ from deeptutor.tools.rag_tool import rag_search
+
+ entry = resolve_kb_metadata(kb_ref)
+ if not entry or entry.get("type") != "kiwix":
+ raise HTTPException(status_code=404, detail="Kiwix knowledge base not found.")
+ result = await rag_search(q, kb_ref, top_k=10)
+ if result.get("error_type"):
+ raise HTTPException(status_code=502, detail=result.get("answer") or "Kiwix search failed.")
+ return {
+ "articles": [
+ {
+ "title": source.get("title") or "",
+ "article_path": source.get("article_path") or "",
+ "excerpt": str(source.get("content") or "")[:500],
+ }
+ for source in result.get("sources") or []
+ if isinstance(source, dict) and source.get("article_path")
+ ]
+ }
+
+
@router.post("/knowledge-bases/probe-ima")
async def probe_ima_route(payload: ProbeImaRequest):
"""Test-connect to a Tencent IMA knowledge base before binding a KB to it.
diff --git a/deeptutor/api/routers/reading.py b/deeptutor/api/routers/reading.py
index b08779743..5a1f12ca5 100644
--- a/deeptutor/api/routers/reading.py
+++ b/deeptutor/api/routers/reading.py
@@ -17,11 +17,13 @@ pulling the whole file before rendering page one.
from __future__ import annotations
import asyncio
+import hashlib
import logging
from pathlib import Path
import shutil
import tempfile
from typing import Any, Literal
+from urllib.parse import quote
import uuid
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, UploadFile
@@ -358,6 +360,13 @@ class UrlImportRequest(BaseModel):
workspace_title: str = ""
+class ZimArticleImportRequest(BaseModel):
+ kb_ref: str = Field(min_length=1, max_length=300)
+ article_path: str = Field(min_length=1, max_length=2048)
+ title: str = Field(default="", max_length=300)
+ workspace_id: str = ""
+
+
class WorkspaceCreateRequest(BaseModel):
title: str = Field(default="Untitled collection", max_length=300)
description: str = Field(default="", max_length=2000)
@@ -559,6 +568,91 @@ async def import_urls(
raise _http_error(exc) from exc
[email protected]("/library/import-zim-article")
+async def import_zim_article(payload: ZimArticleImportRequest) -> dict[str, Any]:
+ """Open one selected Kiwix article in Immersive Reading.
+
+ Only its bounded text snapshot enters the reading store. The ZIM archive
+ stays on the existing kiwix-serve instance, without bulk extraction.
+ """
+ from deeptutor.multi_user.knowledge_access import manager_for_resource, resolve_kb
+ from deeptutor.reading.catalog_models import IngestionStatus, SourceKind
+ from deeptutor.reading.extract import split_markdown_by_headings
+ from deeptutor.services.rag.pipelines.kiwix.client import (
+ KiwixClient,
+ KiwixError,
+ validate_article_path,
+ )
+ from deeptutor.services.workspace.knowledge import library_request
+
+ try:
+ assert_learning_material("", upload=True)
+ # Knowledge Center browses the account library outside the current
+ # conversation's selected KB list. Resolve the source with that same
+ # access scope, then restore the Reading workspace for storage.
+ token = library_request.set(True)
+ try:
+ resource = resolve_kb(payload.kb_ref)
+ entry = manager_for_resource(resource).get_metadata(resource.name)
+ finally:
+ library_request.reset(token)
+ if not isinstance(entry, dict) or entry.get("type") != "kiwix":
+ raise ReadingError("Select a connected Kiwix knowledge base.")
+ path = validate_article_path(payload.article_path)
+ client = KiwixClient(entry["server_url"], entry["zim_name"])
+ text = await client.read_article(path)
+ units, outline = split_markdown_by_headings(text)
+ if not units:
+ raise ReadingError("The selected ZIM article has no readable text.")
+ title = payload.title.strip() or path.rsplit("/", 1)[-1].replace("_", " ")
+ material_id = hashlib.sha256(f"{resource.id}\0{path}".encode()).hexdigest()[:16]
+ catalog = _catalog()
+ store = ReadingStore(catalog.root)
+ source_url = (
+ f"{client.base_url}/content/{client.zim_name}/"
+ + "/".join(quote(segment, safe="") for segment in path.split("/"))
+ )
+ store.ingest_units(
+ material_id,
+ filename=f"{title[:150]}.md",
+ units=units,
+ unit="section",
+ title=title,
+ mime="text/markdown",
+ extractor="kiwix-article",
+ content_format="plain_text",
+ source_type="zim_article",
+ source_url=source_url,
+ outline=outline or None,
+ )
+ material = catalog.upsert_material(
+ content_id=material_id,
+ material_id=material_id,
+ filename=f"{title[:150]}.md",
+ title=title,
+ source_kind=SourceKind.WEB,
+ source_url=source_url,
+ mime="text/markdown",
+ render_mode="text",
+ status=IngestionStatus.READY,
+ )
+ if payload.workspace_id.strip():
+ catalog.add_material(payload.workspace_id.strip(), material_id)
+ workspace = catalog.get_workspace(payload.workspace_id.strip())
+ else:
+ workspace = catalog.create_workspace(title, [material_id])
+ return {
+ "material": material.to_dict(),
+ "workspace": workspace.to_dict() if workspace else None,
+ }
+ except HTTPException:
+ raise
+ except (KiwixError, KeyError) as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ except Exception as exc:
+ raise _http_error(exc) from exc
+
+
@router.post("/materials/{material_id}/retry", status_code=202)
async def retry_import(material_id: str, background_tasks: BackgroundTasks) -> dict[str, Any]:
service = _ingestion()
diff --git a/deeptutor/book/kb_health.py b/deeptutor/book/kb_health.py
index 0f6e31415..e25caba69 100644
--- a/deeptutor/book/kb_health.py
+++ b/deeptutor/book/kb_health.py
@@ -115,6 +115,14 @@ def fingerprint_kb_documents(
mgr = manager or _current_manager()
if kb_name not in mgr.list_knowledge_bases():
return {}
+ entry = mgr.get_metadata(kb_name)
+ if isinstance(entry, dict) and entry.get("type") == "kiwix":
+ # A connected archive has no raw/ directory. Fingerprint the binding
+ # so Book can detect a changed server/ZIM pointer without hashing or
+ # copying a multi-GB archive. Content changed behind the same pointer
+ # cannot be detected without a server-provided revision identifier.
+ identity = f"{entry.get('server_url', '')}\0{entry.get('zim_name', '')}"
+ return {"zim-archive": f"sha256:{hashlib.sha256(identity.encode()).hexdigest()}"}
raw_dir = mgr.base_dir / kb_name / "raw"
if not raw_dir.exists():
return {}
diff --git a/deeptutor/knowledge/kb_types.py b/deeptutor/knowledge/kb_types.py
index 5d8c76a45..be13893e4 100644
--- a/deeptutor/knowledge/kb_types.py
+++ b/deeptutor/knowledge/kb_types.py
@@ -34,12 +34,16 @@ flavours exist today:
self-hosted Tencent WeKnora deployment. Retrieval is offloaded to WeKnora's
knowledge-search API by the bound ``weknora`` provider; no documents are
copied or indexed locally. See ``services/rag/pipelines/weknora``.
+* ``kiwix`` — a pointer (``server_url`` + ``zim_name``) to one searchable ZIM
+ served by kiwix-serve. Retrieval reads matched articles over its public HTTP
+ API; DeepTutor neither copies the archive nor builds a second index.
All connected flavours share the same lifecycle quirks: no on-disk folder under
``base_dir``, no embedding reconcile, and deletion must never touch the
external resource. The :func:`is_connected_kb` / :func:`external_root_of` helpers
let the manager treat them uniformly without sprinkling ``type`` literals
-across the codebase. ``subagent``, ``lightrag_server``, ``ima`` and ``weknora`` are connected
+across the codebase. ``subagent``, ``lightrag_server``, ``ima``, ``weknora`` and
+``kiwix`` are connected
but point at no folder, so :func:`external_root_of` returns ``None`` for them — a
subagent is driven by its capability and the two server-backed kinds are reached
over HTTP; none resolves to a local path.
@@ -90,6 +94,10 @@ WEKNORA_KB_TYPE = "weknora"
# See ``capabilities/marginnote4``.
MARGINNOTE4_KB_TYPE = "marginnote4"
+# One archive served by kiwix-serve. The server owns the ZIM and its index;
+# DeepTutor stores only its URL and ZIM name.
+KIWIX_KB_TYPE = "kiwix"
+
# Every pointer/connected KB type. Membership here is what makes the manager
# skip the index pipeline, the orphan prune and the embedding reconcile.
CONNECTED_KB_TYPES = frozenset(
@@ -101,6 +109,7 @@ CONNECTED_KB_TYPES = frozenset(
IMA_KB_TYPE,
WEKNORA_KB_TYPE,
MARGINNOTE4_KB_TYPE,
+ KIWIX_KB_TYPE,
}
)
@@ -146,7 +155,7 @@ def external_root_of(entry: Any) -> str | None:
"""Absolute path a connected KB points at, or ``None`` for ordinary KBs.
``linked`` KBs store it under ``external_path``; ``obsidian`` vaults under
- the older ``vault_path`` field. One accessor so callers don't care which.
+the older ``vault_path`` field. One accessor so callers don't care which.
"""
if not isinstance(entry, dict):
return None
@@ -161,6 +170,7 @@ __all__ = [
"IMA_KB_TYPE",
"WEKNORA_KB_TYPE",
"MARGINNOTE4_KB_TYPE",
+ "KIWIX_KB_TYPE",
"CONNECTED_KB_TYPES",
"NON_RETRIEVABLE_KB_TYPES",
"is_connected_kb",
diff --git a/deeptutor/knowledge/manager.py b/deeptutor/knowledge/manager.py
index 0c0a011dc..b1467dbe9 100644
--- a/deeptutor/knowledge/manager.py
+++ b/deeptutor/knowledge/manager.py
@@ -1126,6 +1126,46 @@ class KnowledgeBaseManager:
self._save_config()
return entry
+ def register_kiwix_kb(
+ self,
+ name: str,
+ server_url: str,
+ zim_name: str,
+ *,
+ zim_title: str = "",
+ ) -> dict:
+ """Register a read-only pointer to one ZIM already served by Kiwix."""
+ from deeptutor.knowledge.kb_types import KIWIX_KB_TYPE
+ from deeptutor.services.rag.pipelines.kiwix.client import (
+ normalize_base_url,
+ validate_zim_name,
+ )
+
+ name = validate_knowledge_base_name(name)
+ server_url = normalize_base_url(server_url)
+ zim_name = validate_zim_name(zim_name)
+ self.config = self._load_config()
+ bases = self.config.setdefault("knowledge_bases", {})
+ if name in bases:
+ raise ValueError(f"A knowledge base named '{name}' already exists.")
+ now = datetime.now().isoformat()
+ entry = {
+ "path": name,
+ "type": KIWIX_KB_TYPE,
+ "rag_provider": KIWIX_KB_TYPE,
+ "server_url": server_url,
+ "zim_name": zim_name,
+ "zim_title": zim_title[:300],
+ "description": f"Kiwix archive: {zim_title or zim_name}",
+ "status": "ready",
+ "needs_reindex": False,
+ "created_at": now,
+ "updated_at": now,
+ }
+ bases[name] = entry
+ self._save_config()
+ return entry
+
def get_knowledge_base_path(self, name: str | None = None) -> Path:
"""Get path to a knowledge base.
@@ -1284,6 +1324,8 @@ class KnowledgeBaseManager:
# LightRAG server pointer (the URL is safe to surface; the API
# key deliberately is not).
"server_url": kb_config.get("server_url"),
+ "zim_name": kb_config.get("zim_name"),
+ "zim_title": kb_config.get("zim_title"),
# IMA pointer. The library id identifies which IMA knowledge
# base this KB reads; the client id and API key are credentials
# and are deliberately absent from this allowlist.
@@ -1446,6 +1488,9 @@ class KnowledgeBaseManager:
# Same split for IMA: the library id is shown, the credentials are not.
if kb_config.get("knowledge_base_id"):
metadata["knowledge_base_id"] = kb_config.get("knowledge_base_id")
+ if kb_config.get("zim_name"):
+ metadata["zim_name"] = kb_config.get("zim_name")
+ metadata["zim_title"] = kb_config.get("zim_title") or ""
if rag_provider == LIGHTRAG_PROVIDER:
from deeptutor.services.rag.pipelines.lightrag.storage import (
diff --git a/deeptutor/knowledge/manifest.py b/deeptutor/knowledge/manifest.py
index 2b9ab9915..759701746 100644
--- a/deeptutor/knowledge/manifest.py
+++ b/deeptutor/knowledge/manifest.py
@@ -45,6 +45,7 @@ from typing import Any
from deeptutor.knowledge.kb_types import (
IMA_KB_TYPE,
+ KIWIX_KB_TYPE,
LIGHTRAG_SERVER_KB_TYPE,
SUBAGENT_KB_TYPE,
external_root_of,
@@ -71,6 +72,7 @@ UNAVAILABLE_MISSING = "missing"
_NON_DOCUMENT_KB_TYPES: dict[str, str] = {
LIGHTRAG_SERVER_KB_TYPE: UNAVAILABLE_REMOTE,
IMA_KB_TYPE: UNAVAILABLE_REMOTE,
+ KIWIX_KB_TYPE: UNAVAILABLE_REMOTE,
SUBAGENT_KB_TYPE: UNAVAILABLE_AGENT,
}
diff --git a/deeptutor/services/rag/factory.py b/deeptutor/services/rag/factory.py
index 85cd1c558..1dcdadfe0 100644
--- a/deeptutor/services/rag/factory.py
+++ b/deeptutor/services/rag/factory.py
@@ -21,6 +21,9 @@ today:
self-hosted Tencent WeKnora deployment. No local
index or document copy; each KB is a connection
pointer queried over WeKnora's REST API.
+* ``kiwix`` — retrieval from one ZIM already served by
+ kiwix-serve. Only matched articles are fetched;
+ no local archive or index copy is made.
A KB is bound to one provider at creation time; later adds and retrieval always
go through that same pipeline (enforced upstream in the knowledge router).
@@ -39,6 +42,7 @@ LIGHTRAG_PROVIDER = "lightrag"
LIGHTRAG_SERVER_PROVIDER = "lightrag-server"
IMA_PROVIDER = "ima"
WEKNORA_PROVIDER = "weknora"
+KIWIX_PROVIDER = "kiwix"
# Providers the factory can instantiate. Unknown / legacy strings fall back to
# the default with a re-index hint upstream.
@@ -52,6 +56,7 @@ KNOWN_PROVIDERS = frozenset(
LIGHTRAG_SERVER_PROVIDER,
IMA_PROVIDER,
WEKNORA_PROVIDER,
+ KIWIX_PROVIDER,
}
)
@@ -96,6 +101,7 @@ def version_matches_provider(entry: dict[str, Any], provider: Optional[str]) ->
LIGHTRAG_SERVER_PROVIDER,
IMA_PROVIDER,
WEKNORA_PROVIDER,
+ KIWIX_PROVIDER,
}
return entry_provider == resolved or signature == resolved
@@ -170,6 +176,13 @@ def _build_pipeline(provider: str, kb_base_dir: Optional[str], **kwargs: Any):
kwargs.setdefault("kb_base_dir", kb_base_dir)
return WeKnoraPipeline(**kwargs)
+ if provider == KIWIX_PROVIDER:
+ from .pipelines.kiwix.pipeline import KiwixPipeline
+
+ if kb_base_dir is not None:
+ kwargs.setdefault("kb_base_dir", kb_base_dir)
+ return KiwixPipeline(**kwargs)
+
from .pipelines.llamaindex.pipeline import LlamaIndexPipeline
if kb_base_dir is not None:
diff --git a/deeptutor/services/rag/pipelines/kiwix/__init__.py b/deeptutor/services/rag/pipelines/kiwix/__init__.py
new file mode 100644
index 000000000..8c62c1996
--- /dev/null
+++ b/deeptutor/services/rag/pipelines/kiwix/__init__.py
@@ -0,0 +1 @@
+"""Read-only retrieval from ZIM archives already served by kiwix-serve."""
diff --git a/deeptutor/services/rag/pipelines/kiwix/client.py b/deeptutor/services/rag/pipelines/kiwix/client.py
new file mode 100644
index 000000000..e714068e1
--- /dev/null
+++ b/deeptutor/services/rag/pipelines/kiwix/client.py
@@ -0,0 +1,328 @@
+"""Bounded client for kiwix-serve's public search and raw-content APIs.
+
+One connected KB points to one ZIM name on one server. Search uses the ZIM's
+existing full-text index; only matched articles are read, so no archive is
+uploaded, unpacked, or indexed again by DeepTutor.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from html import unescape
+from html.parser import HTMLParser
+import re
+from urllib.parse import quote, unquote, urlsplit
+
+from defusedxml import ElementTree
+import httpx
+
+MAX_SEARCH_BYTES = 512_000
+MAX_CATALOG_BYTES = 1_000_000
+MAX_ARTICLE_BYTES = 2_000_000
+MAX_ARTICLE_CHARS = 12_000
+MAX_TITLE_BYTES = 32_000
+_ZIM_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
+_BLOCKS = frozenset({"article", "br", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "tr"})
+_SKIP = frozenset({"script", "style", "nav", "footer", "header", "noscript"})
+
+
+class KiwixError(ValueError):
+ """The configured server or selected archive cannot satisfy a request."""
+
+
+def normalize_base_url(value: str) -> str:
+ raw = str(value or "").strip()
+ if len(raw) > 2048 or any(ord(char) < 32 for char in raw):
+ raise KiwixError("Enter a valid kiwix-serve base URL.")
+ parsed = urlsplit(raw)
+ try:
+ port = parsed.port
+ except ValueError as exc:
+ raise KiwixError("Enter a valid kiwix-serve host and port.") from exc
+ if (
+ parsed.scheme not in {"http", "https"}
+ or not parsed.hostname
+ or (port is not None and not 1 <= port <= 65535)
+ or parsed.username
+ or parsed.password
+ or parsed.query
+ or parsed.fragment
+ or any(part in {".", ".."} for part in unquote(parsed.path).split("/"))
+ ):
+ raise KiwixError("Enter a kiwix-serve HTTP(S) base URL without credentials or query parameters.")
+ return parsed.geturl().rstrip("/")
+
+
+def validate_zim_name(value: str) -> str:
+ name = str(value or "").strip()
+ if not _ZIM_NAME.fullmatch(name):
+ raise KiwixError("Enter one ZIM name from the Kiwix library (letters, numbers, _, - or .).")
+ return name
+
+
+def validate_article_path(value: str) -> str:
+ path = unquote(str(value or "").strip()).lstrip("/")
+ if (
+ not path
+ or len(path) > 2048
+ or any(part in {"", ".", ".."} for part in path.split("/"))
+ or any(ord(char) < 32 for char in path)
+ or "\\" in path
+ ):
+ raise KiwixError("Invalid ZIM article path.")
+ return path
+
+
+class _PlainText(HTMLParser):
+ def __init__(self) -> None:
+ super().__init__(convert_charrefs=True)
+ self.parts: list[str] = []
+ self.skip_depth = 0
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ if tag in _SKIP:
+ self.skip_depth += 1
+ elif not self.skip_depth and tag in _BLOCKS:
+ self.parts.append("\n")
+
+ def handle_endtag(self, tag: str) -> None:
+ if tag in _SKIP and self.skip_depth:
+ self.skip_depth -= 1
+ elif not self.skip_depth and tag in _BLOCKS:
+ self.parts.append("\n")
+
+ def handle_data(self, data: str) -> None:
+ if not self.skip_depth:
+ self.parts.append(data)
+
+
+def html_to_text(raw: str, *, max_chars: int = MAX_ARTICLE_CHARS) -> str:
+ parser = _PlainText()
+ parser.feed(raw)
+ text = unescape("".join(parser.parts))
+ return re.sub(r"\n{3,}", "\n\n", "\n".join(" ".join(line.split()) for line in text.splitlines())).strip()[:max_chars]
+
+
+@dataclass(frozen=True)
+class KiwixHit:
+ title: str
+ article_path: str
+ snippet: str
+
+
+@dataclass(frozen=True)
+class KiwixArchive:
+ zim_name: str
+ title: str
+
+
+def _local_name(tag: str) -> str:
+ return tag.rsplit("}", 1)[-1]
+
+
+def parse_catalog_xml(data: bytes, *, base_url: str) -> list[KiwixArchive]:
+ """Read exact ZIM identifiers from a bounded legacy OPDS acquisition feed."""
+ try:
+ root = ElementTree.fromstring(data)
+ except Exception as exc:
+ raise KiwixError("kiwix-serve returned invalid OPDS catalog XML.") from exc
+ if _local_name(root.tag) != "feed":
+ raise KiwixError("kiwix-serve returned an unexpected OPDS catalog format.")
+ base = urlsplit(base_url)
+ prefix = f"{base.path.rstrip('/')}/content/"
+ archives: list[KiwixArchive] = []
+ seen: set[str] = set()
+ for entry in root:
+ if _local_name(entry.tag) != "entry":
+ continue
+ fields = {_local_name(child.tag): child for child in entry}
+ name = ""
+ # OPDS <name> may be a date-free alias. The same-server content link
+ # identifies the exact loaded archive/version selected by the user.
+ for link in entry:
+ if (
+ _local_name(link.tag) != "link"
+ or link.attrib.get("type", "").split(";", 1)[0].lower() != "text/html"
+ ):
+ continue
+ url = urlsplit(link.attrib.get("href") or "")
+ if url.scheme and (url.scheme, url.netloc) != (base.scheme, base.netloc):
+ continue
+ if url.netloc and not url.scheme:
+ continue
+ if url.query or not url.path.startswith(prefix):
+ continue
+ name = unquote(url.path[len(prefix):].split("/", 1)[0])
+ break
+ if not name:
+ name = (fields.get("name").text or "").strip() if fields.get("name") is not None else ""
+ try:
+ name = validate_zim_name(name)
+ except KiwixError:
+ continue
+ if name in seen:
+ continue
+ seen.add(name)
+ title = " ".join((fields.get("title").text or "").split()) if fields.get("title") is not None else ""
+ archives.append(KiwixArchive(zim_name=name, title=title[:300] or name))
+ if len(archives) >= 50:
+ break
+ return archives
+
+
+def parse_search_xml(data: bytes, *, base_url: str, zim_name: str) -> list[KiwixHit]:
+ """Accept only RSS items linked to the selected archive on this server."""
+ try:
+ root = ElementTree.fromstring(data)
+ except Exception as exc:
+ raise KiwixError("kiwix-serve returned invalid search XML.") from exc
+ if root.tag != "rss":
+ raise KiwixError("kiwix-serve returned an unexpected search format.")
+ base = urlsplit(base_url)
+ prefix = f"{base.path.rstrip('/')}/content/{quote(zim_name, safe='')}/"
+ hits: list[KiwixHit] = []
+ for item in root.findall("./channel/item"):
+ link = (item.findtext("link") or "").strip()
+ parsed = urlsplit(link)
+ if parsed.scheme and (parsed.scheme, parsed.netloc) != (base.scheme, base.netloc):
+ continue
+ if parsed.query or not parsed.path.startswith(prefix):
+ continue
+ try:
+ article_path = validate_article_path(parsed.path[len(prefix):])
+ except KiwixError:
+ continue
+ title = " ".join((item.findtext("title") or "").split())[:300]
+ description = item.find("description")
+ snippet = " ".join("".join(description.itertext()).split())[:1000] if description is not None else ""
+ hits.append(KiwixHit(title=title or article_path, article_path=article_path, snippet=snippet))
+ return hits
+
+
+class KiwixClient:
+ def __init__(self, base_url: str, zim_name: str, *, transport: httpx.AsyncBaseTransport | None = None) -> None:
+ self.base_url = normalize_base_url(base_url)
+ self.zim_name = validate_zim_name(zim_name)
+ self.transport = transport
+
+ def _client(self) -> httpx.AsyncClient:
+ return httpx.AsyncClient(
+ transport=self.transport,
+ timeout=httpx.Timeout(10.0),
+ follow_redirects=False,
+ trust_env=False,
+ )
+
+ @classmethod
+ async def list_archives(
+ cls, base_url: str, query: str = "", *, transport: httpx.AsyncBaseTransport | None = None
+ ) -> list[KiwixArchive]:
+ """Browse a server's loaded books via legacy OPDS, whose v2 feed can omit them."""
+ client = cls(base_url, "catalog", transport=transport)
+ params: dict[str, str | int] = {"count": 50, "start": 0}
+ if query.strip():
+ params["q"] = query.strip()[:100]
+ async with client._client() as http:
+ raw, _ = await client._get(http, "catalog/search", params=params, limit=MAX_CATALOG_BYTES)
+ return parse_catalog_xml(raw, base_url=client.base_url)
+
+ async def _get(self, client: httpx.AsyncClient, path: str, *, params: dict[str, str | int] | None = None, limit: int) -> tuple[bytes, str]:
+ url = f"{self.base_url}/{path.lstrip('/')}"
+ try:
+ async with client.stream("GET", url, params=params) as response:
+ if response.status_code != 200:
+ raise KiwixError(f"kiwix-serve returned HTTP {response.status_code}.")
+ content_type = response.headers.get("content-type", "").split(";", 1)[0].lower()
+ body = bytearray()
+ async for chunk in response.aiter_bytes():
+ body.extend(chunk)
+ if len(body) > limit:
+ raise KiwixError("kiwix-serve response exceeded the article size limit.")
+ return bytes(body), content_type
+ except httpx.HTTPError as exc:
+ raise KiwixError("Could not reach kiwix-serve.") from exc
+
+ async def probe(self) -> str:
+ """Check metadata and a real one-result search without scanning the ZIM."""
+ async with self._client() as client:
+ raw, _ = await self._get(
+ client,
+ f"raw/{quote(self.zim_name, safe='')}/meta/Title",
+ limit=MAX_TITLE_BYTES,
+ )
+ try:
+ xml, _ = await self._get(
+ client,
+ "search",
+ params={"books.name": self.zim_name, "pattern": "a", "format": "xml", "pageLength": 1},
+ limit=MAX_SEARCH_BYTES,
+ )
+ parse_search_xml(xml, base_url=self.base_url, zim_name=self.zim_name)
+ except KiwixError as exc:
+ raise KiwixError(
+ "The archive is readable, but kiwix-serve full-text search is unavailable. "
+ "Book and Guided Learning need a searchable ZIM."
+ ) from exc
+ title = raw.decode("utf-8", errors="replace").strip()
+ if not title:
+ raise KiwixError("The selected ZIM archive has no readable title metadata.")
+ return title[:300]
+
+ async def search(self, query: str, *, top_k: int = 5) -> list[tuple[KiwixHit, str]]:
+ count = max(1, min(int(top_k), 10))
+ async with self._client() as client:
+ xml, _ = await self._get(
+ client,
+ "search",
+ params={"books.name": self.zim_name, "pattern": query[:500], "format": "xml", "pageLength": count},
+ limit=MAX_SEARCH_BYTES,
+ )
+ hits = parse_search_xml(xml, base_url=self.base_url, zim_name=self.zim_name)[:count]
+ # Hydrate only a few hits. The Kiwix search snippet remains useful
+ # evidence for the rest, and an unreadable title alone is never cited.
+ result: list[tuple[KiwixHit, str]] = []
+ for hit in hits:
+ text = ""
+ if len(result) < 4:
+ try:
+ text = await self._read_article(client, hit.article_path)
+ except KiwixError:
+ pass
+ if text or hit.snippet:
+ # The first article characters can be unrelated to a hit
+ # thousands of words later. Book clips source text to 1200
+ # chars, so lead with Kiwix's matched search passage.
+ context = (
+ f"Search match: {hit.snippet}\n\nArticle opening: {text}"
+ if hit.snippet and text
+ else f"Search match: {hit.snippet}"
+ if hit.snippet
+ else text
+ )
+ result.append((hit, context))
+ return result
+
+ async def _read_article(
+ self,
+ client: httpx.AsyncClient,
+ article_path: str,
+ *,
+ max_chars: int = MAX_ARTICLE_CHARS,
+ ) -> str:
+ path = validate_article_path(article_path)
+ raw, mime = await self._get(
+ client,
+ f"raw/{quote(self.zim_name, safe='')}/content/{quote(path, safe='/')}",
+ limit=MAX_ARTICLE_BYTES,
+ )
+ if mime not in {"text/html", "application/xhtml+xml", "text/plain"}:
+ raise KiwixError("The selected ZIM entry is not a readable article.")
+ decoded = raw.decode("utf-8", errors="replace")
+ return html_to_text(decoded, max_chars=max_chars) if mime != "text/plain" else decoded.strip()[:max_chars]
+
+ async def read_article(self, article_path: str) -> str:
+ async with self._client() as client:
+ text = await self._read_article(client, article_path, max_chars=500_000)
+ if not text:
+ raise KiwixError("The selected ZIM article has no readable text.")
+ return text
diff --git a/deeptutor/services/rag/pipelines/kiwix/pipeline.py b/deeptutor/services/rag/pipelines/kiwix/pipeline.py
new file mode 100644
index 000000000..af09a7400
--- /dev/null
+++ b/deeptutor/services/rag/pipelines/kiwix/pipeline.py
@@ -0,0 +1,73 @@
+"""Retrieval-only RAG pipeline over an existing kiwix-serve ZIM archive."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from deeptutor.runtime.home import get_runtime_data_root
+from deeptutor.services.rag.provider_binding import load_kb_config_entry
+
+from .client import KiwixClient, KiwixError
+
+PROVIDER = "kiwix"
+
+
+class KiwixPipeline:
+ def __init__(self, kb_base_dir: str | None = None, *, client_factory=None, **_: Any) -> None:
+ self.kb_base_dir = kb_base_dir or str(get_runtime_data_root() / "knowledge_bases")
+ self._client_factory = client_factory or KiwixClient
+
+ async def search(self, query: str, kb_name: str, **kwargs: Any) -> dict[str, Any]:
+ entry = load_kb_config_entry(self.kb_base_dir, kb_name)
+ if entry.get("type") != PROVIDER:
+ return self._error(query, "This knowledge base is not connected to Kiwix.", "not_configured")
+ try:
+ client = self._client_factory(entry["server_url"], entry["zim_name"])
+ hits = await client.search(query, top_k=kwargs.get("top_k") or 5)
+ except (KiwixError, KeyError, TypeError, ValueError) as exc:
+ return self._error(query, str(exc), "retrieval_error")
+ sources = [
+ {
+ "title": hit.title,
+ "content": content,
+ "source": f"{entry.get('zim_title') or entry['zim_name']} / {hit.title}",
+ "id": f"{entry['zim_name']}:{hit.article_path}",
+ "chunk_id": f"{entry['zim_name']}:{hit.article_path}",
+ "article_path": hit.article_path,
+ "archive": entry["zim_name"],
+ }
+ for hit, content in hits
+ if content.strip()
+ ]
+ context = "\n\n".join(
+ f"[{index}] {source['source']}\n{source['content']}"
+ for index, source in enumerate(sources, start=1)
+ )
+ return {
+ "query": query,
+ "answer": context,
+ "content": context,
+ "sources": sources,
+ "provider": PROVIDER,
+ }
+
+ @staticmethod
+ def _error(query: str, message: str, error_type: str) -> dict[str, Any]:
+ return {
+ "query": query,
+ "answer": message,
+ "content": "",
+ "sources": [],
+ "provider": PROVIDER,
+ "error_type": error_type,
+ }
+
+ async def initialize(self, kb_name: str, file_paths: list[str], **kwargs: Any) -> bool:
+ raise RuntimeError("Kiwix owns this ZIM archive and its search index. Add content there.")
+
+ async def add_documents(self, kb_name: str, file_paths: list[str], **kwargs: Any) -> bool:
+ return await self.initialize(kb_name, file_paths, **kwargs)
+
+ async def delete(self, kb_name: str, **kwargs: Any) -> bool:
+ # The manager deletes only its pointer; never delete a user's archive.
+ return True
diff --git a/deeptutor_cli/kb.py b/deeptutor_cli/kb.py
index bbc4a6c7c..913e6b772 100644
--- a/deeptutor_cli/kb.py
+++ b/deeptutor_cli/kb.py
@@ -2,7 +2,7 @@
CLI Knowledge Base Command
===========================
-Manage llamaindex knowledge bases from the command line.
+Manage knowledge bases from the command line.
"""
from __future__ import annotations
@@ -182,6 +182,24 @@ def register(app: typer.Typer) -> None:
raise typer.Exit(code=1) from exc
console.print("[green]Knowledge base created successfully.[/]")
+ @app.command("connect-kiwix")
+ def kb_connect_kiwix(
+ name: str = typer.Argument(..., help="New KB name."),
+ server_url: str = typer.Option(..., "--server-url", help="kiwix-serve base URL."),
+ zim_name: str = typer.Option(..., "--zim-name", help="One ZIM name served by Kiwix."),
+ ) -> None:
+ """Connect an existing ZIM archive without copying or indexing it."""
+ from deeptutor.services.rag.pipelines.kiwix.client import KiwixClient
+
+ try:
+ client = KiwixClient(server_url, zim_name)
+ title = asyncio.run(client.probe())
+ _get_kb_manager().register_kiwix_kb(name, client.base_url, client.zim_name, zim_title=title)
+ except Exception as exc:
+ console.print(f"[red]Could not connect Kiwix archive: {exc}[/]")
+ raise typer.Exit(code=1) from exc
+ console.print(f"[green]Connected '{name}' to Kiwix archive '{title}'.[/]")
+
@app.command("add")
def kb_add(
name: str = typer.Argument(..., help="KB name."),
diff --git a/tests/services/rag/test_kiwix_pipeline.py b/tests/services/rag/test_kiwix_pipeline.py
new file mode 100644
index 000000000..be60a0d81
--- /dev/null
+++ b/tests/services/rag/test_kiwix_pipeline.py
@@ -0,0 +1,308 @@
+"""A connected Kiwix KB searches its original ZIM without an ingest step."""
+
+from __future__ import annotations
+
+import importlib
+from types import SimpleNamespace
+
+from fastapi import HTTPException
+import httpx
+import pytest
+
+from deeptutor.book.agents.source_explorer import SourceExplorer
+from deeptutor.book.kb_health import fingerprint_kb_documents
+from deeptutor.knowledge.manager import KnowledgeBaseManager
+from deeptutor.learning.models import TopicSource, TopicSourceKind
+from deeptutor.learning.topic_generation import _ground_knowledge_base_source
+from deeptutor.services.rag.pipelines.kiwix.client import (
+ KiwixClient,
+ KiwixError,
+ parse_catalog_xml,
+ parse_search_xml,
+)
+from deeptutor.services.rag.pipelines.kiwix.pipeline import KiwixPipeline
+
+SEARCH_XML = b"""<?xml version="1.0"?>
+<rss version="2.0"><channel>
+ <item><title>Jazz</title><link>/content/music/A/Jazz</link>
+ <description>Music <b>jazz</b> history</description></item>
+ <item><title>Other book</title><link>/content/private/A/Secret</link>
+ <description>must never be read</description></item>
+</channel></rss>"""
+
+
+def _transport(request: httpx.Request) -> httpx.Response:
+ if request.url.path == "/raw/music/meta/Title":
+ return httpx.Response(200, text="Music archive")
+ if request.url.path == "/search":
+ assert request.url.params["books.name"] == "music"
+ assert request.url.params["format"] == "xml"
+ return httpx.Response(200, content=SEARCH_XML, headers={"content-type": "application/xml"})
+ if request.url.path == "/raw/music/content/A/Jazz":
+ return httpx.Response(
+ 200,
+ text="<html><nav>menu</nav><main><h1>Jazz</h1><p>Improvised music.</p>"
+ "<script>secret()</script></main></html>",
+ headers={"content-type": "text/html"},
+ )
+ raise AssertionError(f"Unexpected Kiwix request: {request.url}")
+
+
+async def test_connected_kb_search_returns_readable_archive_sources(tmp_path) -> None:
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+ manager.register_kiwix_kb(
+ "offline-music", "http://kiwix.test", "music", zim_title="Music archive"
+ )
+ pipeline = KiwixPipeline(
+ str(tmp_path),
+ client_factory=lambda base, name: KiwixClient(
+ base, name, transport=httpx.MockTransport(_transport)
+ ),
+ )
+ result = await pipeline.search("jazz", "offline-music")
+
+ assert result["provider"] == "kiwix"
+ assert len(result["sources"]) == 1
+ assert result["sources"][0]["article_path"] == "A/Jazz"
+ assert result["sources"][0]["chunk_id"] == "music:A/Jazz"
+ assert result["sources"][0]["id"] == "music:A/Jazz"
+ assert result["sources"][0]["source"] == "Music archive / Jazz"
+ assert "Improvised music." in result["content"]
+ assert "secret" not in result["content"]
+ assert not (tmp_path / "offline-music").exists()
+ assert manager.get_metadata("offline-music")["zim_name"] == "music"
+ assert manager.get_info("offline-music")["metadata"]["zim_title"] == "Music archive"
+ first_fingerprint = fingerprint_kb_documents("offline-music", manager=manager)
+ assert first_fingerprint["zim-archive"].startswith("sha256:")
+ manager.config["knowledge_bases"]["offline-music"]["zim_name"] = "music-updated"
+ manager._save_config()
+ assert fingerprint_kb_documents("offline-music", manager=manager) != first_fingerprint
+
+
+async def test_probe_and_read_one_article_without_archive_copy() -> None:
+ client = KiwixClient("http://kiwix.test", "music", transport=httpx.MockTransport(_transport))
+ assert await client.probe() == "Music archive"
+ assert "Improvised music." in await client.read_article("A/Jazz")
+
+
+def test_search_xml_rejects_foreign_links_and_path_traversal() -> None:
+ xml = b"""<rss><channel>
+ <item><title>external</title><link>https://evil.test/content/music/A/X</link><description>x</description></item>
+ <item><title>escape</title><link>/content/music/../private/X</link><description>x</description></item>
+ <item><title>good</title><link>/content/music/A/Good</link><description>safe</description></item>
+ </channel></rss>"""
+ hits = parse_search_xml(xml, base_url="http://kiwix.test", zim_name="music")
+ assert [hit.article_path for hit in hits] == ["A/Good"]
+
+
+async def test_legacy_opds_discovers_exact_loaded_zim_names() -> None:
+ xml = b"""<feed xmlns="http://www.w3.org/2005/Atom">
+ <entry><title>Music Encyclopedia</title><name>music_2026</name></entry>
+ <entry><title>Python Docs</title><link type="text/html" href="/content/python_docs_2026" /></entry>
+ <entry><title>Versioned Book</title><name>book_alias</name>
+ <link type="text/html" href="/content/book_2026-09" /></entry>
+ <entry><title>Foreign</title><link type="text/html" href="https://other.test/content/private" /></entry>
+ </feed>"""
+
+ def catalog(request: httpx.Request) -> httpx.Response:
+ assert request.url.path == "/catalog/search"
+ assert request.url.params["q"] == "docs"
+ assert request.url.params["count"] == "50"
+ return httpx.Response(200, content=xml)
+
+ archives = await KiwixClient.list_archives(
+ "http://kiwix.test", "docs", transport=httpx.MockTransport(catalog)
+ )
+ assert [(item.zim_name, item.title) for item in archives] == [
+ ("music_2026", "Music Encyclopedia"),
+ ("python_docs_2026", "Python Docs"),
+ ("book_2026-09", "Versioned Book"),
+ ]
+ assert len(parse_catalog_xml(xml, base_url="http://kiwix.test")) == 3
+
+
+async def test_bad_server_response_has_no_citable_source(tmp_path) -> None:
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+ manager.register_kiwix_kb("offline-music", "http://kiwix.test", "music")
+ transport = httpx.MockTransport(lambda _: httpx.Response(500, text="broken"))
+ pipeline = KiwixPipeline(
+ str(tmp_path),
+ client_factory=lambda base, name: KiwixClient(base, name, transport=transport),
+ )
+ result = await pipeline.search("jazz", "offline-music")
+ assert result["error_type"] == "retrieval_error"
+ assert result["sources"] == []
+ assert result["content"] == ""
+
+
+async def test_late_match_stays_visible_when_book_clips_long_article(tmp_path) -> None:
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+ manager.register_kiwix_kb("long-book", "http://kiwix.test", "music", zim_title="Music archive")
+ xml = b"""<rss><channel><item><title>Long article</title>
+ <link>/content/music/A/Long</link>
+ <description>Rare passage about quantum lattice harmony.</description>
+ </item></channel></rss>"""
+
+ def transport(request: httpx.Request) -> httpx.Response:
+ if request.url.path == "/search":
+ return httpx.Response(200, content=xml)
+ if request.url.path == "/raw/music/content/A/Long":
+ return httpx.Response(
+ 200,
+ text=f"<main><p>{'ordinary introduction ' * 1000}</p>"
+ "<p>Rare passage about quantum lattice harmony.</p></main>",
+ headers={"content-type": "text/html"},
+ )
+ raise AssertionError(request.url)
+
+ pipeline = KiwixPipeline(
+ str(tmp_path),
+ client_factory=lambda base, name: KiwixClient(
+ base, name, transport=httpx.MockTransport(transport)
+ ),
+ )
+ result = await pipeline.search("quantum lattice", "long-book")
+ source = result["sources"][0]
+ assert source["id"] == source["chunk_id"] == "music:A/Long"
+ assert source["content"].startswith("Search match: Rare passage about quantum lattice harmony.")
+ assert "quantum lattice" in source["content"][:1200]
+ assert "quantum lattice" not in source["content"][1200:]
+
+
+def test_connection_rejects_invalid_names_and_urls() -> None:
+ with pytest.raises(KiwixError):
+ KiwixClient("file:///etc/passwd", "music")
+ with pytest.raises(KiwixError):
+ KiwixClient("http://kiwix.test", "../private")
+ with pytest.raises(KiwixError):
+ KiwixClient("http://kiwix.test:bad", "music")
+ with pytest.raises(KiwixError):
+ KiwixClient("http://kiwix.test/../admin", "music")
+
+
+async def test_article_fetch_rejects_redirects_and_unbounded_bodies() -> None:
+ redirect = KiwixClient(
+ "http://kiwix.test", "music",
+ transport=httpx.MockTransport(lambda _: httpx.Response(302, headers={"location": "http://other.test/"})),
+ )
+ with pytest.raises(KiwixError, match="HTTP 302"):
+ await redirect.read_article("A/Jazz")
+
+ oversized = KiwixClient(
+ "http://kiwix.test", "music",
+ transport=httpx.MockTransport(lambda _: httpx.Response(200, content=b"x" * 2_000_001, headers={"content-type": "text/plain"})),
+ )
+ with pytest.raises(KiwixError, match="size limit"):
+ await oversized.read_article("A/Jazz")
+
+
+async def test_book_and_mastery_use_same_access_checked_rag_search(tmp_path, monkeypatch) -> None:
+ access = importlib.import_module("deeptutor.multi_user.knowledge_access")
+ from deeptutor.tools.rag_tool import rag_search
+
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+ manager.register_kiwix_kb("offline-music", "http://kiwix.test", "music", zim_title="Music archive")
+ resource = SimpleNamespace(base_dir=tmp_path, name="offline-music", assigned=False)
+ monkeypatch.setattr(access, "resolve_for_rag", lambda _ref: resource)
+ monkeypatch.setattr(access, "resolve_kb", lambda _ref, **_kwargs: resource)
+ monkeypatch.setattr(access, "resolve_kb_metadata", lambda _ref: manager.get_metadata("offline-music"))
+ monkeypatch.setattr(access, "resolve_kb_manifest", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(KiwixClient, "_client", lambda _self: httpx.AsyncClient(transport=httpx.MockTransport(_transport)))
+
+ result = await rag_search("jazz", "offline-music")
+ assert result["provider"] == "kiwix"
+ assert result["sources"][0]["id"] == "music:A/Jazz"
+
+ explorer = object.__new__(SourceExplorer)
+ explorer.chunks_per_query = 4
+ explorer.skipped_knowledge_bases = []
+ chunks = await explorer._retrieve_kb_chunks(["jazz"], ["offline-music"])
+ assert len(chunks) == 1
+ assert chunks[0].ref == "music:A/Jazz"
+ assert chunks[0].text.startswith("Search match: Music jazz history")
+ assert "Improvised music." in chunks[0].text
+ assert explorer.skipped_knowledge_bases == []
+
+ source = TopicSource(
+ id="topic-source-1", kind=TopicSourceKind.KNOWLEDGE_BASE,
+ source_id="offline-music", label="Music archive",
+ )
+ grounded = await _ground_knowledge_base_source(source, query="jazz")
+ assert grounded.available
+ assert grounded.metadata["retrieval_provider"] == "kiwix"
+ assert "Improvised music" in grounded.excerpt
+
+ def forbidden(_ref):
+ raise HTTPException(status_code=403, detail="Not assigned")
+
+ monkeypatch.setattr(access, "resolve_for_rag", forbidden)
+ with pytest.raises(HTTPException) as denied:
+ await rag_search("jazz", "offline-music")
+ assert denied.value.status_code == 403
+
+
+async def test_reading_import_stores_only_selected_text_and_checks_kb_scope(tmp_path, monkeypatch) -> None:
+ reading = importlib.import_module("deeptutor.api.routers.reading")
+ access = importlib.import_module("deeptutor.multi_user.knowledge_access")
+ client_module = importlib.import_module("deeptutor.services.rag.pipelines.kiwix.client")
+ workspace_knowledge = importlib.import_module("deeptutor.services.workspace.knowledge")
+ workspace_resources = importlib.import_module("deeptutor.services.workspace.resources")
+ from deeptutor.reading import ReadingCatalogStore, ReadingStore
+
+ catalog = ReadingCatalogStore(tmp_path / "reading")
+ def scoped_catalog():
+ assert not workspace_knowledge.library_request.get()
+ return catalog
+
+ monkeypatch.setattr(reading, "_catalog", scoped_catalog)
+ monkeypatch.setattr(reading, "assert_learning_material", lambda *_args, **_kwargs: None)
+ resource = SimpleNamespace(id="account:kb:offline-music", name="offline-music")
+ monkeypatch.setattr(workspace_resources, "current_resources", lambda: SimpleNamespace(knowledge_bases=[]))
+
+ def library_resource(ref, **_kwargs):
+ assert ref == "account:kb:offline-music"
+ assert workspace_knowledge.library_request.get()
+ return resource
+
+ monkeypatch.setattr(workspace_knowledge, "resolve_qualified", library_resource)
+ monkeypatch.setattr(access, "manager_for_resource", lambda _resource: SimpleNamespace(get_metadata=lambda _name: {
+ "type": "kiwix", "server_url": "http://kiwix.test", "zim_name": "music"
+ }))
+
+ class FakeClient:
+ base_url = "http://kiwix.test"
+ zim_name = "music"
+
+ def __init__(self, *_args):
+ pass
+
+ async def read_article(self, path):
+ assert path == "A/Jazz"
+ return "Jazz\n\nImprovised music."
+
+ monkeypatch.setattr(client_module, "KiwixClient", FakeClient)
+ payload = reading.ZimArticleImportRequest(kb_ref="account:kb:offline-music", article_path="A/Jazz", title="Jazz")
+ result = await reading.import_zim_article(payload)
+ material_id = result["material"]["material_id"]
+ store = ReadingStore(catalog.root)
+ assert "Improvised music" in store.unit_text(material_id, 1)
+ assert list((catalog.root / material_id).glob("raw/*")) == []
+ assert result["workspace"]["workspace_id"]
+ assert result["material"]["source_url"] == "http://kiwix.test/content/music/A/Jazz"
+
+ def forbidden(_ref, **_kwargs):
+ raise HTTPException(status_code=403, detail="Not assigned")
+
+ monkeypatch.setattr(workspace_knowledge, "resolve_qualified", forbidden)
+ with pytest.raises(HTTPException) as denied:
+ await reading.import_zim_article(payload)
+ assert denied.value.status_code == 403
+ assert not workspace_knowledge.library_request.get()
diff --git a/web/components/knowledge/ConnectKiwixModal.tsx b/web/components/knowledge/ConnectKiwixModal.tsx
new file mode 100644
index 000000000..c49133ade
--- /dev/null
+++ b/web/components/knowledge/ConnectKiwixModal.tsx
@@ -0,0 +1,117 @@
+"use client";
+
+import { useState } from "react";
+import { useTranslation } from "react-i18next";
+import { Loader2, Server } from "lucide-react";
+import Modal from "@/components/common/Modal";
+import { listKiwixArchives, probeKiwix } from "@/features/knowledge/api/catalog";
+import type { KiwixArchive } from "@/features/knowledge/api/client";
+
+interface Props {
+ onClose: () => void;
+ onConnect: (params: { name: string; serverUrl: string; zimName: string }) => Promise<void>;
+}
+
+export default function ConnectKiwixModal({ onClose, onConnect }: Props) {
+ const { t } = useTranslation();
+ const [name, setName] = useState("");
+ const [serverUrl, setServerUrl] = useState("");
+ const [zimName, setZimName] = useState("");
+ const [verifiedTitle, setVerifiedTitle] = useState("");
+ const [catalogQuery, setCatalogQuery] = useState("");
+ const [archives, setArchives] = useState<KiwixArchive[]>([]);
+ const [catalogSearched, setCatalogSearched] = useState(false);
+ const [browsing, setBrowsing] = useState(false);
+ const [busy, setBusy] = useState<"probe" | "connect" | null>(null);
+ const [error, setError] = useState("");
+
+ const inputClass = "mt-1 w-full rounded-lg border border-[var(--border)] bg-[var(--background)] px-3 py-2 text-[13px] text-[var(--foreground)] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[var(--ring)]";
+ const fieldChanged = () => { setVerifiedTitle(""); setError(""); };
+ const browse = async () => {
+ setBrowsing(true);
+ setError("");
+ setCatalogSearched(false);
+ try {
+ setArchives(await listKiwixArchives(serverUrl.trim(), catalogQuery.trim()));
+ setCatalogSearched(true);
+ } catch (cause) {
+ setArchives([]);
+ setError(cause instanceof Error ? cause.message : t("Could not browse Kiwix archives"));
+ } finally {
+ setBrowsing(false);
+ }
+ };
+ const probe = async () => {
+ setBusy("probe");
+ setError("");
+ try {
+ const result = await probeKiwix({ serverUrl: serverUrl.trim(), zimName: zimName.trim() });
+ setVerifiedTitle(result.title);
+ } catch (cause) {
+ setVerifiedTitle("");
+ setError(cause instanceof Error ? cause.message : t("Could not reach Kiwix archive"));
+ } finally {
+ setBusy(null);
+ }
+ };
+ const connect = async () => {
+ setBusy("connect");
+ setError("");
+ try {
+ await onConnect({ name: name.trim(), serverUrl: serverUrl.trim(), zimName: zimName.trim() });
+ onClose();
+ } catch (cause) {
+ setError(cause instanceof Error ? cause.message : t("Could not connect Kiwix archive"));
+ } finally {
+ setBusy(null);
+ }
+ };
+
+ return (
+ <Modal isOpen onClose={onClose} title={t("Connect Kiwix archive")} titleIcon={<Server className="h-4 w-4" />} footer={
+ <div className="flex justify-end gap-2">
+ <button type="button" onClick={onClose} disabled={!!busy} className="rounded-lg px-3 py-2 text-[13px] text-[var(--muted-foreground)]">{t("Cancel")}</button>
+ <button type="button" onClick={() => void connect()} disabled={!!busy || !name.trim() || !serverUrl.trim() || !zimName.trim()} className="rounded-lg bg-[var(--primary)] px-3 py-2 text-[13px] font-medium text-[var(--primary-foreground)] disabled:opacity-50">
+ {busy === "connect" && <Loader2 className="mr-1 inline h-3.5 w-3.5 animate-spin" />}{t("Connect archive")}
+ </button>
+ </div>
+ }>
+ <div className="space-y-4 p-4">
+ <p className="text-[12px] text-[var(--muted-foreground)]">{t("Each knowledge base connects one searchable ZIM on one kiwix-serve instance. Connect other archives or servers separately. DeepTutor reads matching articles on demand.")}</p>
+ <label className="block text-[12px] font-medium">{t("Knowledge base name")}
+ <input value={name} onChange={(event) => { setName(event.target.value); setError(""); }} className={inputClass} placeholder={t("my-archive")} data-autofocus />
+ </label>
+ <label className="block text-[12px] font-medium">{t("Kiwix server URL")}
+ <input value={serverUrl} onChange={(event) => { setServerUrl(event.target.value); setZimName(""); setArchives([]); setCatalogSearched(false); fieldChanged(); }} className={inputClass} placeholder={t("http://localhost:8080")} />
+ </label>
+ <div className="flex gap-2">
+ <label className="min-w-0 flex-1 text-[12px] font-medium">{t("Find archive")}
+ <input value={catalogQuery} onChange={(event) => setCatalogQuery(event.target.value)} className={inputClass} placeholder={t("Search loaded archives")} />
+ </label>
+ <button type="button" onClick={() => void browse()} disabled={!serverUrl.trim() || !!busy || browsing} className="mt-5 shrink-0 rounded-lg border border-[var(--border)] px-3 py-2 text-[12px] font-medium disabled:opacity-50">
+ {browsing && <Loader2 className="mr-1 inline h-3.5 w-3.5 animate-spin" />}{t("Browse archives")}
+ </button>
+ </div>
+ {archives.length > 0 && (
+ <label className="block text-[12px] font-medium">{t("Loaded archives")}
+ <select value={archives.some((archive) => archive.zim_name === zimName) ? zimName : ""} onChange={(event) => { setZimName(event.target.value); fieldChanged(); }} className={inputClass}>
+ <option value="">{t("Choose archive")}</option>
+ {archives.map((archive) => <option key={archive.zim_name} value={archive.zim_name}>{archive.title} ({archive.zim_name})</option>)}
+ </select>
+ </label>
+ )}
+ {catalogSearched && archives.length === 0 && <p className="text-[11px] text-[var(--muted-foreground)]">{t("No archives returned. Enter the ZIM name manually or search by title.")}</p>}
+ <label className="block text-[12px] font-medium">{t("ZIM name")}
+ <input value={zimName} onChange={(event) => { setZimName(event.target.value); fieldChanged(); }} className={inputClass} placeholder={t("wikipedia_en_all")} />
+ </label>
+ <div className="flex items-center gap-3">
+ <button type="button" onClick={() => void probe()} disabled={!!busy || !serverUrl.trim() || !zimName.trim()} className="rounded-lg border border-[var(--border)] px-3 py-2 text-[12px] font-medium disabled:opacity-50">
+ {busy === "probe" && <Loader2 className="mr-1 inline h-3.5 w-3.5 animate-spin" />}{t("Check connection")}
+ </button>
+ {verifiedTitle && <span className="min-w-0 truncate text-[12px] text-emerald-600">{t("Ready")}: {verifiedTitle}</span>}
+ </div>
+ {error && <p role="alert" className="text-[12px] text-red-600">{error}</p>}
+ </div>
+ </Modal>
+ );
+}
diff --git a/web/components/knowledge/KbKiwixArticlesSection.tsx b/web/components/knowledge/KbKiwixArticlesSection.tsx
new file mode 100644
index 000000000..d70319fa4
--- /dev/null
+++ b/web/components/knowledge/KbKiwixArticlesSection.tsx
@@ -0,0 +1,90 @@
+"use client";
+
+import { useState, type FormEvent } from "react";
+import { useRouter } from "next/navigation";
+import { useTranslation } from "react-i18next";
+import { BookOpen, Loader2, Search } from "lucide-react";
+import { importKiwixArticle, searchKiwixArticles, type KiwixArticle } from "@/features/knowledge/api/client";
+import { knowledgeBaseRef, type KnowledgeBase } from "@/lib/knowledge-helpers";
+import { readingCollectionRoute } from "@/lib/learning-routes";
+
+export default function KbKiwixArticlesSection({ kb }: { kb: KnowledgeBase }) {
+ const { t } = useTranslation();
+ const router = useRouter();
+ const [query, setQuery] = useState("");
+ const [results, setResults] = useState<KiwixArticle[]>([]);
+ const [searched, setSearched] = useState(false);
+ const [searching, setSearching] = useState(false);
+ const [openingPath, setOpeningPath] = useState("");
+ const [error, setError] = useState("");
+
+ const search = async (event: FormEvent<HTMLFormElement>) => {
+ event.preventDefault();
+ if (!query.trim()) return;
+ setSearching(true);
+ setError("");
+ setSearched(false);
+ try {
+ setResults(await searchKiwixArticles(knowledgeBaseRef(kb), query.trim()));
+ setSearched(true);
+ } catch (cause) {
+ setResults([]);
+ setError(cause instanceof Error ? cause.message : t("Kiwix search failed"));
+ } finally {
+ setSearching(false);
+ }
+ };
+
+ const openArticle = async (article: KiwixArticle) => {
+ setOpeningPath(article.article_path);
+ setError("");
+ try {
+ const result = await importKiwixArticle({
+ kbRef: knowledgeBaseRef(kb),
+ articlePath: article.article_path,
+ title: article.title,
+ });
+ router.push(readingCollectionRoute(result.workspace.workspace_id));
+ } catch (cause) {
+ setError(cause instanceof Error ? cause.message : t("Could not open Kiwix article"));
+ } finally {
+ setOpeningPath("");
+ }
+ };
+
+ return (
+ <section className="space-y-4">
+ <div>
+ <h2 className="text-[15px] font-semibold">{kb.metadata?.zim_title || t("Kiwix archive")}</h2>
+ <p className="mt-1 text-[12px] text-[var(--muted-foreground)]">
+ {t("Search articles in this ZIM. DeepTutor retrieves matches for chat, Book, and Guided Learning without importing the archive.")}
+ </p>
+ <p className="mt-1 text-[11px] text-[var(--muted-foreground)]">{kb.metadata?.zim_name}</p>
+ </div>
+ <form onSubmit={(event) => void search(event)} className="flex gap-2">
+ <label className="sr-only" htmlFor="kiwix-article-query">{t("Search articles")}</label>
+ <input id="kiwix-article-query" value={query} onChange={(event) => setQuery(event.target.value)} placeholder={t("Search articles")} className="min-w-0 flex-1 rounded-lg border border-[var(--border)] bg-[var(--background)] px-3 py-2 text-[13px]" />
+ <button type="submit" disabled={!query.trim() || searching} className="inline-flex items-center gap-1 rounded-lg bg-[var(--primary)] px-3 py-2 text-[13px] font-medium text-[var(--primary-foreground)] disabled:opacity-50">
+ {searching ? <Loader2 className="h-4 w-4 animate-spin" /> : <Search className="h-4 w-4" />}{t("Search")}
+ </button>
+ </form>
+ {error && <p role="alert" className="text-[12px] text-red-600">{error}</p>}
+ {searched && results.length === 0 && <p className="text-[12px] text-[var(--muted-foreground)]">{t("No readable articles found.")}</p>}
+ <ul className="divide-y divide-[var(--border)] rounded-xl border border-[var(--border)]">
+ {results.map((article) => (
+ <li key={article.article_path} className="p-3">
+ <div className="flex items-start justify-between gap-3">
+ <div className="min-w-0">
+ <p className="text-[13px] font-medium">{article.title}</p>
+ <p className="mt-1 line-clamp-3 text-[11px] text-[var(--muted-foreground)]">{article.excerpt}</p>
+ </div>
+ <button type="button" disabled={!!openingPath} onClick={() => void openArticle(article)} className="inline-flex shrink-0 items-center gap-1 rounded-lg border border-[var(--border)] px-2.5 py-1.5 text-[12px] font-medium disabled:opacity-50">
+ {openingPath === article.article_path ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <BookOpen className="h-3.5 w-3.5" />}{t("Read article")}
+ </button>
+ </div>
+ </li>
+ ))}
+ </ul>
+ </section>
+ );
+}
diff --git a/web/components/knowledge/KnowledgeBaseDetail.tsx b/web/components/knowledge/KnowledgeBaseDetail.tsx
index 5e363bdd4..a287e612a 100644
--- a/web/components/knowledge/KnowledgeBaseDetail.tsx
+++ b/web/components/knowledge/KnowledgeBaseDetail.tsx
@@ -50,6 +50,7 @@ import KbGitHubSourcesSection from "./KbGitHubSourcesSection";
import KbLinkedFoldersSection from "./KbLinkedFoldersSection";
import KbWebSourcesSection from "./KbWebSourcesSection";
import KbMarginNoteDevicesSection from "./KbMarginNoteDevicesSection";
+import KbKiwixArticlesSection from "./KbKiwixArticlesSection";
import KnowledgeEngineIcon, {
knowledgeSourceIconId,
} from "./KnowledgeEngineIcon";
@@ -97,6 +98,7 @@ const SECTION_CHROME: Record<
web: { label: "Web", Icon: Globe },
versions: { label: "Index versions", Icon: Layers },
devices: { label: "Devices", Icon: Smartphone },
+ kiwix: { label: "Articles", Icon: FileText },
settings: { label: "Settings", Icon: SettingsIcon },
};
@@ -155,11 +157,13 @@ export default function KnowledgeBaseDetail({
const isMarginNote = isMarginNoteKb(kb);
// A MarginNote library records no engine and no embedding: defaulting to
// "llamaindex · Default embedding" here described a pipeline it never runs.
- const provider = isMarginNote
+ const provider = kb.metadata?.type === "kiwix"
+ ? t("Kiwix / ZIM")
+ : isMarginNote
? t("MarginNote 4")
: kb.statistics?.rag_provider || "llamaindex";
const pageIndexProvider =
- isMarginNote || !providerUsesEmbeddingMetadata(provider);
+ isMarginNote || kb.metadata?.type === "kiwix" || !providerUsesEmbeddingMetadata(provider);
const embeddingLabel = meta.embedding_model
? typeof meta.embedding_dim === "number"
? `${meta.embedding_model} · ${meta.embedding_dim}${t("d")}`
@@ -174,7 +178,7 @@ export default function KnowledgeBaseDetail({
task.executing === true;
const status = resolveKbStatus(kb);
// Nothing to re-run: its content arrives from the add-on, not an index.
- const canRetry = status === "error" && !kb.read_only && !isMarginNote;
+ const canRetry = status === "error" && !kb.read_only && !isMarginNote && kb.metadata?.type !== "kiwix";
const handleRetry = async () => {
if (!canRetry || retrySubmitting || isReindexingLocally) return;
@@ -369,6 +373,9 @@ export default function KnowledgeBaseDetail({
kb={kb}
/>
)}
+ {activeSection === "kiwix" && (
+ <KbKiwixArticlesSection key={knowledgeBaseRef(kb)} kb={kb} />
+ )}
{activeSection === "settings" && (
<KbSettingsSection
kb={kb}
diff --git a/web/components/knowledge/KnowledgeHome.tsx b/web/components/knowledge/KnowledgeHome.tsx
index 5c5629e84..33c6ebb2e 100644
--- a/web/components/knowledge/KnowledgeHome.tsx
+++ b/web/components/knowledge/KnowledgeHome.tsx
@@ -39,7 +39,7 @@ interface KnowledgeHomeProps {
providers: RagProviderSummary[];
onOpenKb: (name: string) => void;
onOpenEngine: (id: string) => void;
- onOpenSource: (id: "obsidian" | "marginnote4") => void;
+ onOpenSource: (id: "obsidian" | "marginnote4" | "kiwix") => void;
onCreate: () => void;
activeSection: KnowledgeHomeSection;
onSectionChange: (section: KnowledgeHomeSection) => void;
@@ -156,6 +156,13 @@ export default function KnowledgeHome({
action: t("Connect library"),
count: kbs.filter((kb) => kb.metadata?.type === "marginnote4").length,
},
+ {
+ id: "kiwix" as const,
+ name: t("Kiwix / ZIM"),
+ description: t("Search an existing ZIM archive through kiwix-serve. Articles stay on the server."),
+ action: t("Connect archive"),
+ count: kbs.filter((kb) => kb.metadata?.type === "kiwix").length,
+ },
],
[kbs, t],
);
diff --git a/web/components/knowledge/KnowledgePage.tsx b/web/components/knowledge/KnowledgePage.tsx
index 38fde420c..2a101ad03 100644
--- a/web/components/knowledge/KnowledgePage.tsx
+++ b/web/components/knowledge/KnowledgePage.tsx
@@ -44,6 +44,7 @@ const EngineDetail = dynamic(
},
);
const CreateKbModal = dynamic(() => import("./CreateKbModal"));
+const ConnectKiwixModal = dynamic(() => import("./ConnectKiwixModal"));
export default function KnowledgePage() {
const { t } = useTranslation();
@@ -83,6 +84,7 @@ export default function KnowledgePage() {
connectWeKnora,
connectMarginNote4,
connectIma,
+ connectKiwix,
} = useKnowledgeBases();
// Connected subagents are stored as ``type: subagent`` KBs so the chat
@@ -102,6 +104,7 @@ export default function KnowledgePage() {
const [homeSection, setHomeSection] =
useState<KnowledgeHomeSection>(initialHomeSection);
const [createOpen, setCreateOpen] = useState(false);
+ const [kiwixOpen, setKiwixOpen] = useState(false);
const [createPreset, setCreatePreset] = useState<{
mode: "new" | "link";
source?: string;
@@ -111,7 +114,11 @@ export default function KnowledgePage() {
setCreatePreset(null);
setCreateOpen(true);
}, []);
- const openSource = useCallback((source: "obsidian" | "marginnote4") => {
+ const openSource = useCallback((source: "obsidian" | "marginnote4" | "kiwix") => {
+ if (source === "kiwix") {
+ setKiwixOpen(true);
+ return;
+ }
setCreatePreset({ mode: "link", source });
setCreateOpen(true);
}, []);
@@ -464,6 +471,15 @@ export default function KnowledgePage() {
}}
/>
) : null}
+ {kiwixOpen ? (
+ <ConnectKiwixModal
+ onClose={() => setKiwixOpen(false)}
+ onConnect={async (params) => {
+ await connectKiwix(params);
+ openKb(`account:kb:${params.name}`);
+ }}
+ />
+ ) : null}
</div>
);
}
diff --git a/web/features/knowledge/api/catalog.ts b/web/features/knowledge/api/catalog.ts
index b6818cedd..c32737041 100644
--- a/web/features/knowledge/api/catalog.ts
+++ b/web/features/knowledge/api/catalog.ts
@@ -1,5 +1,6 @@
export {
connectImaKnowledgeBase,
+ connectKiwix,
connectLinkedFolder,
connectMarginNote4Library,
connectObsidianVault,
@@ -12,6 +13,7 @@ export {
listKnowledgeBases,
listRagProviders,
probeImaKnowledgeBase,
+ probeKiwix,
probeLinkedFolder,
probeWeKnora,
readErrorDetail,
@@ -19,6 +21,9 @@ export {
getReindexConfig,
type LightRagRebuildConfig,
retryKnowledgeBase,
+ searchKiwixArticles,
+ importKiwixArticle,
+ listKiwixArchives,
setDefaultKnowledgeBase,
} from "./client";
diff --git a/web/features/knowledge/api/client.ts b/web/features/knowledge/api/client.ts
index d07a4c8ca..7a2687d47 100644
--- a/web/features/knowledge/api/client.ts
+++ b/web/features/knowledge/api/client.ts
@@ -798,6 +798,82 @@ export async function createKnowledgeBase(payload: {
return (await res.json()) as KnowledgeTaskResponse;
}
+export interface KiwixArticle {
+ title: string;
+ article_path: string;
+ excerpt: string;
+}
+
+export interface KiwixArchive {
+ zim_name: string;
+ title: string;
+}
+
+export async function listKiwixArchives(serverUrl: string, query = ""): Promise<KiwixArchive[]> {
+ const path = `/api/knowledge-bases/kiwix-catalog?server_url=${encodeURIComponent(serverUrl)}&q=${encodeURIComponent(query)}`;
+ const res = await apiFetch(apiUrl(path), { cache: "no-store" });
+ if (!res.ok) throw new Error(await readErrorDetail(res, "Could not browse Kiwix archives"));
+ const body = (await res.json()) as { archives: KiwixArchive[] };
+ return body.archives;
+}
+
+export async function probeKiwix(payload: {
+ serverUrl: string;
+ zimName: string;
+}): Promise<{ ok: boolean; title: string; zim_name: string }> {
+ const res = await apiFetch(apiUrl("/api/knowledge-bases/probe-kiwix"), {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ server_url: payload.serverUrl, zim_name: payload.zimName }),
+ });
+ if (!res.ok) throw new Error(await readErrorDetail(res, "Could not reach Kiwix archive"));
+ return (await res.json()) as { ok: boolean; title: string; zim_name: string };
+}
+
+export async function connectKiwix(payload: {
+ name: string;
+ serverUrl: string;
+ zimName: string;
+}): Promise<void> {
+ const res = await apiFetch(apiUrl("/api/knowledge-bases/connect-kiwix"), {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ name: payload.name,
+ server_url: payload.serverUrl,
+ zim_name: payload.zimName,
+ }),
+ });
+ if (!res.ok) throw new Error(await readErrorDetail(res, "Could not connect Kiwix archive"));
+ invalidateKnowledgeCaches();
+}
+
+export async function searchKiwixArticles(kbRef: string, query: string): Promise<KiwixArticle[]> {
+ const path = `/api/knowledge-bases/kiwix-articles?kb_ref=${encodeURIComponent(kbRef)}&q=${encodeURIComponent(query)}`;
+ const res = await apiFetch(apiUrl(path), { cache: "no-store" });
+ if (!res.ok) throw new Error(await readErrorDetail(res, "Kiwix search failed"));
+ const body = (await res.json()) as { articles: KiwixArticle[] };
+ return body.articles;
+}
+
+export async function importKiwixArticle(payload: {
+ kbRef: string;
+ articlePath: string;
+ title: string;
+}): Promise<{ workspace: { workspace_id: string } }> {
+ const res = await apiFetch(baseApiUrl("/api/reading/library/import-zim-article"), {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ kb_ref: payload.kbRef,
+ article_path: payload.articlePath,
+ title: payload.title,
+ }),
+ });
+ if (!res.ok) throw new Error(await readErrorDetail(res, "Could not open Kiwix article"));
+ return (await res.json()) as { workspace: { workspace_id: string } };
+}
+
export async function connectObsidianVault(payload: {
name: string;
vaultPath: string;
diff --git a/web/hooks/useKnowledgeBases.ts b/web/hooks/useKnowledgeBases.ts
index 2376cefb3..be56d35a7 100644
--- a/web/hooks/useKnowledgeBases.ts
+++ b/web/hooks/useKnowledgeBases.ts
@@ -6,6 +6,7 @@ import type { EmbeddingModelSelection } from "@/features/knowledge/model/types";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
connectImaKnowledgeBase as connectImaApi,
+ connectKiwix as connectKiwixApi,
connectWeKnora as connectWeKnoraApi,
connectLinkedFolder as connectLinkedFolderApi,
connectMarginNote4Library as connectMarginNote4Api,
@@ -455,6 +456,15 @@ export function useKnowledgeBases() {
[load],
);
+ const connectKiwix = useCallback(
+ async (params: { name: string; serverUrl: string; zimName: string }) => {
+ await connectKiwixApi(params);
+ invalidateKnowledgeCaches();
+ await load({ force: true, showSpinner: false });
+ },
+ [load],
+ );
+
return {
kbs: combinedKbs,
rawKbs: kbs,
@@ -484,6 +494,7 @@ export function useKnowledgeBases() {
connectWeKnora,
connectMarginNote4,
connectIma,
+ connectKiwix,
};
}
diff --git a/web/lib/knowledge-helpers.ts b/web/lib/knowledge-helpers.ts
index 2b47e9350..8c4ceff21 100644
--- a/web/lib/knowledge-helpers.ts
+++ b/web/lib/knowledge-helpers.ts
@@ -186,6 +186,10 @@ export interface KnowledgeBase {
vault_path?: string;
/** SQLite store of a connected MarginNote 4 library (when type === "marginnote4"). */
db_path?: string;
+ /** Connected Kiwix archive; article bytes remain on the configured server. */
+ server_url?: string;
+ zim_name?: string;
+ zim_title?: string;
/** Backend of a connected subagent (when type === "subagent"): "claude_code" | "codex" | "antigravity" | "kimi" | "opencode" | "mimo" | "hermes" | "openclaw" | "deepseek_harness" | "partner". */
agent_kind?: string;
/** Bound partner id when agent_kind === "partner". */
@@ -332,6 +336,7 @@ export const KB_DETAIL_SECTIONS = [
"web",
"versions",
"devices",
+ "kiwix",
"settings",
] as const;
@@ -347,8 +352,10 @@ export type KbDetailSection = (typeof KB_DETAIL_SECTIONS)[number];
export const kbDetailSections = (kb: KnowledgeBase): KbDetailSection[] =>
isMarginNoteKb(kb)
? ["devices", "settings"]
+ : kb.metadata?.type === "kiwix"
+ ? ["kiwix", "settings"]
: KB_DETAIL_SECTIONS.filter(
- (section) => section !== "devices" && (section !== "folders" || !kb.metadata?.type),
+ (section) => section !== "devices" && section !== "kiwix" && (section !== "folders" || !kb.metadata?.type),
);
/** Local source folders belong to ordinary, DeepTutor-managed indexed KBs. */
@@ -359,6 +366,7 @@ export const kbSupportsLinkedFolders = (kb: KnowledgeBase): boolean =>
export const kbProvider = (kb: KnowledgeBase): string => {
if (kb.metadata?.type === "obsidian") return "obsidian";
if (isMarginNoteKb(kb)) return MARGINNOTE4_KB_TYPE;
+ if (kb.metadata?.type === "kiwix") return "kiwix";
return (
(kb.statistics?.rag_provider as string | undefined) ||
(kb.metadata?.rag_provider as string | undefined) ||
@@ -368,6 +376,7 @@ export const kbProvider = (kb: KnowledgeBase): string => {
/** Source-document count for a KB, or null when unknown. */
export const kbDocCount = (kb: KnowledgeBase): number | null => {
+ if (kb.metadata?.type === "kiwix") return null;
const raw = kb.statistics?.raw_documents;
if (typeof raw === "number") return raw;
const indexed = kb.metadata?.last_indexed_count;
diff --git a/web/locales/de/app.json b/web/locales/de/app.json
index 2b7cbcf41..c11be2618 100644
--- a/web/locales/de/app.json
+++ b/web/locales/de/app.json
@@ -5235,5 +5235,33 @@
"kanban.restore": "Wiederherstellen",
"kanban.emptyArchive": "Keine archivierten Aufgaben",
"This unsent message could not be saved in your browser. Copy it before leaving this page.": "Diese nicht gesendete Nachricht konnte nicht im Browser gespeichert werden. Kopiere sie, bevor du diese Seite verlässt.",
- "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Einige nicht gesendete Nachrichten konnten nicht in diese Unterhaltung übernommen werden. Öffne einen neuen Chat, um sie wiederherzustellen."
+ "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Einige nicht gesendete Nachrichten konnten nicht in diese Unterhaltung übernommen werden. Öffne einen neuen Chat, um sie wiederherzustellen.",
+ "Kiwix / ZIM": "Kiwix / ZIM",
+ "Kiwix archive": "Kiwix-Archiv",
+ "Kiwix server URL": "Kiwix-Server-URL",
+ "ZIM name": "ZIM-Name",
+ "Connect Kiwix archive": "Kiwix-Archiv verbinden",
+ "Connect archive": "Archiv verbinden",
+ "Check connection": "Verbindung prüfen",
+ "Could not reach Kiwix archive": "Kiwix-Archiv ist nicht erreichbar",
+ "Could not connect Kiwix archive": "Kiwix-Archiv konnte nicht verbunden werden",
+ "Kiwix search failed": "Kiwix-Suche fehlgeschlagen",
+ "Could not open Kiwix article": "Kiwix-Artikel konnte nicht geöffnet werden",
+ "Search articles": "Artikel suchen",
+ "Read article": "Artikel lesen",
+ "No readable articles found.": "Keine lesbaren Artikel gefunden.",
+ "Search an existing ZIM archive through kiwix-serve. Articles stay on the server.": "Ein bestehendes ZIM-Archiv über kiwix-serve durchsuchen. Die Artikel bleiben auf dem Server.",
+ "Connect to a searchable ZIM already served by kiwix-serve. DeepTutor reads matching articles on demand and does not import the archive.": "Ein durch kiwix-serve durchsuchbares ZIM-Archiv verbinden. DeepTutor liest passende Artikel bei Bedarf und importiert das Archiv nicht.",
+ "Search articles in this ZIM. DeepTutor retrieves matches for chat, Book, and Guided Learning without importing the archive.": "Artikel in diesem ZIM durchsuchen. DeepTutor ruft Treffer für Chat, Book und geführtes Lernen ab, ohne das Archiv zu importieren.",
+ "http://localhost:8080": "http://localhost:8080",
+ "my-archive": "my-archive",
+ "wikipedia_en_all": "wikipedia_en_all",
+ "Browse archives": "Archive durchsuchen",
+ "Choose archive": "Archiv auswählen",
+ "Could not browse Kiwix archives": "Kiwix-Archive konnten nicht durchsucht werden",
+ "Find archive": "Archiv finden",
+ "Loaded archives": "Geladene Archive",
+ "Search loaded archives": "Geladene Archive durchsuchen",
+ "No archives returned. Enter the ZIM name manually or search by title.": "Keine Archive gefunden. Gib den ZIM-Namen manuell ein oder suche nach dem Titel.",
+ "Each knowledge base connects one searchable ZIM on one kiwix-serve instance. Connect other archives or servers separately. DeepTutor reads matching articles on demand.": "Jede Wissensdatenbank verbindet ein durchsuchbares ZIM auf einer kiwix-serve-Instanz. Verbinde weitere Archive oder Server einzeln. DeepTutor liest passende Artikel bei Bedarf."
}
diff --git a/web/locales/en/app.json b/web/locales/en/app.json
index 833df31c3..65baf8df2 100644
--- a/web/locales/en/app.json
+++ b/web/locales/en/app.json
@@ -5235,5 +5235,33 @@
"kanban.restore": "Restore",
"kanban.emptyArchive": "No archived tasks",
"This unsent message could not be saved in your browser. Copy it before leaving this page.": "This unsent message could not be saved in your browser. Copy it before leaving this page.",
- "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Some unsent messages could not be moved into this conversation. Open a new chat to recover them."
+ "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.",
+ "Kiwix / ZIM": "Kiwix / ZIM",
+ "Kiwix archive": "Kiwix archive",
+ "Kiwix server URL": "Kiwix server URL",
+ "ZIM name": "ZIM name",
+ "Connect Kiwix archive": "Connect Kiwix archive",
+ "Connect archive": "Connect archive",
+ "Check connection": "Check connection",
+ "Could not reach Kiwix archive": "Could not reach Kiwix archive",
+ "Could not connect Kiwix archive": "Could not connect Kiwix archive",
+ "Kiwix search failed": "Kiwix search failed",
+ "Could not open Kiwix article": "Could not open Kiwix article",
+ "Search articles": "Search articles",
+ "Read article": "Read article",
+ "No readable articles found.": "No readable articles found.",
+ "Search an existing ZIM archive through kiwix-serve. Articles stay on the server.": "Search an existing ZIM archive through kiwix-serve. Articles stay on the server.",
+ "Connect to a searchable ZIM already served by kiwix-serve. DeepTutor reads matching articles on demand and does not import the archive.": "Connect to a searchable ZIM already served by kiwix-serve. DeepTutor reads matching articles on demand and does not import the archive.",
+ "Search articles in this ZIM. DeepTutor retrieves matches for chat, Book, and Guided Learning without importing the archive.": "Search articles in this ZIM. DeepTutor retrieves matches for chat, Book, and Guided Learning without importing the archive.",
+ "http://localhost:8080": "http://localhost:8080",
+ "my-archive": "my-archive",
+ "wikipedia_en_all": "wikipedia_en_all",
+ "Browse archives": "Browse archives",
+ "Choose archive": "Choose archive",
+ "Could not browse Kiwix archives": "Could not browse Kiwix archives",
+ "Find archive": "Find archive",
+ "Loaded archives": "Loaded archives",
+ "Search loaded archives": "Search loaded archives",
+ "No archives returned. Enter the ZIM name manually or search by title.": "No archives returned. Enter the ZIM name manually or search by title.",
+ "Each knowledge base connects one searchable ZIM on one kiwix-serve instance. Connect other archives or servers separately. DeepTutor reads matching articles on demand.": "Each knowledge base connects one searchable ZIM on one kiwix-serve instance. Connect other archives or servers separately. DeepTutor reads matching articles on demand."
}
diff --git a/web/locales/fr/app.json b/web/locales/fr/app.json
index 5859ca4be..a115873c5 100644
--- a/web/locales/fr/app.json
+++ b/web/locales/fr/app.json
@@ -5679,5 +5679,33 @@
"kanban.restore": "Restaurer",
"kanban.emptyArchive": "Aucune tâche archivée",
"This unsent message could not be saved in your browser. Copy it before leaving this page.": "Ce message non envoyé n’a pas pu être enregistré dans votre navigateur. Copiez-le avant de quitter cette page.",
- "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Certains messages non envoyés n’ont pas pu être transférés dans cette conversation. Ouvrez une nouvelle discussion pour les récupérer."
+ "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Certains messages non envoyés n’ont pas pu être transférés dans cette conversation. Ouvrez une nouvelle discussion pour les récupérer.",
+ "Kiwix / ZIM": "Kiwix / ZIM",
+ "Kiwix archive": "Archive Kiwix",
+ "Kiwix server URL": "URL du serveur Kiwix",
+ "ZIM name": "Nom du ZIM",
+ "Connect Kiwix archive": "Connecter une archive Kiwix",
+ "Connect archive": "Connecter l’archive",
+ "Check connection": "Vérifier la connexion",
+ "Could not reach Kiwix archive": "Archive Kiwix inaccessible",
+ "Could not connect Kiwix archive": "Impossible de connecter l’archive Kiwix",
+ "Kiwix search failed": "La recherche Kiwix a échoué",
+ "Could not open Kiwix article": "Impossible d’ouvrir l’article Kiwix",
+ "Search articles": "Rechercher des articles",
+ "Read article": "Lire l’article",
+ "No readable articles found.": "Aucun article lisible trouvé.",
+ "Search an existing ZIM archive through kiwix-serve. Articles stay on the server.": "Rechercher dans une archive ZIM existante via kiwix-serve. Les articles restent sur le serveur.",
+ "Connect to a searchable ZIM already served by kiwix-serve. DeepTutor reads matching articles on demand and does not import the archive.": "Connectez une archive ZIM interrogeable via kiwix-serve. DeepTutor lit les articles correspondants à la demande sans importer l’archive.",
+ "Search articles in this ZIM. DeepTutor retrieves matches for chat, Book, and Guided Learning without importing the archive.": "Recherchez des articles dans ce ZIM. DeepTutor récupère les résultats pour le chat, Book et l’apprentissage guidé sans importer l’archive.",
+ "http://localhost:8080": "http://localhost:8080",
+ "my-archive": "my-archive",
+ "wikipedia_en_all": "wikipedia_en_all",
+ "Browse archives": "Parcourir les archives",
+ "Choose archive": "Choisir une archive",
+ "Could not browse Kiwix archives": "Impossible de parcourir les archives Kiwix",
+ "Find archive": "Trouver une archive",
+ "Loaded archives": "Archives chargées",
+ "Search loaded archives": "Rechercher dans les archives chargées",
+ "No archives returned. Enter the ZIM name manually or search by title.": "Aucune archive trouvée. Saisissez le nom du ZIM ou recherchez par titre.",
+ "Each knowledge base connects one searchable ZIM on one kiwix-serve instance. Connect other archives or servers separately. DeepTutor reads matching articles on demand.": "Chaque base de connaissances relie un ZIM interrogeable sur une instance kiwix-serve. Connectez les autres archives ou serveurs séparément. DeepTutor lit les articles correspondants à la demande."
}
diff --git a/web/locales/uk/app.json b/web/locales/uk/app.json
index 9c1a6c9fa..9fd7ce507 100644
--- a/web/locales/uk/app.json
+++ b/web/locales/uk/app.json
@@ -5251,5 +5251,33 @@
"kanban.restore": "Відновити",
"kanban.emptyArchive": "Немає архівних завдань",
"This unsent message could not be saved in your browser. Copy it before leaving this page.": "Не вдалося зберегти це ненадіслане повідомлення у браузері. Скопіюйте його, перш ніж залишити цю сторінку.",
- "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Деякі ненадіслані повідомлення не вдалося перенести до цієї розмови. Відкрийте новий чат, щоб відновити їх."
+ "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "Деякі ненадіслані повідомлення не вдалося перенести до цієї розмови. Відкрийте новий чат, щоб відновити їх.",
+ "Kiwix / ZIM": "Kiwix / ZIM",
+ "Kiwix archive": "Архів Kiwix",
+ "Kiwix server URL": "Адреса сервера Kiwix",
+ "ZIM name": "Назва ZIM",
+ "Connect Kiwix archive": "Підключити архів Kiwix",
+ "Connect archive": "Підключити архів",
+ "Check connection": "Перевірити з’єднання",
+ "Could not reach Kiwix archive": "Не вдалося отримати доступ до архіву Kiwix",
+ "Could not connect Kiwix archive": "Не вдалося підключити архів Kiwix",
+ "Kiwix search failed": "Помилка пошуку Kiwix",
+ "Could not open Kiwix article": "Не вдалося відкрити статтю Kiwix",
+ "Search articles": "Шукати статті",
+ "Read article": "Читати статтю",
+ "No readable articles found.": "Не знайдено статей для читання.",
+ "Search an existing ZIM archive through kiwix-serve. Articles stay on the server.": "Шукайте в наявному архіві ZIM через kiwix-serve. Статті залишаються на сервері.",
+ "Connect to a searchable ZIM already served by kiwix-serve. DeepTutor reads matching articles on demand and does not import the archive.": "Підключіть доступний для пошуку архів ZIM через kiwix-serve. DeepTutor читає знайдені статті за потреби й не імпортує архів.",
+ "Search articles in this ZIM. DeepTutor retrieves matches for chat, Book, and Guided Learning without importing the archive.": "Шукайте статті в цьому ZIM. DeepTutor отримує результати для чату, Book і керованого навчання без імпорту архіву.",
+ "http://localhost:8080": "http://localhost:8080",
+ "my-archive": "my-archive",
+ "wikipedia_en_all": "wikipedia_en_all",
+ "Browse archives": "Переглянути архіви",
+ "Choose archive": "Вибрати архів",
+ "Could not browse Kiwix archives": "Не вдалося переглянути архіви Kiwix",
+ "Find archive": "Знайти архів",
+ "Loaded archives": "Завантажені архіви",
+ "Search loaded archives": "Пошук серед завантажених архівів",
+ "No archives returned. Enter the ZIM name manually or search by title.": "Архівів не знайдено. Введіть назву ZIM вручну або шукайте за заголовком.",
+ "Each knowledge base connects one searchable ZIM on one kiwix-serve instance. Connect other archives or servers separately. DeepTutor reads matching articles on demand.": "Кожна база знань підключає один доступний для пошуку ZIM на одному сервері kiwix-serve. Інші архіви або сервери підключайте окремо. DeepTutor читає знайдені статті за потреби."
}
diff --git a/web/locales/zh/app.json b/web/locales/zh/app.json
index 63465a075..d42c0018a 100644
--- a/web/locales/zh/app.json
+++ b/web/locales/zh/app.json
@@ -5235,5 +5235,33 @@
"kanban.restore": "恢复",
"kanban.emptyArchive": "没有已归档任务",
"This unsent message could not be saved in your browser. Copy it before leaving this page.": "这条未发送的消息无法保存到浏览器。离开此页面前请先复制。",
- "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "部分未发送的消息无法转入此对话。请打开新对话找回。"
+ "Some unsent messages could not be moved into this conversation. Open a new chat to recover them.": "部分未发送的消息无法转入此对话。请打开新对话找回。",
+ "Kiwix / ZIM": "Kiwix / ZIM",
+ "Kiwix archive": "Kiwix 档案",
+ "Kiwix server URL": "Kiwix 服务器地址",
+ "ZIM name": "ZIM 名称",
+ "Connect Kiwix archive": "连接 Kiwix 档案",
+ "Connect archive": "连接档案",
+ "Check connection": "检查连接",
+ "Could not reach Kiwix archive": "无法访问 Kiwix 档案",
+ "Could not connect Kiwix archive": "无法连接 Kiwix 档案",
+ "Kiwix search failed": "Kiwix 搜索失败",
+ "Could not open Kiwix article": "无法打开 Kiwix 文章",
+ "Search articles": "搜索文章",
+ "Read article": "阅读文章",
+ "No readable articles found.": "未找到可阅读的文章。",
+ "Search an existing ZIM archive through kiwix-serve. Articles stay on the server.": "通过 kiwix-serve 搜索现有 ZIM 档案。文章仍保存在服务器上。",
+ "Connect to a searchable ZIM already served by kiwix-serve. DeepTutor reads matching articles on demand and does not import the archive.": "连接由 kiwix-serve 提供搜索的 ZIM 档案。DeepTutor 按需读取匹配文章,不导入整个档案。",
+ "Search articles in this ZIM. DeepTutor retrieves matches for chat, Book, and Guided Learning without importing the archive.": "搜索此 ZIM 中的文章。DeepTutor 可为聊天、Book 和引导式学习检索匹配内容,无需导入整个档案。",
+ "http://localhost:8080": "http://localhost:8080",
+ "my-archive": "my-archive",
+ "wikipedia_en_all": "wikipedia_en_all",
+ "Browse archives": "浏览档案",
+ "Choose archive": "选择档案",
+ "Could not browse Kiwix archives": "无法浏览 Kiwix 档案",
+ "Find archive": "查找档案",
+ "Loaded archives": "已加载的档案",
+ "Search loaded archives": "搜索已加载的档案",
+ "No archives returned. Enter the ZIM name manually or search by title.": "未返回档案。请手动输入 ZIM 名称或按标题搜索。",
+ "Each knowledge base connects one searchable ZIM on one kiwix-serve instance. Connect other archives or servers separately. DeepTutor reads matching articles on demand.": "每个知识库连接一个 kiwix-serve 实例上的一个可搜索 ZIM。其他档案或服务器请分别连接。DeepTutor 按需读取匹配文章。"
}