Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

feat(web-source): add bilingual source pairing (#902)

XZH0e260130bea8
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index 10907877e..02cff0807 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -4682,6 +4682,18 @@ class AddWebSourceRequest(BaseModel):
max_pages: int = Field(default=200, ge=1, le=200)
+class BilingualPairingInfo(BaseModel):
+ pairing_id: str
+ source_url: str
+ target_url: str
+ source_file: str = ""
+ target_file: str = ""
+ source_lang: str = ""
+ target_lang: str = ""
+ pairing_method: str = "hreflang"
+ updated_at: int = 0
+
+
class WebSourceInfo(BaseModel):
id: str
url: str
@@ -4696,6 +4708,7 @@ class WebSourceInfo(BaseModel):
last_sync_error: str | None = None
added_at: str = ""
navigation: dict | None = None
+ bilingual_pairings: list[dict[str, Any]] = Field(default_factory=list)
class WebSourceScheduleUpdate(BaseModel):
@@ -4836,6 +4849,33 @@ async def get_web_source_sync_jobs(kb_name: str):
return [WebSourceSyncJobInfo(**job.public_dict()) for job in jobs]
+ "/knowledge-bases/{kb_name}/web-source/{source_id}/pairings",
+ response_model=list[BilingualPairingInfo],
+)
+async def get_web_source_pairings(kb_name: str, source_id: str):
+ with _knowledge_source_errors(kb_name):
+ manager, resolved_name, _ = _writable_kb(kb_name)
+ source = next(
+ (
+ item
+ for item in manager.get_web_sources(resolved_name)
+ if item.get("id") == source_id
+ ),
+ None,
+ )
+ if source is None:
+ raise HTTPException(status_code=404, detail=f"Source '{source_id}' not found")
+ scheduler = get_web_source_sync_scheduler()
+ repo_pairings = scheduler.repo.list_pairings(
+ get_current_user().id, resolved_name, source_id
+ )
+ if repo_pairings:
+ return [BilingualPairingInfo(**p.public_dict()) for p in repo_pairings]
+ meta_pairings = source.get("bilingual_pairings") or []
+ return [BilingualPairingInfo(**p) for p in meta_pairings]
+
+
@router.put(
"/knowledge-bases/{kb_name}/web-source/{source_id}/schedule",
response_model=WebSourceInfo,
diff --git a/deeptutor/services/web_source/bilingual.py b/deeptutor/services/web_source/bilingual.py
new file mode 100644
index 000000000..216ff3f93
--- /dev/null
+++ b/deeptutor/services/web_source/bilingual.py
@@ -0,0 +1,255 @@
+"""Bilingual source pairing and language variant alignment for web sources.
+
+Detects language variants (via hreflang links or URL language patterns),
+associates variant pages with canonical source pages, and prepares pairing
+records for storage and API exposure.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+from urllib.parse import urlparse
+
+# Canonical supported language codes and their common regional/script aliases.
+_CANONICAL_LANGUAGES: dict[str, str] = {
+ "en": "en",
+ "en-us": "en",
+ "en-gb": "en",
+ "en-au": "en",
+ "en-ca": "en",
+ "zh": "zh",
+ "zh-cn": "zh",
+ "zh-hans": "zh",
+ "zh-sg": "zh",
+ "zh-tw": "zh",
+ "zh-hant": "zh",
+ "zh-hk": "zh",
+ "ja": "ja",
+ "ja-jp": "ja",
+ "ko": "ko",
+ "ko-kr": "ko",
+ "fr": "fr",
+ "fr-fr": "fr",
+ "fr-ca": "fr",
+ "de": "de",
+ "de-de": "de",
+ "de-at": "de",
+ "de-ch": "de",
+ "es": "es",
+ "es-es": "es",
+ "es-419": "es",
+ "es-mx": "es",
+ "ru": "ru",
+ "ru-ru": "ru",
+ "pt": "pt",
+ "pt-br": "pt",
+ "pt-pt": "pt",
+ "it": "it",
+ "it-it": "it",
+}
+
+_KNOWN_LANG_CODES = frozenset(_CANONICAL_LANGUAGES.keys())
+
+SUPPORTED_LANGUAGE_PAIRS = frozenset(
+ {
+ frozenset({"en", "zh"}),
+ frozenset({"en", "ja"}),
+ frozenset({"en", "ko"}),
+ frozenset({"en", "fr"}),
+ frozenset({"en", "de"}),
+ frozenset({"en", "es"}),
+ frozenset({"en", "ru"}),
+ frozenset({"en", "pt"}),
+ frozenset({"en", "it"}),
+ }
+)
+
+
+def normalize_language(code: str | None) -> str:
+ """Normalize a language tag to its canonical primary language code.
+
+ Examples:
+ 'zh-CN' -> 'zh'
+ 'en-US' -> 'en'
+ 'zh-Hans' -> 'zh'
+ 'FR' -> 'fr'
+ 'x-default' -> ''
+ """
+ if not code:
+ return ""
+ cleaned = code.strip().lower().replace("_", "-")
+ if cleaned in ("x-default", "default"):
+ return ""
+ if cleaned in _CANONICAL_LANGUAGES:
+ return _CANONICAL_LANGUAGES[cleaned]
+ primary = cleaned.split("-", 1)[0]
+ return _CANONICAL_LANGUAGES.get(primary, primary)
+
+
+def is_supported_pair(lang_a: str, lang_b: str) -> bool:
+ """Check if two languages form a supported bilingual pairing."""
+ if not lang_a or not lang_b or lang_a == lang_b:
+ return False
+ pair = frozenset({lang_a, lang_b})
+ if pair in SUPPORTED_LANGUAGE_PAIRS:
+ return True
+ return "en" in pair or "zh" in pair
+
+
+def extract_url_language_and_stem(url: str) -> tuple[str, str]:
+ """Extract language code (if in URL path) and the language-agnostic stem path.
+
+ Handles leading language segments (/zh/docs/intro -> ("zh", "/docs/intro"))
+ and common nested prefixes (/docs/zh/intro -> ("zh", "/docs/intro")).
+ """
+ parsed = urlparse(url)
+ raw_path = parsed.path.rstrip("/")
+ if not raw_path:
+ return "", "/"
+
+ segments = [s for s in raw_path.split("/") if s]
+ if not segments:
+ return "", "/"
+
+ first_norm = normalize_language(segments[0])
+ if segments[0].lower() in _KNOWN_LANG_CODES or first_norm in SUPPORTED_LANGUAGE_PAIRS:
+ stem = "/" + "/".join(segments[1:])
+ return first_norm, stem or "/"
+
+ if len(segments) >= 2:
+ second_norm = normalize_language(segments[1])
+ if segments[1].lower() in _KNOWN_LANG_CODES or second_norm in SUPPORTED_LANGUAGE_PAIRS:
+ stem = f"/{segments[0]}" + ("/" + "/".join(segments[2:]) if len(segments) > 2 else "")
+ return second_norm, stem or "/"
+
+ return "", raw_path or "/"
+
+
+def detect_bilingual_pairings(
+ pages: list[Any],
+ page_file_map: dict[str, str] | None = None,
+) -> list[dict[str, Any]]:
+ """Detect bilingual page pairings among crawled pages.
+
+ Uses both explicit hreflang alternate declarations and URL language prefix
+ patterns. Returns a deduplicated, deterministically sorted list of pairing
+ records.
+ """
+ if not pages:
+ return []
+
+ file_map = page_file_map or {}
+ pages_by_url = {page.url: page for page in pages}
+ pairings: dict[str, dict[str, Any]] = {}
+
+ # 1. Hreflang alternate links (highest confidence)
+ for page in pages:
+ p1_lang = (
+ normalize_language(getattr(page, "language", ""))
+ or extract_url_language_and_stem(page.url)[0]
+ or "en"
+ )
+ alternate_links: dict[str, str] = getattr(page, "alternate_links", {})
+ for hreflang, target_url in alternate_links.items():
+ if target_url not in pages_by_url or target_url == page.url:
+ continue
+ target_page = pages_by_url[target_url]
+ p2_lang = (
+ normalize_language(hreflang)
+ or normalize_language(getattr(target_page, "language", ""))
+ or extract_url_language_and_stem(target_url)[0]
+ or "zh"
+ )
+ if not is_supported_pair(p1_lang, p2_lang):
+ continue
+
+ if p1_lang == "en":
+ src, tgt = page, target_page
+ s_lang, t_lang = p1_lang, p2_lang
+ elif p2_lang == "en":
+ src, tgt = target_page, page
+ s_lang, t_lang = p2_lang, p1_lang
+ elif page.url <= target_page.url:
+ src, tgt = page, target_page
+ s_lang, t_lang = p1_lang, p2_lang
+ else:
+ src, tgt = target_page, page
+ s_lang, t_lang = p2_lang, p1_lang
+
+ pairing_id = hashlib.sha256(f"{src.url}::{tgt.url}".encode("utf-8")).hexdigest()[:16]
+ pairings[pairing_id] = {
+ "pairing_id": pairing_id,
+ "source_url": src.url,
+ "target_url": tgt.url,
+ "source_file": file_map.get(src.url, ""),
+ "target_file": file_map.get(tgt.url, ""),
+ "source_lang": s_lang,
+ "target_lang": t_lang,
+ "pairing_method": "hreflang",
+ }
+
+ # 2. URL language stem pattern matching
+ stem_groups: dict[str, list[tuple[Any, str]]] = {}
+ for page in pages:
+ url_lang, stem = extract_url_language_and_stem(page.url)
+ page_lang = normalize_language(getattr(page, "language", "")) or url_lang
+ stem_groups.setdefault(stem, []).append((page, page_lang))
+
+ for stem, group in stem_groups.items():
+ if len(group) < 2:
+ continue
+ for i in range(len(group)):
+ for j in range(i + 1, len(group)):
+ p1, l1 = group[i]
+ p2, l2 = group[j]
+
+ # If one page is unprefixed and the other has a language tag,
+ # infer default 'en' for the unprefixed base documentation page.
+ if not l1 and l2 and l2 != "en":
+ l1 = "en"
+ elif not l2 and l1 and l1 != "en":
+ l2 = "en"
+
+ if not is_supported_pair(l1, l2):
+ continue
+
+ if l1 == "en":
+ src, tgt = p1, p2
+ s_lang, t_lang = l1, l2
+ elif l2 == "en":
+ src, tgt = p2, p1
+ s_lang, t_lang = l2, l1
+ elif p1.url <= p2.url:
+ src, tgt = p1, p2
+ s_lang, t_lang = l1, l2
+ else:
+ src, tgt = p2, p1
+ s_lang, t_lang = l2, l1
+
+ pairing_id = hashlib.sha256(f"{src.url}::{tgt.url}".encode("utf-8")).hexdigest()[
+ :16
+ ]
+ if pairing_id not in pairings:
+ pairings[pairing_id] = {
+ "pairing_id": pairing_id,
+ "source_url": src.url,
+ "target_url": tgt.url,
+ "source_file": file_map.get(src.url, ""),
+ "target_file": file_map.get(tgt.url, ""),
+ "source_lang": s_lang,
+ "target_lang": t_lang,
+ "pairing_method": "url_pattern",
+ }
+
+ result = list(pairings.values())
+ result.sort(key=lambda item: (item["source_url"], item["target_url"]))
+ return result
+
+
+__all__ = [
+ "detect_bilingual_pairings",
+ "extract_url_language_and_stem",
+ "is_supported_pair",
+ "normalize_language",
+]
diff --git a/deeptutor/services/web_source/crawler.py b/deeptutor/services/web_source/crawler.py
index f00f9f4de..3abe14615 100644
--- a/deeptutor/services/web_source/crawler.py
+++ b/deeptutor/services/web_source/crawler.py
@@ -59,6 +59,8 @@ class CrawledPage:
markdown: str
content_hash: str
headings: list[dict] = field(default_factory=list)
+ language: str = ""
+ alternate_links: dict[str, str] = field(default_factory=dict)
@dataclass
@@ -304,6 +306,7 @@ async def _process_page(
extract_article_markdown,
extract_headings,
extract_navigation,
+ extract_page_language_and_alternates,
)
try:
@@ -316,6 +319,7 @@ async def _process_page(
nav = extract_navigation(html, final_url) if depth <= 1 else []
page_headings = extract_headings(body)
+ lang, alternates = extract_page_language_and_alternates(html, final_url or url)
if len(body) > DEFAULT_MAX_CHARS:
body = body[:DEFAULT_MAX_CHARS].rstrip() + "\n…[truncated]"
@@ -326,6 +330,8 @@ async def _process_page(
markdown=body,
content_hash=content_hash,
headings=page_headings,
+ language=lang,
+ alternate_links=alternates,
)
links: list[str] = []
@@ -533,6 +539,7 @@ class CrawlDiff:
pages_removed: list[str] = field(default_factory=list)
changed_paths: list[str] = field(default_factory=list)
navigation: dict = field(default_factory=dict)
+ pairings: list[dict] = field(default_factory=list)
@property
def page_count(self) -> int:
@@ -642,6 +649,12 @@ async def crawl_and_diff(
page_urls,
)
+ # 6. Detect bilingual page pairings across discovered pages
+ from deeptutor.services.web_source.bilingual import detect_bilingual_pairings
+
+ page_file_map = {page.url: fname for page, fname in zip(result.pages, page_files)}
+ pairings = detect_bilingual_pairings(result.pages, page_file_map)
+
return CrawlDiff(
ok=True,
url=url,
@@ -654,4 +667,5 @@ async def crawl_and_diff(
pages_removed=removed,
changed_paths=changed_paths,
navigation=nav_manifest,
+ pairings=pairings,
)
diff --git a/deeptutor/services/web_source/html_extractor.py b/deeptutor/services/web_source/html_extractor.py
index 89b00fc2a..19456c233 100644
--- a/deeptutor/services/web_source/html_extractor.py
+++ b/deeptutor/services/web_source/html_extractor.py
@@ -579,3 +579,65 @@ def _clean_markdown(md: str) -> str:
md = "\n".join(lines)
md = re.sub(r"\n{3,}", "\n\n", md)
return md.strip()
+
+
+def extract_page_language_and_alternates(
+ raw_html: str,
+ base_url: str = "",
+) -> tuple[str, dict[str, str]]:
+ """Extract page language attribute and alternate hreflang URLs.
+
+ Returns:
+ tuple of (language_code, {normalized_hreflang: absolute_url})
+ """
+ from urllib.parse import urldefrag, urljoin
+
+ from lxml import html as lxml_html # nosec B410 - HTML parser, not XML
+
+ from deeptutor.services.web_source.bilingual import normalize_language
+
+ language = ""
+ alternates: dict[str, str] = {}
+
+ if not raw_html:
+ return language, alternates
+
+ try:
+ tree = lxml_html.fromstring(raw_html)
+ except Exception:
+ return language, alternates
+
+ # 1. Page language from <html lang="..."> or <html xml:lang="...">
+ html_lang = tree.get("lang") or tree.get("xml:lang") or ""
+ if html_lang:
+ language = normalize_language(html_lang)
+
+ # 2. Fall back to <meta http-equiv="content-language" content="..."> or <meta name="language" content="...">
+ if not language:
+ meta_nodes = tree.xpath("//meta[@http-equiv or @name]")
+ for meta in meta_nodes:
+ equiv = (meta.get("http-equiv") or meta.get("name") or "").strip().lower()
+ if equiv in ("content-language", "language"):
+ content = (meta.get("content") or "").strip()
+ if content:
+ language = normalize_language(content)
+ break
+
+ # 3. Alternate links: <link rel="alternate" hreflang="..." href="...">
+ link_nodes = tree.xpath("//link[contains(@rel, 'alternate') and @hreflang and @href]")
+ for link in link_nodes:
+ hreflang = (link.get("hreflang") or "").strip()
+ href = (link.get("href") or "").strip()
+ if not hreflang or not href or hreflang.lower() in ("x-default", "default"):
+ continue
+ norm_lang = normalize_language(hreflang)
+ if not norm_lang:
+ continue
+ href, _ = urldefrag(href)
+ if base_url:
+ resolved = urljoin(base_url, href)
+ else:
+ resolved = href
+ alternates[norm_lang] = resolved
+
+ return language, alternates
diff --git a/deeptutor/services/web_source/repository.py b/deeptutor/services/web_source/repository.py
index 614767171..1387d3db9 100644
--- a/deeptutor/services/web_source/repository.py
+++ b/deeptutor/services/web_source/repository.py
@@ -61,6 +61,54 @@ class WebSourceSyncJob:
}
+@dataclass(slots=True)
+class WebSourceBilingualPairing:
+ """One durable bilingual pairing between two crawled pages."""
+
+ owner_id: str
+ kb_name: str
+ source_id: str
+ pairing_id: str
+ source_url: str
+ target_url: str
+ source_file: str
+ target_file: str
+ source_lang: str
+ target_lang: str
+ pairing_method: str
+ updated_at_ms: int
+
+ @classmethod
+ def from_row(cls, row: sqlite3.Row) -> "WebSourceBilingualPairing":
+ return cls(
+ owner_id=str(row["owner_id"]),
+ kb_name=str(row["kb_name"]),
+ source_id=str(row["source_id"]),
+ pairing_id=str(row["pairing_id"]),
+ source_url=str(row["source_url"]),
+ target_url=str(row["target_url"]),
+ source_file=str(row["source_file"]),
+ target_file=str(row["target_file"]),
+ source_lang=str(row["source_lang"]),
+ target_lang=str(row["target_lang"]),
+ pairing_method=str(row["pairing_method"]),
+ updated_at_ms=int(row["updated_at_ms"]),
+ )
+
+ def public_dict(self) -> dict[str, Any]:
+ return {
+ "pairing_id": self.pairing_id,
+ "source_url": self.source_url,
+ "target_url": self.target_url,
+ "source_file": self.source_file,
+ "target_file": self.target_file,
+ "source_lang": self.source_lang,
+ "target_lang": self.target_lang,
+ "pairing_method": self.pairing_method,
+ "updated_at": self.updated_at_ms,
+ }
+
+
class SQLiteWebSourceSyncRepository:
"""WAL-backed job state shared by API requests and the scheduler."""
@@ -102,6 +150,29 @@ class SQLiteWebSourceSyncRepository:
"CREATE INDEX IF NOT EXISTS idx_web_source_sync_due "
"ON web_source_sync_jobs(next_run_at_ms, state)"
)
+ connection.execute(
+ """
+ CREATE TABLE IF NOT EXISTS web_source_bilingual_pairings (
+ owner_id TEXT NOT NULL,
+ kb_name TEXT NOT NULL,
+ source_id TEXT NOT NULL,
+ pairing_id TEXT NOT NULL,
+ source_url TEXT NOT NULL,
+ target_url TEXT NOT NULL,
+ source_file TEXT NOT NULL,
+ target_file TEXT NOT NULL,
+ source_lang TEXT NOT NULL,
+ target_lang TEXT NOT NULL,
+ pairing_method TEXT NOT NULL,
+ updated_at_ms INTEGER NOT NULL,
+ PRIMARY KEY (owner_id, kb_name, source_id, pairing_id)
+ )
+ """
+ )
+ connection.execute(
+ "CREATE INDEX IF NOT EXISTS idx_web_source_pairings "
+ "ON web_source_bilingual_pairings(owner_id, kb_name, source_id)"
+ )
@staticmethod
def key(job: WebSourceSyncJob) -> tuple[str, str, str]:
@@ -148,6 +219,13 @@ class SQLiteWebSourceSyncRepository:
""",
(owner_id, kb_name, source_id),
)
+ connection.execute(
+ """
+ DELETE FROM web_source_bilingual_pairings
+ WHERE owner_id=? AND kb_name=? AND source_id=?
+ """,
+ (owner_id, kb_name, source_id),
+ )
connection.execute("COMMIT")
except Exception:
connection.execute("ROLLBACK")
@@ -419,6 +497,13 @@ class SQLiteWebSourceSyncRepository:
""",
job_key,
)
+ connection.execute(
+ """
+ DELETE FROM web_source_bilingual_pairings
+ WHERE owner_id=? AND kb_name=? AND source_id=?
+ """,
+ job_key,
+ )
return int(cursor.rowcount or 0) > 0
def retry(self, job_key: tuple[str, str, str]) -> WebSourceSyncJob | None:
@@ -451,5 +536,97 @@ class SQLiteWebSourceSyncRepository:
).fetchone()
return WebSourceSyncJob.from_row(row) if row else None
+ def record_pairings(
+ self,
+ owner_id: str,
+ kb_name: str,
+ source_id: str,
+ pairings: list[dict[str, Any]],
+ ) -> list[WebSourceBilingualPairing]:
+ """Atomically replace bilingual pairings for one web source."""
+ now = self._now_ms()
+ with self._connect() as connection:
+ connection.execute("BEGIN IMMEDIATE")
+ try:
+ connection.execute(
+ """
+ DELETE FROM web_source_bilingual_pairings
+ WHERE owner_id=? AND kb_name=? AND source_id=?
+ """,
+ (owner_id, kb_name, source_id),
+ )
+ for item in pairings:
+ connection.execute(
+ """
+ INSERT INTO web_source_bilingual_pairings
+ (owner_id, kb_name, source_id, pairing_id, source_url,
+ target_url, source_file, target_file, source_lang,
+ target_lang, pairing_method, updated_at_ms)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ owner_id,
+ kb_name,
+ source_id,
+ str(item["pairing_id"]),
+ str(item["source_url"]),
+ str(item["target_url"]),
+ str(item.get("source_file", "")),
+ str(item.get("target_file", "")),
+ str(item.get("source_lang", "")),
+ str(item.get("target_lang", "")),
+ str(item.get("pairing_method", "hreflang")),
+ now,
+ ),
+ )
+ connection.execute("COMMIT")
+ except Exception:
+ connection.execute("ROLLBACK")
+ raise
+ return self.list_pairings(owner_id, kb_name, source_id)
+
+ def list_pairings(
+ self,
+ owner_id: str,
+ kb_name: str,
+ source_id: str | None = None,
+ ) -> list[WebSourceBilingualPairing]:
+ with self._connect() as connection:
+ if source_id is not None:
+ rows = connection.execute(
+ """
+ SELECT * FROM web_source_bilingual_pairings
+ WHERE owner_id=? AND kb_name=? AND source_id=?
+ ORDER BY source_url, target_url
+ """,
+ (owner_id, kb_name, source_id),
+ ).fetchall()
+ else:
+ rows = connection.execute(
+ """
+ SELECT * FROM web_source_bilingual_pairings
+ WHERE owner_id=? AND kb_name=?
+ ORDER BY source_id, source_url, target_url
+ """,
+ (owner_id, kb_name),
+ ).fetchall()
+ return [WebSourceBilingualPairing.from_row(row) for row in rows]
+
+ def delete_pairings(
+ self,
+ owner_id: str,
+ kb_name: str,
+ source_id: str,
+ ) -> int:
+ with self._connect() as connection:
+ cursor = connection.execute(
+ """
+ DELETE FROM web_source_bilingual_pairings
+ WHERE owner_id=? AND kb_name=? AND source_id=?
+ """,
+ (owner_id, kb_name, source_id),
+ )
+ return int(cursor.rowcount or 0)
+
-__all__ = ["SQLiteWebSourceSyncRepository", "WebSourceSyncJob"]
+__all__ = ["SQLiteWebSourceSyncRepository", "WebSourceBilingualPairing", "WebSourceSyncJob"]
diff --git a/deeptutor/services/web_source/sync.py b/deeptutor/services/web_source/sync.py
index 0bcba2d7c..fcf63aa62 100644
--- a/deeptutor/services/web_source/sync.py
+++ b/deeptutor/services/web_source/sync.py
@@ -35,6 +35,7 @@ class WebSyncResult:
pages_removed: int = 0
pages_unchanged: int = 0
error: str = ""
+ bilingual_pairs_count: int = 0
@property
def total_changes(self) -> int:
@@ -185,16 +186,33 @@ async def sync_source(
last_sync_status="success",
last_sync_error=None,
navigation=diff.navigation,
+ bilingual_pairings=diff.pairings,
)
+ try:
+ from deeptutor.multi_user.context import get_current_user
+ from deeptutor.services.web_source.scheduler import get_web_source_sync_scheduler
+
+ user = get_current_user()
+ scheduler = get_web_source_sync_scheduler()
+ scheduler.repo.record_pairings(
+ owner_id=user.id,
+ kb_name=kb_name,
+ source_id=source["id"],
+ pairings=diff.pairings,
+ )
+ except Exception:
+ logger.debug("Failed to record pairings in SQLite repo", exc_info=True)
+
logger.info(
- "Web sync %s: +%d ~%d -%d (%d unchanged), %d indexed",
+ "Web sync %s: +%d ~%d -%d (%d unchanged), %d indexed, %d bilingual pairs",
diff.url,
len(diff.pages_added),
len(diff.pages_updated),
removed_count,
len(diff.pages_unchanged),
indexed,
+ len(diff.pairings),
)
return WebSyncResult(
@@ -203,6 +221,7 @@ async def sync_source(
pages_updated=len(diff.pages_updated),
pages_removed=removed_count,
pages_unchanged=len(diff.pages_unchanged),
+ bilingual_pairs_count=len(diff.pairings),
)
diff --git a/tests/api/test_web_source_schedule_api.py b/tests/api/test_web_source_schedule_api.py
index 5395825e1..5ac54faff 100644
--- a/tests/api/test_web_source_schedule_api.py
+++ b/tests/api/test_web_source_schedule_api.py
@@ -99,3 +99,58 @@ def test_schedule_rejects_out_of_range_interval(client) -> None:
json={"auto_sync_enabled": True, "sync_interval_hours": 0},
)
assert response.status_code == 422
+
+
+def test_web_source_pairings_api(client) -> None:
+ test_client, manager = client
+ source = manager.add_web_source("kb", "https://example.com/docs/")
+ source_id = source["id"]
+
+ # Initial web-sources listing has empty pairings
+ list_res = test_client.get("/api/knowledge-bases/kb/web-sources")
+ assert list_res.status_code == 200
+ sources = list_res.json()
+ assert len(sources) == 1
+ assert "bilingual_pairings" in sources[0]
+ assert sources[0]["bilingual_pairings"] == []
+
+ # Initial pairings endpoint returns empty
+ pair_res = test_client.get(f"/api/knowledge-bases/kb/web-source/{source_id}/pairings")
+ assert pair_res.status_code == 200
+ assert pair_res.json() == []
+
+ # Record pairings into repo
+ scheduler_module._scheduler.repo.record_pairings(
+ "local-admin",
+ "kb",
+ source_id,
+ [
+ {
+ "pairing_id": "pair-1",
+ "source_url": "https://example.com/docs/intro",
+ "target_url": "https://example.com/zh/docs/intro",
+ "source_file": "_web/s/docs/intro.md",
+ "target_file": "_web/s/zh/docs/intro.md",
+ "source_lang": "en",
+ "target_lang": "zh",
+ "pairing_method": "hreflang",
+ }
+ ],
+ )
+
+ pair_res = test_client.get(f"/api/knowledge-bases/kb/web-source/{source_id}/pairings")
+ assert pair_res.status_code == 200
+ pairings = pair_res.json()
+ assert len(pairings) == 1
+ assert pairings[0]["pairing_id"] == "pair-1"
+ assert pairings[0]["source_lang"] == "en"
+ assert pairings[0]["target_lang"] == "zh"
+
+ # Non-existent source returns 404
+ missing_res = test_client.get("/api/knowledge-bases/kb/web-source/nonexistent/pairings")
+ assert missing_res.status_code == 404
+
+ # Deleting source removes pairings
+ del_res = test_client.delete(f"/api/knowledge-bases/kb/web-source/{source_id}")
+ assert del_res.status_code == 200
+ assert scheduler_module._scheduler.repo.list_pairings("local-admin", "kb", source_id) == []
diff --git a/tests/services/test_web_source_sync.py b/tests/services/test_web_source_sync.py
index 1312c9716..b9d16473e 100644
--- a/tests/services/test_web_source_sync.py
+++ b/tests/services/test_web_source_sync.py
@@ -489,3 +489,53 @@ async def test_sync_persists_navigation(tmp_path: Path):
assert len(nav["nodes"]) >= 1
assert sources_after[0]["page_count"] == 2
+
+
+async def test_sync_persists_bilingual_pairings(tmp_path: Path):
+ base_dir, kb_dir = _make_kb(tmp_path)
+ from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+ mgr = KnowledgeBaseManager(base_dir=base_dir)
+ source = mgr.add_web_source("kb", "https://example.com/docs/")
+
+ mock_result = CrawlResult(
+ pages=[
+ CrawledPage(
+ url="https://example.com/docs/en/intro",
+ title="Intro",
+ markdown="# Intro",
+ content_hash="aaa",
+ language="en",
+ ),
+ CrawledPage(
+ url="https://example.com/docs/zh/intro",
+ title="介绍",
+ markdown="# 介绍",
+ content_hash="bbb",
+ language="zh",
+ ),
+ ],
+ )
+
+ with patch(
+ "deeptutor.services.web_source.crawler.crawl_docs_site", new_callable=AsyncMock
+ ) as mock_crawl:
+ mock_crawl.return_value = mock_result
+ with patch(
+ "deeptutor.knowledge.add_documents.add_documents", new_callable=AsyncMock
+ ) as mock_add:
+ mock_add.return_value = 2
+ result = await sync_source("kb", source, base_dir=base_dir)
+
+ assert result.ok is True
+ assert result.bilingual_pairs_count == 1
+
+ sources_after = mgr.get_web_sources("kb")
+ assert len(sources_after) == 1
+ pairings = sources_after[0].get("bilingual_pairings", [])
+ assert len(pairings) == 1
+ assert pairings[0]["source_url"] == "https://example.com/docs/en/intro"
+ assert pairings[0]["target_url"] == "https://example.com/docs/zh/intro"
+ assert pairings[0]["source_lang"] == "en"
+ assert pairings[0]["target_lang"] == "zh"
diff --git a/tests/services/web_source/test_bilingual_pairing.py b/tests/services/web_source/test_bilingual_pairing.py
new file mode 100644
index 000000000..fa13527ac
--- /dev/null
+++ b/tests/services/web_source/test_bilingual_pairing.py
@@ -0,0 +1,232 @@
+"""Tests for web-source bilingual page pairing and persistence."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+from deeptutor.services.web_source.bilingual import (
+ detect_bilingual_pairings,
+ extract_url_language_and_stem,
+ is_supported_pair,
+ normalize_language,
+)
+from deeptutor.services.web_source.crawler import CrawledPage
+from deeptutor.services.web_source.html_extractor import extract_page_language_and_alternates
+from deeptutor.services.web_source.repository import (
+ SQLiteWebSourceSyncRepository,
+ WebSourceBilingualPairing,
+)
+
+
+def test_normalize_language() -> None:
+ assert normalize_language("en-US") == "en"
+ assert normalize_language("EN") == "en"
+ assert normalize_language("zh-CN") == "zh"
+ assert normalize_language("zh_cn") == "zh"
+ assert normalize_language("zh-Hans") == "zh"
+ assert normalize_language("zh-Hant") == "zh"
+ assert normalize_language("ja-JP") == "ja"
+ assert normalize_language("x-default") == ""
+ assert normalize_language("") == ""
+ assert normalize_language(None) == ""
+
+
+def test_is_supported_pair() -> None:
+ assert is_supported_pair("en", "zh") is True
+ assert is_supported_pair("zh", "en") is True
+ assert is_supported_pair("en", "ja") is True
+ assert is_supported_pair("en", "fr") is True
+ assert is_supported_pair("en", "en") is False
+ assert is_supported_pair("", "en") is False
+
+
+def test_extract_url_language_and_stem() -> None:
+ assert extract_url_language_and_stem("https://example.com/en/docs/intro") == (
+ "en",
+ "/docs/intro",
+ )
+ assert extract_url_language_and_stem("https://example.com/zh-cn/docs/intro") == (
+ "zh",
+ "/docs/intro",
+ )
+ assert extract_url_language_and_stem("https://example.com/docs/en/intro") == (
+ "en",
+ "/docs/intro",
+ )
+ assert extract_url_language_and_stem("https://example.com/docs/zh/intro") == (
+ "zh",
+ "/docs/intro",
+ )
+ assert extract_url_language_and_stem("https://example.com/docs/intro") == (
+ "",
+ "/docs/intro",
+ )
+ assert extract_url_language_and_stem("https://example.com/") == ("", "/")
+
+
+def test_extract_page_language_and_alternates() -> None:
+ html = """<!DOCTYPE html>
+ <html lang="en">
+ <head>
+ <link rel="alternate" hreflang="zh-CN" href="/zh/docs/intro" />
+ <link rel="alternate" hreflang="en" href="/docs/intro" />
+ <link rel="alternate" hreflang="x-default" href="/docs/intro" />
+ </head>
+ <body><h1>Introduction</h1></body>
+ </html>"""
+ lang, alternates = extract_page_language_and_alternates(html, "https://example.com/docs/intro")
+ assert lang == "en"
+ assert alternates == {
+ "zh": "https://example.com/zh/docs/intro",
+ "en": "https://example.com/docs/intro",
+ }
+
+
+def test_detect_bilingual_pairings_hreflang() -> None:
+ p_en = CrawledPage(
+ url="https://example.com/docs/intro",
+ title="Intro",
+ markdown="# Intro",
+ content_hash="hash-en",
+ language="en",
+ alternate_links={"zh": "https://example.com/zh/docs/intro"},
+ )
+ p_zh = CrawledPage(
+ url="https://example.com/zh/docs/intro",
+ title="介绍",
+ markdown="# 介绍",
+ content_hash="hash-zh",
+ language="zh",
+ alternate_links={"en": "https://example.com/docs/intro"},
+ )
+
+ file_map = {
+ "https://example.com/docs/intro": "_web/abc/docs/intro.md",
+ "https://example.com/zh/docs/intro": "_web/abc/zh/docs/intro.md",
+ }
+
+ pairings = detect_bilingual_pairings([p_en, p_zh], file_map)
+ assert len(pairings) == 1
+ pair = pairings[0]
+ assert pair["source_url"] == "https://example.com/docs/intro"
+ assert pair["target_url"] == "https://example.com/zh/docs/intro"
+ assert pair["source_lang"] == "en"
+ assert pair["target_lang"] == "zh"
+ assert pair["source_file"] == "_web/abc/docs/intro.md"
+ assert pair["target_file"] == "_web/abc/zh/docs/intro.md"
+ assert pair["pairing_method"] == "hreflang"
+
+
+def test_detect_bilingual_pairings_url_pattern() -> None:
+ p_en = CrawledPage(
+ url="https://example.com/en/guide/install",
+ title="Install",
+ markdown="# Install",
+ content_hash="hash-en",
+ language="en",
+ )
+ p_zh = CrawledPage(
+ url="https://example.com/zh/guide/install",
+ title="安装",
+ markdown="# 安装",
+ content_hash="hash-zh",
+ language="zh",
+ )
+
+ pairings = detect_bilingual_pairings([p_en, p_zh])
+ assert len(pairings) == 1
+ pair = pairings[0]
+ assert pair["source_url"] == "https://example.com/en/guide/install"
+ assert pair["target_url"] == "https://example.com/zh/guide/install"
+ assert pair["source_lang"] == "en"
+ assert pair["target_lang"] == "zh"
+ assert pair["pairing_method"] == "url_pattern"
+
+
+def test_repository_pairings_crud(tmp_path: Path) -> None:
+ repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")
+ owner_id = "user-123"
+ kb_name = "test-kb"
+ source_id = "src-456"
+
+ pairings_data = [
+ {
+ "pairing_id": "pair-001",
+ "source_url": "https://example.com/docs/intro",
+ "target_url": "https://example.com/zh/docs/intro",
+ "source_file": "_web/s1/docs/intro.md",
+ "target_file": "_web/s1/zh/docs/intro.md",
+ "source_lang": "en",
+ "target_lang": "zh",
+ "pairing_method": "hreflang",
+ }
+ ]
+
+ saved = repo.record_pairings(owner_id, kb_name, source_id, pairings_data)
+ assert len(saved) == 1
+ assert isinstance(saved[0], WebSourceBilingualPairing)
+ assert saved[0].pairing_id == "pair-001"
+ assert saved[0].source_url == "https://example.com/docs/intro"
+
+ # List pairings for specific source
+ listed = repo.list_pairings(owner_id, kb_name, source_id)
+ assert len(listed) == 1
+ assert listed[0].public_dict()["pairing_id"] == "pair-001"
+
+ # List pairings across all sources in KB
+ all_kb_pairs = repo.list_pairings(owner_id, kb_name)
+ assert len(all_kb_pairs) == 1
+
+ # Atomic replace with empty
+ replaced = repo.record_pairings(owner_id, kb_name, source_id, [])
+ assert len(replaced) == 0
+ assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 0
+
+
+def test_delete_source_cleans_up_pairings(tmp_path: Path) -> None:
+ repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")
+ owner_id = "user-123"
+ kb_name = "test-kb"
+ source_id = "src-456"
+
+ repo.ensure_source((owner_id, kb_name, source_id))
+ repo.record_pairings(
+ owner_id,
+ kb_name,
+ source_id,
+ [
+ {
+ "pairing_id": "pair-001",
+ "source_url": "https://example.com/docs/intro",
+ "target_url": "https://example.com/zh/docs/intro",
+ "source_file": "_web/s1/docs/intro.md",
+ "target_file": "_web/s1/zh/docs/intro.md",
+ "source_lang": "en",
+ "target_lang": "zh",
+ "pairing_method": "hreflang",
+ }
+ ],
+ )
+ assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 1
+
+ # Deleting source deletes both job and pairings
+ repo.delete((owner_id, kb_name, source_id))
+ assert repo.get((owner_id, kb_name, source_id)) is None
+ assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 0
+
+
+def test_non_web_kb_isolation(tmp_path: Path) -> None:
+ """Ensure non-web knowledge bases are isolated and do not interact with pairings."""
+ repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")
+ owner_id = "user-123"
+
+ # Querying a non-web KB returns empty list, never raises
+ pairs = repo.list_pairings(owner_id, "marginnote-kb")
+ assert pairs == []
+
+ # Deleting non-existent returns 0
+ deleted = repo.delete_pairings(owner_id, "marginnote-kb", "src-nonexistent")
+ assert deleted == 0
diff --git a/web/components/knowledge/KbWebSourcesSection.tsx b/web/components/knowledge/KbWebSourcesSection.tsx
index 31fb26e51..d5c8645e2 100644
--- a/web/components/knowledge/KbWebSourcesSection.tsx
+++ b/web/components/knowledge/KbWebSourcesSection.tsx
@@ -381,6 +381,19 @@ function WebSourceCard({
{t("Pages")}: {source.page_count}
</span>
)}
+ {source.bilingual_pairings && source.bilingual_pairings.length > 0 && (
+ <span
+ className="font-medium text-emerald-600 dark:text-emerald-400"
+ title={source.bilingual_pairings
+ .map(
+ (p) =>
+ `${p.source_lang.toUpperCase()} ↔ ${p.target_lang.toUpperCase()}: ${p.source_url} ↔ ${p.target_url}`,
+ )
+ .join("\n")}
+ >
+ {t("Bilingual pairs")}: {source.bilingual_pairings.length}
+ </span>
+ )}
{lastSync && (
<span>
{t("Synced")}: {lastSync}
diff --git a/web/features/knowledge/api/client.ts b/web/features/knowledge/api/client.ts
index 083f23553..02b6f63a6 100644
--- a/web/features/knowledge/api/client.ts
+++ b/web/features/knowledge/api/client.ts
@@ -1656,6 +1656,18 @@ export async function syncGitHubSources(
// ── Web sources ──────────────────────────────────────────────────────
+export interface BilingualPairing {
+ pairing_id: string;
+ source_url: string;
+ target_url: string;
+ source_file: string;
+ target_file: string;
+ source_lang: string;
+ target_lang: string;
+ pairing_method: string;
+ updated_at?: number;
+}
+
export interface WebSource {
id: string;
url: string;
@@ -1669,6 +1681,7 @@ export interface WebSource {
last_sync_status: string;
last_sync_error: string | null;
added_at: string;
+ bilingual_pairings?: BilingualPairing[];
}
export interface AddWebSourcePayload {
@@ -1728,6 +1741,28 @@ export async function listWebSources(
return (await res.json()) as WebSource[];
}
+export async function listWebSourcePairings(
+ kbName: string,
+ sourceId: string,
+ options?: { signal?: AbortSignal },
+): Promise<BilingualPairing[]> {
+ const res = await apiFetch(
+ apiUrl(
+ `/api/knowledge-bases/${encodeURIComponent(kbName)}/web-source/${encodeURIComponent(sourceId)}/pairings`,
+ ),
+ { signal: options?.signal },
+ );
+ if (!res.ok) {
+ throw new Error(
+ await readErrorDetail(
+ res,
+ `Failed to list web source pairings (${res.status})`,
+ ),
+ );
+ }
+ return (await res.json()) as BilingualPairing[];
+}
+
export async function addWebSource(
kbName: string,
payload: AddWebSourcePayload,
diff --git a/web/features/knowledge/api/sources.ts b/web/features/knowledge/api/sources.ts
index 82f970682..ce499a00b 100644
--- a/web/features/knowledge/api/sources.ts
+++ b/web/features/knowledge/api/sources.ts
@@ -6,6 +6,7 @@ export {
listGitHubSources,
listLinkedFolders,
listWebSources,
+ listWebSourcePairings,
listWebSourceSyncJobs,
removeGitHubSource,
removeWebSource,
@@ -20,6 +21,7 @@ export {
export type {
AddGitHubSourcePayload,
AddWebSourcePayload,
+ BilingualPairing,
GitHubSource,
GitHubSyncResult,
LinkedFolderInfo,
diff --git a/web/features/knowledge/model/types.ts b/web/features/knowledge/model/types.ts
index bed2800fc..ec738803f 100644
--- a/web/features/knowledge/model/types.ts
+++ b/web/features/knowledge/model/types.ts
@@ -3,6 +3,7 @@ export type {
EmbeddingUsage,
AddGitHubSourcePayload,
AddWebSourcePayload,
+ BilingualPairing,
EnginePreflight,
GitHubSource,
GitHubSyncResult,
diff --git a/web/locales/de/app.json b/web/locales/de/app.json
index 80071a718..a91d2cc4c 100644
--- a/web/locales/de/app.json
+++ b/web/locales/de/app.json
@@ -811,6 +811,7 @@
"Schedule": "Zeitplan",
"Next run": "Nächster Lauf",
"Automatic sync": "Automatische Synchronisierung",
+ "Bilingual pairs": "Zweisprachige Paare",
"Interval hours": "Intervall (Stunden)",
"Cancel sync": "Synchronisierung abbrechen",
"Retry sync": "Synchronisierung erneut versuchen",
diff --git a/web/locales/en/app.json b/web/locales/en/app.json
index c6167eb21..d88102b0b 100644
--- a/web/locales/en/app.json
+++ b/web/locales/en/app.json
@@ -811,6 +811,7 @@
"Schedule": "Schedule",
"Next run": "Next run",
"Automatic sync": "Automatic sync",
+ "Bilingual pairs": "Bilingual pairs",
"Interval hours": "Interval hours",
"Cancel sync": "Cancel sync",
"Retry sync": "Retry sync",
diff --git a/web/locales/fr/app.json b/web/locales/fr/app.json
index 996e47743..2f4205924 100644
--- a/web/locales/fr/app.json
+++ b/web/locales/fr/app.json
@@ -4279,6 +4279,7 @@
"Schedule": "Planification",
"Next run": "Prochaine exécution",
"Automatic sync": "Synchronisation automatique",
+ "Bilingual pairs": "Paires bilingues",
"Interval hours": "Intervalle (heures)",
"Cancel sync": "Annuler la synchronisation",
"Retry sync": "Réessayer la synchronisation",
diff --git a/web/locales/uk/app.json b/web/locales/uk/app.json
index 36788d166..fe91bb306 100644
--- a/web/locales/uk/app.json
+++ b/web/locales/uk/app.json
@@ -4054,6 +4054,7 @@
"Schedule": "Розклад",
"Next run": "Наступний запуск",
"Automatic sync": "Автоматична синхронізація",
+ "Bilingual pairs": "Двомовні пари",
"Interval hours": "Інтервал (год)",
"Cancel sync": "Скасувати синхронізацію",
"Retry sync": "Повторити синхронізацію",
diff --git a/web/locales/zh/app.json b/web/locales/zh/app.json
index 4312257b3..53ba5032d 100644
--- a/web/locales/zh/app.json
+++ b/web/locales/zh/app.json
@@ -840,6 +840,7 @@
"Schedule": "调度",
"Next run": "下次运行",
"Automatic sync": "自动同步",
+ "Bilingual pairs": "双语配对",
"Interval hours": "间隔小时数",
"Cancel sync": "取消同步",
"Retry sync": "重试同步",
diff --git a/web/tests/web-source-bilingual-pairing.spec.tsx b/web/tests/web-source-bilingual-pairing.spec.tsx
new file mode 100644
index 000000000..fec11aa2c
--- /dev/null
+++ b/web/tests/web-source-bilingual-pairing.spec.tsx
@@ -0,0 +1,47 @@
+import React from "react";
+import { cleanup, render, screen } from "@testing-library/react";
+import { afterEach, expect, it, vi } from "vitest";
+import KbWebSourcesSection from "@/components/knowledge/KbWebSourcesSection";
+import * as sourcesApi from "@/features/knowledge/api/sources";
+
+vi.mock("react-i18next", () => ({
+ useTranslation: () => ({ t: (key: string) => key }),
+}));
+
+afterEach(cleanup);
+
+it("renders bilingual pairs count when pairings exist", async () => {
+ vi.spyOn(sourcesApi, "listWebSources").mockResolvedValue([
+ {
+ id: "src-1",
+ url: "https://example.com/docs/",
+ max_depth: 3,
+ max_pages: 50,
+ enabled: true,
+ auto_sync_enabled: true,
+ sync_interval_hours: 24,
+ page_count: 10,
+ last_synced_at: "2026-09-28T12:00:00Z",
+ last_sync_status: "success",
+ last_sync_error: null,
+ added_at: "2026-09-28T10:00:00Z",
+ bilingual_pairings: [
+ {
+ pairing_id: "pair-1",
+ source_url: "https://example.com/docs/en/intro",
+ target_url: "https://example.com/docs/zh/intro",
+ source_file: "intro.md",
+ target_file: "intro.zh.md",
+ source_lang: "en",
+ target_lang: "zh",
+ pairing_method: "hreflang",
+ },
+ ],
+ },
+ ]);
+ vi.spyOn(sourcesApi, "listWebSourceSyncJobs").mockResolvedValue([]);
+
+ render(<KbWebSourcesSection kbName="test-kb" />);
+
+ expect(await screen.findByText(/Bilingual pairs: 1/)).toBeDefined();
+});