DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
feat(web-source): add bilingual source pairing (#902)
XZH0e260130bea8
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex 10907877e..02cff0807 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -4682,6 +4682,18 @@ class AddWebSourceRequest(BaseModel):max_pages: int = Field(default=200, ge=1, le=200)+class BilingualPairingInfo(BaseModel):+ pairing_id: str+ source_url: str+ target_url: str+ source_file: str = ""+ target_file: str = ""+ source_lang: str = ""+ target_lang: str = ""+ pairing_method: str = "hreflang"+ updated_at: int = 0++class WebSourceInfo(BaseModel):id: strurl: str@@ -4696,6 +4708,7 @@ class WebSourceInfo(BaseModel):last_sync_error: str | None = Noneadded_at: str = ""navigation: dict | None = None+ bilingual_pairings: list[dict[str, Any]] = Field(default_factory=list)class WebSourceScheduleUpdate(BaseModel):@@ -4836,6 +4849,33 @@ async def get_web_source_sync_jobs(kb_name: str):return [WebSourceSyncJobInfo(**job.public_dict()) for job in jobs]+ "/knowledge-bases/{kb_name}/web-source/{source_id}/pairings",+ response_model=list[BilingualPairingInfo],+)+async def get_web_source_pairings(kb_name: str, source_id: str):+ with _knowledge_source_errors(kb_name):+ manager, resolved_name, _ = _writable_kb(kb_name)+ source = next(+ (+ item+ for item in manager.get_web_sources(resolved_name)+ if item.get("id") == source_id+ ),+ None,+ )+ if source is None:+ raise HTTPException(status_code=404, detail=f"Source '{source_id}' not found")+ scheduler = get_web_source_sync_scheduler()+ repo_pairings = scheduler.repo.list_pairings(+ get_current_user().id, resolved_name, source_id+ )+ if repo_pairings:+ return [BilingualPairingInfo(**p.public_dict()) for p in repo_pairings]+ meta_pairings = source.get("bilingual_pairings") or []+ return [BilingualPairingInfo(**p) for p in meta_pairings]++@router.put("/knowledge-bases/{kb_name}/web-source/{source_id}/schedule",response_model=WebSourceInfo,diff --git a/deeptutor/services/web_source/bilingual.py b/deeptutor/services/web_source/bilingual.pynew file mode 100644index 000000000..216ff3f93--- /dev/null+++ b/deeptutor/services/web_source/bilingual.py@@ -0,0 +1,255 @@+"""Bilingual source pairing and language variant alignment for web sources.++Detects language variants (via hreflang links or URL language patterns),+associates variant pages with canonical source pages, and prepares pairing+records for storage and API exposure.+"""++from __future__ import annotations++import hashlib+from typing import Any+from urllib.parse import urlparse++# Canonical supported language codes and their common regional/script aliases.+_CANONICAL_LANGUAGES: dict[str, str] = {+ "en": "en",+ "en-us": "en",+ "en-gb": "en",+ "en-au": "en",+ "en-ca": "en",+ "zh": "zh",+ "zh-cn": "zh",+ "zh-hans": "zh",+ "zh-sg": "zh",+ "zh-tw": "zh",+ "zh-hant": "zh",+ "zh-hk": "zh",+ "ja": "ja",+ "ja-jp": "ja",+ "ko": "ko",+ "ko-kr": "ko",+ "fr": "fr",+ "fr-fr": "fr",+ "fr-ca": "fr",+ "de": "de",+ "de-de": "de",+ "de-at": "de",+ "de-ch": "de",+ "es": "es",+ "es-es": "es",+ "es-419": "es",+ "es-mx": "es",+ "ru": "ru",+ "ru-ru": "ru",+ "pt": "pt",+ "pt-br": "pt",+ "pt-pt": "pt",+ "it": "it",+ "it-it": "it",+}++_KNOWN_LANG_CODES = frozenset(_CANONICAL_LANGUAGES.keys())++SUPPORTED_LANGUAGE_PAIRS = frozenset(+ {+ frozenset({"en", "zh"}),+ frozenset({"en", "ja"}),+ frozenset({"en", "ko"}),+ frozenset({"en", "fr"}),+ frozenset({"en", "de"}),+ frozenset({"en", "es"}),+ frozenset({"en", "ru"}),+ frozenset({"en", "pt"}),+ frozenset({"en", "it"}),+ }+)+++def normalize_language(code: str | None) -> str:+ """Normalize a language tag to its canonical primary language code.++ Examples:+ 'zh-CN' -> 'zh'+ 'en-US' -> 'en'+ 'zh-Hans' -> 'zh'+ 'FR' -> 'fr'+ 'x-default' -> ''+ """+ if not code:+ return ""+ cleaned = code.strip().lower().replace("_", "-")+ if cleaned in ("x-default", "default"):+ return ""+ if cleaned in _CANONICAL_LANGUAGES:+ return _CANONICAL_LANGUAGES[cleaned]+ primary = cleaned.split("-", 1)[0]+ return _CANONICAL_LANGUAGES.get(primary, primary)+++def is_supported_pair(lang_a: str, lang_b: str) -> bool:+ """Check if two languages form a supported bilingual pairing."""+ if not lang_a or not lang_b or lang_a == lang_b:+ return False+ pair = frozenset({lang_a, lang_b})+ if pair in SUPPORTED_LANGUAGE_PAIRS:+ return True+ return "en" in pair or "zh" in pair+++def extract_url_language_and_stem(url: str) -> tuple[str, str]:+ """Extract language code (if in URL path) and the language-agnostic stem path.++ Handles leading language segments (/zh/docs/intro -> ("zh", "/docs/intro"))+ and common nested prefixes (/docs/zh/intro -> ("zh", "/docs/intro")).+ """+ parsed = urlparse(url)+ raw_path = parsed.path.rstrip("/")+ if not raw_path:+ return "", "/"++ segments = [s for s in raw_path.split("/") if s]+ if not segments:+ return "", "/"++ first_norm = normalize_language(segments[0])+ if segments[0].lower() in _KNOWN_LANG_CODES or first_norm in SUPPORTED_LANGUAGE_PAIRS:+ stem = "/" + "/".join(segments[1:])+ return first_norm, stem or "/"++ if len(segments) >= 2:+ second_norm = normalize_language(segments[1])+ if segments[1].lower() in _KNOWN_LANG_CODES or second_norm in SUPPORTED_LANGUAGE_PAIRS:+ stem = f"/{segments[0]}" + ("/" + "/".join(segments[2:]) if len(segments) > 2 else "")+ return second_norm, stem or "/"++ return "", raw_path or "/"+++def detect_bilingual_pairings(+ pages: list[Any],+ page_file_map: dict[str, str] | None = None,+) -> list[dict[str, Any]]:+ """Detect bilingual page pairings among crawled pages.++ Uses both explicit hreflang alternate declarations and URL language prefix+ patterns. Returns a deduplicated, deterministically sorted list of pairing+ records.+ """+ if not pages:+ return []++ file_map = page_file_map or {}+ pages_by_url = {page.url: page for page in pages}+ pairings: dict[str, dict[str, Any]] = {}++ # 1. Hreflang alternate links (highest confidence)+ for page in pages:+ p1_lang = (+ normalize_language(getattr(page, "language", ""))+ or extract_url_language_and_stem(page.url)[0]+ or "en"+ )+ alternate_links: dict[str, str] = getattr(page, "alternate_links", {})+ for hreflang, target_url in alternate_links.items():+ if target_url not in pages_by_url or target_url == page.url:+ continue+ target_page = pages_by_url[target_url]+ p2_lang = (+ normalize_language(hreflang)+ or normalize_language(getattr(target_page, "language", ""))+ or extract_url_language_and_stem(target_url)[0]+ or "zh"+ )+ if not is_supported_pair(p1_lang, p2_lang):+ continue++ if p1_lang == "en":+ src, tgt = page, target_page+ s_lang, t_lang = p1_lang, p2_lang+ elif p2_lang == "en":+ src, tgt = target_page, page+ s_lang, t_lang = p2_lang, p1_lang+ elif page.url <= target_page.url:+ src, tgt = page, target_page+ s_lang, t_lang = p1_lang, p2_lang+ else:+ src, tgt = target_page, page+ s_lang, t_lang = p2_lang, p1_lang++ pairing_id = hashlib.sha256(f"{src.url}::{tgt.url}".encode("utf-8")).hexdigest()[:16]+ pairings[pairing_id] = {+ "pairing_id": pairing_id,+ "source_url": src.url,+ "target_url": tgt.url,+ "source_file": file_map.get(src.url, ""),+ "target_file": file_map.get(tgt.url, ""),+ "source_lang": s_lang,+ "target_lang": t_lang,+ "pairing_method": "hreflang",+ }++ # 2. URL language stem pattern matching+ stem_groups: dict[str, list[tuple[Any, str]]] = {}+ for page in pages:+ url_lang, stem = extract_url_language_and_stem(page.url)+ page_lang = normalize_language(getattr(page, "language", "")) or url_lang+ stem_groups.setdefault(stem, []).append((page, page_lang))++ for stem, group in stem_groups.items():+ if len(group) < 2:+ continue+ for i in range(len(group)):+ for j in range(i + 1, len(group)):+ p1, l1 = group[i]+ p2, l2 = group[j]++ # If one page is unprefixed and the other has a language tag,+ # infer default 'en' for the unprefixed base documentation page.+ if not l1 and l2 and l2 != "en":+ l1 = "en"+ elif not l2 and l1 and l1 != "en":+ l2 = "en"++ if not is_supported_pair(l1, l2):+ continue++ if l1 == "en":+ src, tgt = p1, p2+ s_lang, t_lang = l1, l2+ elif l2 == "en":+ src, tgt = p2, p1+ s_lang, t_lang = l2, l1+ elif p1.url <= p2.url:+ src, tgt = p1, p2+ s_lang, t_lang = l1, l2+ else:+ src, tgt = p2, p1+ s_lang, t_lang = l2, l1++ pairing_id = hashlib.sha256(f"{src.url}::{tgt.url}".encode("utf-8")).hexdigest()[+ :16+ ]+ if pairing_id not in pairings:+ pairings[pairing_id] = {+ "pairing_id": pairing_id,+ "source_url": src.url,+ "target_url": tgt.url,+ "source_file": file_map.get(src.url, ""),+ "target_file": file_map.get(tgt.url, ""),+ "source_lang": s_lang,+ "target_lang": t_lang,+ "pairing_method": "url_pattern",+ }++ result = list(pairings.values())+ result.sort(key=lambda item: (item["source_url"], item["target_url"]))+ return result+++__all__ = [+ "detect_bilingual_pairings",+ "extract_url_language_and_stem",+ "is_supported_pair",+ "normalize_language",+]diff --git a/deeptutor/services/web_source/crawler.py b/deeptutor/services/web_source/crawler.pyindex f00f9f4de..3abe14615 100644--- a/deeptutor/services/web_source/crawler.py+++ b/deeptutor/services/web_source/crawler.py@@ -59,6 +59,8 @@ class CrawledPage:markdown: strcontent_hash: strheadings: list[dict] = field(default_factory=list)+ language: str = ""+ alternate_links: dict[str, str] = field(default_factory=dict)@dataclass@@ -304,6 +306,7 @@ async def _process_page(extract_article_markdown,extract_headings,extract_navigation,+ extract_page_language_and_alternates,)try:@@ -316,6 +319,7 @@ async def _process_page(nav = extract_navigation(html, final_url) if depth <= 1 else []page_headings = extract_headings(body)+ lang, alternates = extract_page_language_and_alternates(html, final_url or url)if len(body) > DEFAULT_MAX_CHARS:body = body[:DEFAULT_MAX_CHARS].rstrip() + "\n…[truncated]"@@ -326,6 +330,8 @@ async def _process_page(markdown=body,content_hash=content_hash,headings=page_headings,+ language=lang,+ alternate_links=alternates,)links: list[str] = []@@ -533,6 +539,7 @@ class CrawlDiff:pages_removed: list[str] = field(default_factory=list)changed_paths: list[str] = field(default_factory=list)navigation: dict = field(default_factory=dict)+ pairings: list[dict] = field(default_factory=list)@propertydef page_count(self) -> int:@@ -642,6 +649,12 @@ async def crawl_and_diff(page_urls,)+ # 6. Detect bilingual page pairings across discovered pages+ from deeptutor.services.web_source.bilingual import detect_bilingual_pairings++ page_file_map = {page.url: fname for page, fname in zip(result.pages, page_files)}+ pairings = detect_bilingual_pairings(result.pages, page_file_map)+return CrawlDiff(ok=True,url=url,@@ -654,4 +667,5 @@ async def crawl_and_diff(pages_removed=removed,changed_paths=changed_paths,navigation=nav_manifest,+ pairings=pairings,)diff --git a/deeptutor/services/web_source/html_extractor.py b/deeptutor/services/web_source/html_extractor.pyindex 89b00fc2a..19456c233 100644--- a/deeptutor/services/web_source/html_extractor.py+++ b/deeptutor/services/web_source/html_extractor.py@@ -579,3 +579,65 @@ def _clean_markdown(md: str) -> str:md = "\n".join(lines)md = re.sub(r"\n{3,}", "\n\n", md)return md.strip()+++def extract_page_language_and_alternates(+ raw_html: str,+ base_url: str = "",+) -> tuple[str, dict[str, str]]:+ """Extract page language attribute and alternate hreflang URLs.++ Returns:+ tuple of (language_code, {normalized_hreflang: absolute_url})+ """+ from urllib.parse import urldefrag, urljoin++ from lxml import html as lxml_html # nosec B410 - HTML parser, not XML++ from deeptutor.services.web_source.bilingual import normalize_language++ language = ""+ alternates: dict[str, str] = {}++ if not raw_html:+ return language, alternates++ try:+ tree = lxml_html.fromstring(raw_html)+ except Exception:+ return language, alternates++ # 1. Page language from <html lang="..."> or <html xml:lang="...">+ html_lang = tree.get("lang") or tree.get("xml:lang") or ""+ if html_lang:+ language = normalize_language(html_lang)++ # 2. Fall back to <meta http-equiv="content-language" content="..."> or <meta name="language" content="...">+ if not language:+ meta_nodes = tree.xpath("//meta[@http-equiv or @name]")+ for meta in meta_nodes:+ equiv = (meta.get("http-equiv") or meta.get("name") or "").strip().lower()+ if equiv in ("content-language", "language"):+ content = (meta.get("content") or "").strip()+ if content:+ language = normalize_language(content)+ break++ # 3. Alternate links: <link rel="alternate" hreflang="..." href="...">+ link_nodes = tree.xpath("//link[contains(@rel, 'alternate') and @hreflang and @href]")+ for link in link_nodes:+ hreflang = (link.get("hreflang") or "").strip()+ href = (link.get("href") or "").strip()+ if not hreflang or not href or hreflang.lower() in ("x-default", "default"):+ continue+ norm_lang = normalize_language(hreflang)+ if not norm_lang:+ continue+ href, _ = urldefrag(href)+ if base_url:+ resolved = urljoin(base_url, href)+ else:+ resolved = href+ alternates[norm_lang] = resolved++ return language, alternatesdiff --git a/deeptutor/services/web_source/repository.py b/deeptutor/services/web_source/repository.pyindex 614767171..1387d3db9 100644--- a/deeptutor/services/web_source/repository.py+++ b/deeptutor/services/web_source/repository.py@@ -61,6 +61,54 @@ class WebSourceSyncJob:}+@dataclass(slots=True)+class WebSourceBilingualPairing:+ """One durable bilingual pairing between two crawled pages."""++ owner_id: str+ kb_name: str+ source_id: str+ pairing_id: str+ source_url: str+ target_url: str+ source_file: str+ target_file: str+ source_lang: str+ target_lang: str+ pairing_method: str+ updated_at_ms: int++ @classmethod+ def from_row(cls, row: sqlite3.Row) -> "WebSourceBilingualPairing":+ return cls(+ owner_id=str(row["owner_id"]),+ kb_name=str(row["kb_name"]),+ source_id=str(row["source_id"]),+ pairing_id=str(row["pairing_id"]),+ source_url=str(row["source_url"]),+ target_url=str(row["target_url"]),+ source_file=str(row["source_file"]),+ target_file=str(row["target_file"]),+ source_lang=str(row["source_lang"]),+ target_lang=str(row["target_lang"]),+ pairing_method=str(row["pairing_method"]),+ updated_at_ms=int(row["updated_at_ms"]),+ )++ def public_dict(self) -> dict[str, Any]:+ return {+ "pairing_id": self.pairing_id,+ "source_url": self.source_url,+ "target_url": self.target_url,+ "source_file": self.source_file,+ "target_file": self.target_file,+ "source_lang": self.source_lang,+ "target_lang": self.target_lang,+ "pairing_method": self.pairing_method,+ "updated_at": self.updated_at_ms,+ }++class SQLiteWebSourceSyncRepository:"""WAL-backed job state shared by API requests and the scheduler."""@@ -102,6 +150,29 @@ class SQLiteWebSourceSyncRepository:"CREATE INDEX IF NOT EXISTS idx_web_source_sync_due ""ON web_source_sync_jobs(next_run_at_ms, state)")+ connection.execute(+ """+ CREATE TABLE IF NOT EXISTS web_source_bilingual_pairings (+ owner_id TEXT NOT NULL,+ kb_name TEXT NOT NULL,+ source_id TEXT NOT NULL,+ pairing_id TEXT NOT NULL,+ source_url TEXT NOT NULL,+ target_url TEXT NOT NULL,+ source_file TEXT NOT NULL,+ target_file TEXT NOT NULL,+ source_lang TEXT NOT NULL,+ target_lang TEXT NOT NULL,+ pairing_method TEXT NOT NULL,+ updated_at_ms INTEGER NOT NULL,+ PRIMARY KEY (owner_id, kb_name, source_id, pairing_id)+ )+ """+ )+ connection.execute(+ "CREATE INDEX IF NOT EXISTS idx_web_source_pairings "+ "ON web_source_bilingual_pairings(owner_id, kb_name, source_id)"+ )@staticmethoddef key(job: WebSourceSyncJob) -> tuple[str, str, str]:@@ -148,6 +219,13 @@ class SQLiteWebSourceSyncRepository:""",(owner_id, kb_name, source_id),)+ connection.execute(+ """+ DELETE FROM web_source_bilingual_pairings+ WHERE owner_id=? AND kb_name=? AND source_id=?+ """,+ (owner_id, kb_name, source_id),+ )connection.execute("COMMIT")except Exception:connection.execute("ROLLBACK")@@ -419,6 +497,13 @@ class SQLiteWebSourceSyncRepository:""",job_key,)+ connection.execute(+ """+ DELETE FROM web_source_bilingual_pairings+ WHERE owner_id=? AND kb_name=? AND source_id=?+ """,+ job_key,+ )return int(cursor.rowcount or 0) > 0def retry(self, job_key: tuple[str, str, str]) -> WebSourceSyncJob | None:@@ -451,5 +536,97 @@ class SQLiteWebSourceSyncRepository:).fetchone()return WebSourceSyncJob.from_row(row) if row else None+ def record_pairings(+ self,+ owner_id: str,+ kb_name: str,+ source_id: str,+ pairings: list[dict[str, Any]],+ ) -> list[WebSourceBilingualPairing]:+ """Atomically replace bilingual pairings for one web source."""+ now = self._now_ms()+ with self._connect() as connection:+ connection.execute("BEGIN IMMEDIATE")+ try:+ connection.execute(+ """+ DELETE FROM web_source_bilingual_pairings+ WHERE owner_id=? AND kb_name=? AND source_id=?+ """,+ (owner_id, kb_name, source_id),+ )+ for item in pairings:+ connection.execute(+ """+ INSERT INTO web_source_bilingual_pairings+ (owner_id, kb_name, source_id, pairing_id, source_url,+ target_url, source_file, target_file, source_lang,+ target_lang, pairing_method, updated_at_ms)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ """,+ (+ owner_id,+ kb_name,+ source_id,+ str(item["pairing_id"]),+ str(item["source_url"]),+ str(item["target_url"]),+ str(item.get("source_file", "")),+ str(item.get("target_file", "")),+ str(item.get("source_lang", "")),+ str(item.get("target_lang", "")),+ str(item.get("pairing_method", "hreflang")),+ now,+ ),+ )+ connection.execute("COMMIT")+ except Exception:+ connection.execute("ROLLBACK")+ raise+ return self.list_pairings(owner_id, kb_name, source_id)++ def list_pairings(+ self,+ owner_id: str,+ kb_name: str,+ source_id: str | None = None,+ ) -> list[WebSourceBilingualPairing]:+ with self._connect() as connection:+ if source_id is not None:+ rows = connection.execute(+ """+ SELECT * FROM web_source_bilingual_pairings+ WHERE owner_id=? AND kb_name=? AND source_id=?+ ORDER BY source_url, target_url+ """,+ (owner_id, kb_name, source_id),+ ).fetchall()+ else:+ rows = connection.execute(+ """+ SELECT * FROM web_source_bilingual_pairings+ WHERE owner_id=? AND kb_name=?+ ORDER BY source_id, source_url, target_url+ """,+ (owner_id, kb_name),+ ).fetchall()+ return [WebSourceBilingualPairing.from_row(row) for row in rows]++ def delete_pairings(+ self,+ owner_id: str,+ kb_name: str,+ source_id: str,+ ) -> int:+ with self._connect() as connection:+ cursor = connection.execute(+ """+ DELETE FROM web_source_bilingual_pairings+ WHERE owner_id=? AND kb_name=? AND source_id=?+ """,+ (owner_id, kb_name, source_id),+ )+ return int(cursor.rowcount or 0)+-__all__ = ["SQLiteWebSourceSyncRepository", "WebSourceSyncJob"]+__all__ = ["SQLiteWebSourceSyncRepository", "WebSourceBilingualPairing", "WebSourceSyncJob"]diff --git a/deeptutor/services/web_source/sync.py b/deeptutor/services/web_source/sync.pyindex 0bcba2d7c..fcf63aa62 100644--- a/deeptutor/services/web_source/sync.py+++ b/deeptutor/services/web_source/sync.py@@ -35,6 +35,7 @@ class WebSyncResult:pages_removed: int = 0pages_unchanged: int = 0error: str = ""+ bilingual_pairs_count: int = 0@propertydef total_changes(self) -> int:@@ -185,16 +186,33 @@ async def sync_source(last_sync_status="success",last_sync_error=None,navigation=diff.navigation,+ bilingual_pairings=diff.pairings,)+ try:+ from deeptutor.multi_user.context import get_current_user+ from deeptutor.services.web_source.scheduler import get_web_source_sync_scheduler++ user = get_current_user()+ scheduler = get_web_source_sync_scheduler()+ scheduler.repo.record_pairings(+ owner_id=user.id,+ kb_name=kb_name,+ source_id=source["id"],+ pairings=diff.pairings,+ )+ except Exception:+ logger.debug("Failed to record pairings in SQLite repo", exc_info=True)+logger.info(- "Web sync %s: +%d ~%d -%d (%d unchanged), %d indexed",+ "Web sync %s: +%d ~%d -%d (%d unchanged), %d indexed, %d bilingual pairs",diff.url,len(diff.pages_added),len(diff.pages_updated),removed_count,len(diff.pages_unchanged),indexed,+ len(diff.pairings),)return WebSyncResult(@@ -203,6 +221,7 @@ async def sync_source(pages_updated=len(diff.pages_updated),pages_removed=removed_count,pages_unchanged=len(diff.pages_unchanged),+ bilingual_pairs_count=len(diff.pairings),)diff --git a/tests/api/test_web_source_schedule_api.py b/tests/api/test_web_source_schedule_api.pyindex 5395825e1..5ac54faff 100644--- a/tests/api/test_web_source_schedule_api.py+++ b/tests/api/test_web_source_schedule_api.py@@ -99,3 +99,58 @@ def test_schedule_rejects_out_of_range_interval(client) -> None:json={"auto_sync_enabled": True, "sync_interval_hours": 0},)assert response.status_code == 422+++def test_web_source_pairings_api(client) -> None:+ test_client, manager = client+ source = manager.add_web_source("kb", "https://example.com/docs/")+ source_id = source["id"]++ # Initial web-sources listing has empty pairings+ list_res = test_client.get("/api/knowledge-bases/kb/web-sources")+ assert list_res.status_code == 200+ sources = list_res.json()+ assert len(sources) == 1+ assert "bilingual_pairings" in sources[0]+ assert sources[0]["bilingual_pairings"] == []++ # Initial pairings endpoint returns empty+ pair_res = test_client.get(f"/api/knowledge-bases/kb/web-source/{source_id}/pairings")+ assert pair_res.status_code == 200+ assert pair_res.json() == []++ # Record pairings into repo+ scheduler_module._scheduler.repo.record_pairings(+ "local-admin",+ "kb",+ source_id,+ [+ {+ "pairing_id": "pair-1",+ "source_url": "https://example.com/docs/intro",+ "target_url": "https://example.com/zh/docs/intro",+ "source_file": "_web/s/docs/intro.md",+ "target_file": "_web/s/zh/docs/intro.md",+ "source_lang": "en",+ "target_lang": "zh",+ "pairing_method": "hreflang",+ }+ ],+ )++ pair_res = test_client.get(f"/api/knowledge-bases/kb/web-source/{source_id}/pairings")+ assert pair_res.status_code == 200+ pairings = pair_res.json()+ assert len(pairings) == 1+ assert pairings[0]["pairing_id"] == "pair-1"+ assert pairings[0]["source_lang"] == "en"+ assert pairings[0]["target_lang"] == "zh"++ # Non-existent source returns 404+ missing_res = test_client.get("/api/knowledge-bases/kb/web-source/nonexistent/pairings")+ assert missing_res.status_code == 404++ # Deleting source removes pairings+ del_res = test_client.delete(f"/api/knowledge-bases/kb/web-source/{source_id}")+ assert del_res.status_code == 200+ assert scheduler_module._scheduler.repo.list_pairings("local-admin", "kb", source_id) == []diff --git a/tests/services/test_web_source_sync.py b/tests/services/test_web_source_sync.pyindex 1312c9716..b9d16473e 100644--- a/tests/services/test_web_source_sync.py+++ b/tests/services/test_web_source_sync.py@@ -489,3 +489,53 @@ async def test_sync_persists_navigation(tmp_path: Path):assert len(nav["nodes"]) >= 1assert sources_after[0]["page_count"] == 2+++async def test_sync_persists_bilingual_pairings(tmp_path: Path):+ base_dir, kb_dir = _make_kb(tmp_path)+ from deeptutor.knowledge.manager import KnowledgeBaseManager++ mgr = KnowledgeBaseManager(base_dir=base_dir)+ source = mgr.add_web_source("kb", "https://example.com/docs/")++ mock_result = CrawlResult(+ pages=[+ CrawledPage(+ url="https://example.com/docs/en/intro",+ title="Intro",+ markdown="# Intro",+ content_hash="aaa",+ language="en",+ ),+ CrawledPage(+ url="https://example.com/docs/zh/intro",+ title="介绍",+ markdown="# 介绍",+ content_hash="bbb",+ language="zh",+ ),+ ],+ )++ with patch(+ "deeptutor.services.web_source.crawler.crawl_docs_site", new_callable=AsyncMock+ ) as mock_crawl:+ mock_crawl.return_value = mock_result+ with patch(+ "deeptutor.knowledge.add_documents.add_documents", new_callable=AsyncMock+ ) as mock_add:+ mock_add.return_value = 2+ result = await sync_source("kb", source, base_dir=base_dir)++ assert result.ok is True+ assert result.bilingual_pairs_count == 1++ sources_after = mgr.get_web_sources("kb")+ assert len(sources_after) == 1+ pairings = sources_after[0].get("bilingual_pairings", [])+ assert len(pairings) == 1+ assert pairings[0]["source_url"] == "https://example.com/docs/en/intro"+ assert pairings[0]["target_url"] == "https://example.com/docs/zh/intro"+ assert pairings[0]["source_lang"] == "en"+ assert pairings[0]["target_lang"] == "zh"diff --git a/tests/services/web_source/test_bilingual_pairing.py b/tests/services/web_source/test_bilingual_pairing.pynew file mode 100644index 000000000..fa13527ac--- /dev/null+++ b/tests/services/web_source/test_bilingual_pairing.py@@ -0,0 +1,232 @@+"""Tests for web-source bilingual page pairing and persistence."""++from __future__ import annotations++from pathlib import Path+from unittest.mock import MagicMock++import pytest++from deeptutor.services.web_source.bilingual import (+ detect_bilingual_pairings,+ extract_url_language_and_stem,+ is_supported_pair,+ normalize_language,+)+from deeptutor.services.web_source.crawler import CrawledPage+from deeptutor.services.web_source.html_extractor import extract_page_language_and_alternates+from deeptutor.services.web_source.repository import (+ SQLiteWebSourceSyncRepository,+ WebSourceBilingualPairing,+)+++def test_normalize_language() -> None:+ assert normalize_language("en-US") == "en"+ assert normalize_language("EN") == "en"+ assert normalize_language("zh-CN") == "zh"+ assert normalize_language("zh_cn") == "zh"+ assert normalize_language("zh-Hans") == "zh"+ assert normalize_language("zh-Hant") == "zh"+ assert normalize_language("ja-JP") == "ja"+ assert normalize_language("x-default") == ""+ assert normalize_language("") == ""+ assert normalize_language(None) == ""+++def test_is_supported_pair() -> None:+ assert is_supported_pair("en", "zh") is True+ assert is_supported_pair("zh", "en") is True+ assert is_supported_pair("en", "ja") is True+ assert is_supported_pair("en", "fr") is True+ assert is_supported_pair("en", "en") is False+ assert is_supported_pair("", "en") is False+++def test_extract_url_language_and_stem() -> None:+ assert extract_url_language_and_stem("https://example.com/en/docs/intro") == (+ "en",+ "/docs/intro",+ )+ assert extract_url_language_and_stem("https://example.com/zh-cn/docs/intro") == (+ "zh",+ "/docs/intro",+ )+ assert extract_url_language_and_stem("https://example.com/docs/en/intro") == (+ "en",+ "/docs/intro",+ )+ assert extract_url_language_and_stem("https://example.com/docs/zh/intro") == (+ "zh",+ "/docs/intro",+ )+ assert extract_url_language_and_stem("https://example.com/docs/intro") == (+ "",+ "/docs/intro",+ )+ assert extract_url_language_and_stem("https://example.com/") == ("", "/")+++def test_extract_page_language_and_alternates() -> None:+ html = """<!DOCTYPE html>+ <html lang="en">+ <head>+ <link rel="alternate" hreflang="zh-CN" href="/zh/docs/intro" />+ <link rel="alternate" hreflang="en" href="/docs/intro" />+ <link rel="alternate" hreflang="x-default" href="/docs/intro" />+ </head>+ <body><h1>Introduction</h1></body>+ </html>"""+ lang, alternates = extract_page_language_and_alternates(html, "https://example.com/docs/intro")+ assert lang == "en"+ assert alternates == {+ "zh": "https://example.com/zh/docs/intro",+ "en": "https://example.com/docs/intro",+ }+++def test_detect_bilingual_pairings_hreflang() -> None:+ p_en = CrawledPage(+ url="https://example.com/docs/intro",+ title="Intro",+ markdown="# Intro",+ content_hash="hash-en",+ language="en",+ alternate_links={"zh": "https://example.com/zh/docs/intro"},+ )+ p_zh = CrawledPage(+ url="https://example.com/zh/docs/intro",+ title="介绍",+ markdown="# 介绍",+ content_hash="hash-zh",+ language="zh",+ alternate_links={"en": "https://example.com/docs/intro"},+ )++ file_map = {+ "https://example.com/docs/intro": "_web/abc/docs/intro.md",+ "https://example.com/zh/docs/intro": "_web/abc/zh/docs/intro.md",+ }++ pairings = detect_bilingual_pairings([p_en, p_zh], file_map)+ assert len(pairings) == 1+ pair = pairings[0]+ assert pair["source_url"] == "https://example.com/docs/intro"+ assert pair["target_url"] == "https://example.com/zh/docs/intro"+ assert pair["source_lang"] == "en"+ assert pair["target_lang"] == "zh"+ assert pair["source_file"] == "_web/abc/docs/intro.md"+ assert pair["target_file"] == "_web/abc/zh/docs/intro.md"+ assert pair["pairing_method"] == "hreflang"+++def test_detect_bilingual_pairings_url_pattern() -> None:+ p_en = CrawledPage(+ url="https://example.com/en/guide/install",+ title="Install",+ markdown="# Install",+ content_hash="hash-en",+ language="en",+ )+ p_zh = CrawledPage(+ url="https://example.com/zh/guide/install",+ title="安装",+ markdown="# 安装",+ content_hash="hash-zh",+ language="zh",+ )++ pairings = detect_bilingual_pairings([p_en, p_zh])+ assert len(pairings) == 1+ pair = pairings[0]+ assert pair["source_url"] == "https://example.com/en/guide/install"+ assert pair["target_url"] == "https://example.com/zh/guide/install"+ assert pair["source_lang"] == "en"+ assert pair["target_lang"] == "zh"+ assert pair["pairing_method"] == "url_pattern"+++def test_repository_pairings_crud(tmp_path: Path) -> None:+ repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")+ owner_id = "user-123"+ kb_name = "test-kb"+ source_id = "src-456"++ pairings_data = [+ {+ "pairing_id": "pair-001",+ "source_url": "https://example.com/docs/intro",+ "target_url": "https://example.com/zh/docs/intro",+ "source_file": "_web/s1/docs/intro.md",+ "target_file": "_web/s1/zh/docs/intro.md",+ "source_lang": "en",+ "target_lang": "zh",+ "pairing_method": "hreflang",+ }+ ]++ saved = repo.record_pairings(owner_id, kb_name, source_id, pairings_data)+ assert len(saved) == 1+ assert isinstance(saved[0], WebSourceBilingualPairing)+ assert saved[0].pairing_id == "pair-001"+ assert saved[0].source_url == "https://example.com/docs/intro"++ # List pairings for specific source+ listed = repo.list_pairings(owner_id, kb_name, source_id)+ assert len(listed) == 1+ assert listed[0].public_dict()["pairing_id"] == "pair-001"++ # List pairings across all sources in KB+ all_kb_pairs = repo.list_pairings(owner_id, kb_name)+ assert len(all_kb_pairs) == 1++ # Atomic replace with empty+ replaced = repo.record_pairings(owner_id, kb_name, source_id, [])+ assert len(replaced) == 0+ assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 0+++def test_delete_source_cleans_up_pairings(tmp_path: Path) -> None:+ repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")+ owner_id = "user-123"+ kb_name = "test-kb"+ source_id = "src-456"++ repo.ensure_source((owner_id, kb_name, source_id))+ repo.record_pairings(+ owner_id,+ kb_name,+ source_id,+ [+ {+ "pairing_id": "pair-001",+ "source_url": "https://example.com/docs/intro",+ "target_url": "https://example.com/zh/docs/intro",+ "source_file": "_web/s1/docs/intro.md",+ "target_file": "_web/s1/zh/docs/intro.md",+ "source_lang": "en",+ "target_lang": "zh",+ "pairing_method": "hreflang",+ }+ ],+ )+ assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 1++ # Deleting source deletes both job and pairings+ repo.delete((owner_id, kb_name, source_id))+ assert repo.get((owner_id, kb_name, source_id)) is None+ assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 0+++def test_non_web_kb_isolation(tmp_path: Path) -> None:+ """Ensure non-web knowledge bases are isolated and do not interact with pairings."""+ repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")+ owner_id = "user-123"++ # Querying a non-web KB returns empty list, never raises+ pairs = repo.list_pairings(owner_id, "marginnote-kb")+ assert pairs == []++ # Deleting non-existent returns 0+ deleted = repo.delete_pairings(owner_id, "marginnote-kb", "src-nonexistent")+ assert deleted == 0diff --git a/web/components/knowledge/KbWebSourcesSection.tsx b/web/components/knowledge/KbWebSourcesSection.tsxindex 31fb26e51..d5c8645e2 100644--- a/web/components/knowledge/KbWebSourcesSection.tsx+++ b/web/components/knowledge/KbWebSourcesSection.tsx@@ -381,6 +381,19 @@ function WebSourceCard({{t("Pages")}: {source.page_count}</span>)}+ {source.bilingual_pairings && source.bilingual_pairings.length > 0 && (+ <span+ className="font-medium text-emerald-600 dark:text-emerald-400"+ title={source.bilingual_pairings+ .map(+ (p) =>+ `${p.source_lang.toUpperCase()} ↔ ${p.target_lang.toUpperCase()}: ${p.source_url} ↔ ${p.target_url}`,+ )+ .join("\n")}+ >+ {t("Bilingual pairs")}: {source.bilingual_pairings.length}+ </span>+ )}{lastSync && (<span>{t("Synced")}: {lastSync}diff --git a/web/features/knowledge/api/client.ts b/web/features/knowledge/api/client.tsindex 083f23553..02b6f63a6 100644--- a/web/features/knowledge/api/client.ts+++ b/web/features/knowledge/api/client.ts@@ -1656,6 +1656,18 @@ export async function syncGitHubSources(// ── Web sources ──────────────────────────────────────────────────────+export interface BilingualPairing {+ pairing_id: string;+ source_url: string;+ target_url: string;+ source_file: string;+ target_file: string;+ source_lang: string;+ target_lang: string;+ pairing_method: string;+ updated_at?: number;+}+export interface WebSource {id: string;url: string;@@ -1669,6 +1681,7 @@ export interface WebSource {last_sync_status: string;last_sync_error: string | null;added_at: string;+ bilingual_pairings?: BilingualPairing[];}export interface AddWebSourcePayload {@@ -1728,6 +1741,28 @@ export async function listWebSources(return (await res.json()) as WebSource[];}+export async function listWebSourcePairings(+ kbName: string,+ sourceId: string,+ options?: { signal?: AbortSignal },+): Promise<BilingualPairing[]> {+ const res = await apiFetch(+ apiUrl(+ `/api/knowledge-bases/${encodeURIComponent(kbName)}/web-source/${encodeURIComponent(sourceId)}/pairings`,+ ),+ { signal: options?.signal },+ );+ if (!res.ok) {+ throw new Error(+ await readErrorDetail(+ res,+ `Failed to list web source pairings (${res.status})`,+ ),+ );+ }+ return (await res.json()) as BilingualPairing[];+}+export async function addWebSource(kbName: string,payload: AddWebSourcePayload,diff --git a/web/features/knowledge/api/sources.ts b/web/features/knowledge/api/sources.tsindex 82f970682..ce499a00b 100644--- a/web/features/knowledge/api/sources.ts+++ b/web/features/knowledge/api/sources.ts@@ -6,6 +6,7 @@ export {listGitHubSources,listLinkedFolders,listWebSources,+ listWebSourcePairings,listWebSourceSyncJobs,removeGitHubSource,removeWebSource,@@ -20,6 +21,7 @@ export {export type {AddGitHubSourcePayload,AddWebSourcePayload,+ BilingualPairing,GitHubSource,GitHubSyncResult,LinkedFolderInfo,diff --git a/web/features/knowledge/model/types.ts b/web/features/knowledge/model/types.tsindex bed2800fc..ec738803f 100644--- a/web/features/knowledge/model/types.ts+++ b/web/features/knowledge/model/types.ts@@ -3,6 +3,7 @@ export type {EmbeddingUsage,AddGitHubSourcePayload,AddWebSourcePayload,+ BilingualPairing,EnginePreflight,GitHubSource,GitHubSyncResult,diff --git a/web/locales/de/app.json b/web/locales/de/app.jsonindex 80071a718..a91d2cc4c 100644--- a/web/locales/de/app.json+++ b/web/locales/de/app.json@@ -811,6 +811,7 @@"Schedule": "Zeitplan","Next run": "Nächster Lauf","Automatic sync": "Automatische Synchronisierung",+ "Bilingual pairs": "Zweisprachige Paare","Interval hours": "Intervall (Stunden)","Cancel sync": "Synchronisierung abbrechen","Retry sync": "Synchronisierung erneut versuchen",diff --git a/web/locales/en/app.json b/web/locales/en/app.jsonindex c6167eb21..d88102b0b 100644--- a/web/locales/en/app.json+++ b/web/locales/en/app.json@@ -811,6 +811,7 @@"Schedule": "Schedule","Next run": "Next run","Automatic sync": "Automatic sync",+ "Bilingual pairs": "Bilingual pairs","Interval hours": "Interval hours","Cancel sync": "Cancel sync","Retry sync": "Retry sync",diff --git a/web/locales/fr/app.json b/web/locales/fr/app.jsonindex 996e47743..2f4205924 100644--- a/web/locales/fr/app.json+++ b/web/locales/fr/app.json@@ -4279,6 +4279,7 @@"Schedule": "Planification","Next run": "Prochaine exécution","Automatic sync": "Synchronisation automatique",+ "Bilingual pairs": "Paires bilingues","Interval hours": "Intervalle (heures)","Cancel sync": "Annuler la synchronisation","Retry sync": "Réessayer la synchronisation",diff --git a/web/locales/uk/app.json b/web/locales/uk/app.jsonindex 36788d166..fe91bb306 100644--- a/web/locales/uk/app.json+++ b/web/locales/uk/app.json@@ -4054,6 +4054,7 @@"Schedule": "Розклад","Next run": "Наступний запуск","Automatic sync": "Автоматична синхронізація",+ "Bilingual pairs": "Двомовні пари","Interval hours": "Інтервал (год)","Cancel sync": "Скасувати синхронізацію","Retry sync": "Повторити синхронізацію",diff --git a/web/locales/zh/app.json b/web/locales/zh/app.jsonindex 4312257b3..53ba5032d 100644--- a/web/locales/zh/app.json+++ b/web/locales/zh/app.json@@ -840,6 +840,7 @@"Schedule": "调度","Next run": "下次运行","Automatic sync": "自动同步",+ "Bilingual pairs": "双语配对","Interval hours": "间隔小时数","Cancel sync": "取消同步","Retry sync": "重试同步",diff --git a/web/tests/web-source-bilingual-pairing.spec.tsx b/web/tests/web-source-bilingual-pairing.spec.tsxnew file mode 100644index 000000000..fec11aa2c--- /dev/null+++ b/web/tests/web-source-bilingual-pairing.spec.tsx@@ -0,0 +1,47 @@+import React from "react";+import { cleanup, render, screen } from "@testing-library/react";+import { afterEach, expect, it, vi } from "vitest";+import KbWebSourcesSection from "@/components/knowledge/KbWebSourcesSection";+import * as sourcesApi from "@/features/knowledge/api/sources";++vi.mock("react-i18next", () => ({+ useTranslation: () => ({ t: (key: string) => key }),+}));++afterEach(cleanup);++it("renders bilingual pairs count when pairings exist", async () => {+ vi.spyOn(sourcesApi, "listWebSources").mockResolvedValue([+ {+ id: "src-1",+ url: "https://example.com/docs/",+ max_depth: 3,+ max_pages: 50,+ enabled: true,+ auto_sync_enabled: true,+ sync_interval_hours: 24,+ page_count: 10,+ last_synced_at: "2026-09-28T12:00:00Z",+ last_sync_status: "success",+ last_sync_error: null,+ added_at: "2026-09-28T10:00:00Z",+ bilingual_pairings: [+ {+ pairing_id: "pair-1",+ source_url: "https://example.com/docs/en/intro",+ target_url: "https://example.com/docs/zh/intro",+ source_file: "intro.md",+ target_file: "intro.zh.md",+ source_lang: "en",+ target_lang: "zh",+ pairing_method: "hreflang",+ },+ ],+ },+ ]);+ vi.spyOn(sourcesApi, "listWebSourceSyncJobs").mockResolvedValue([]);++ render(<KbWebSourcesSection kbName="test-kb" />);++ expect(await screen.findByText(/Bilingual pairs: 1/)).toBeDefined();+});