Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

7.8 KB
Raw Blame
"""Tests for web-source bilingual page pairing and persistence."""

from __future__ import annotations

from pathlib import Path
from unittest.mock import MagicMock

import pytest

from deeptutor.services.web_source.bilingual import (
    detect_bilingual_pairings,
    extract_url_language_and_stem,
    is_supported_pair,
    normalize_language,
)
from deeptutor.services.web_source.crawler import CrawledPage
from deeptutor.services.web_source.html_extractor import extract_page_language_and_alternates
from deeptutor.services.web_source.repository import (
    SQLiteWebSourceSyncRepository,
    WebSourceBilingualPairing,
)


def test_normalize_language() -> None:
    assert normalize_language("en-US") == "en"
    assert normalize_language("EN") == "en"
    assert normalize_language("zh-CN") == "zh"
    assert normalize_language("zh_cn") == "zh"
    assert normalize_language("zh-Hans") == "zh"
    assert normalize_language("zh-Hant") == "zh"
    assert normalize_language("ja-JP") == "ja"
    assert normalize_language("x-default") == ""
    assert normalize_language("") == ""
    assert normalize_language(None) == ""


def test_is_supported_pair() -> None:
    assert is_supported_pair("en", "zh") is True
    assert is_supported_pair("zh", "en") is True
    assert is_supported_pair("en", "ja") is True
    assert is_supported_pair("en", "fr") is True
    assert is_supported_pair("en", "en") is False
    assert is_supported_pair("", "en") is False


def test_extract_url_language_and_stem() -> None:
    assert extract_url_language_and_stem("https://example.com/en/docs/intro") == (
        "en",
        "/docs/intro",
    )
    assert extract_url_language_and_stem("https://example.com/zh-cn/docs/intro") == (
        "zh",
        "/docs/intro",
    )
    assert extract_url_language_and_stem("https://example.com/docs/en/intro") == (
        "en",
        "/docs/intro",
    )
    assert extract_url_language_and_stem("https://example.com/docs/zh/intro") == (
        "zh",
        "/docs/intro",
    )
    assert extract_url_language_and_stem("https://example.com/docs/intro") == (
        "",
        "/docs/intro",
    )
    assert extract_url_language_and_stem("https://example.com/") == ("", "/")


def test_extract_page_language_and_alternates() -> None:
    html = """<!DOCTYPE html>
    <html lang="en">
    <head>
        <link rel="alternate" hreflang="zh-CN" href="/zh/docs/intro" />
        <link rel="alternate" hreflang="en" href="/docs/intro" />
        <link rel="alternate" hreflang="x-default" href="/docs/intro" />
    </head>
    <body><h1>Introduction</h1></body>
    </html>"""
    lang, alternates = extract_page_language_and_alternates(html, "https://example.com/docs/intro")
    assert lang == "en"
    assert alternates == {
        "zh": "https://example.com/zh/docs/intro",
        "en": "https://example.com/docs/intro",
    }


def test_detect_bilingual_pairings_hreflang() -> None:
    p_en = CrawledPage(
        url="https://example.com/docs/intro",
        title="Intro",
        markdown="# Intro",
        content_hash="hash-en",
        language="en",
        alternate_links={"zh": "https://example.com/zh/docs/intro"},
    )
    p_zh = CrawledPage(
        url="https://example.com/zh/docs/intro",
        title="介绍",
        markdown="# 介绍",
        content_hash="hash-zh",
        language="zh",
        alternate_links={"en": "https://example.com/docs/intro"},
    )

    file_map = {
        "https://example.com/docs/intro": "_web/abc/docs/intro.md",
        "https://example.com/zh/docs/intro": "_web/abc/zh/docs/intro.md",
    }

    pairings = detect_bilingual_pairings([p_en, p_zh], file_map)
    assert len(pairings) == 1
    pair = pairings[0]
    assert pair["source_url"] == "https://example.com/docs/intro"
    assert pair["target_url"] == "https://example.com/zh/docs/intro"
    assert pair["source_lang"] == "en"
    assert pair["target_lang"] == "zh"
    assert pair["source_file"] == "_web/abc/docs/intro.md"
    assert pair["target_file"] == "_web/abc/zh/docs/intro.md"
    assert pair["pairing_method"] == "hreflang"


def test_detect_bilingual_pairings_url_pattern() -> None:
    p_en = CrawledPage(
        url="https://example.com/en/guide/install",
        title="Install",
        markdown="# Install",
        content_hash="hash-en",
        language="en",
    )
    p_zh = CrawledPage(
        url="https://example.com/zh/guide/install",
        title="安装",
        markdown="# 安装",
        content_hash="hash-zh",
        language="zh",
    )

    pairings = detect_bilingual_pairings([p_en, p_zh])
    assert len(pairings) == 1
    pair = pairings[0]
    assert pair["source_url"] == "https://example.com/en/guide/install"
    assert pair["target_url"] == "https://example.com/zh/guide/install"
    assert pair["source_lang"] == "en"
    assert pair["target_lang"] == "zh"
    assert pair["pairing_method"] == "url_pattern"


def test_repository_pairings_crud(tmp_path: Path) -> None:
    repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")
    owner_id = "user-123"
    kb_name = "test-kb"
    source_id = "src-456"

    pairings_data = [
        {
            "pairing_id": "pair-001",
            "source_url": "https://example.com/docs/intro",
            "target_url": "https://example.com/zh/docs/intro",
            "source_file": "_web/s1/docs/intro.md",
            "target_file": "_web/s1/zh/docs/intro.md",
            "source_lang": "en",
            "target_lang": "zh",
            "pairing_method": "hreflang",
        }
    ]

    saved = repo.record_pairings(owner_id, kb_name, source_id, pairings_data)
    assert len(saved) == 1
    assert isinstance(saved[0], WebSourceBilingualPairing)
    assert saved[0].pairing_id == "pair-001"
    assert saved[0].source_url == "https://example.com/docs/intro"

    # List pairings for specific source
    listed = repo.list_pairings(owner_id, kb_name, source_id)
    assert len(listed) == 1
    assert listed[0].public_dict()["pairing_id"] == "pair-001"

    # List pairings across all sources in KB
    all_kb_pairs = repo.list_pairings(owner_id, kb_name)
    assert len(all_kb_pairs) == 1

    # Atomic replace with empty
    replaced = repo.record_pairings(owner_id, kb_name, source_id, [])
    assert len(replaced) == 0
    assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 0


def test_delete_source_cleans_up_pairings(tmp_path: Path) -> None:
    repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")
    owner_id = "user-123"
    kb_name = "test-kb"
    source_id = "src-456"

    repo.ensure_source((owner_id, kb_name, source_id))
    repo.record_pairings(
        owner_id,
        kb_name,
        source_id,
        [
            {
                "pairing_id": "pair-001",
                "source_url": "https://example.com/docs/intro",
                "target_url": "https://example.com/zh/docs/intro",
                "source_file": "_web/s1/docs/intro.md",
                "target_file": "_web/s1/zh/docs/intro.md",
                "source_lang": "en",
                "target_lang": "zh",
                "pairing_method": "hreflang",
            }
        ],
    )
    assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 1

    # Deleting source deletes both job and pairings
    repo.delete((owner_id, kb_name, source_id))
    assert repo.get((owner_id, kb_name, source_id)) is None
    assert len(repo.list_pairings(owner_id, kb_name, source_id)) == 0


def test_non_web_kb_isolation(tmp_path: Path) -> None:
    """Ensure non-web knowledge bases are isolated and do not interact with pairings."""
    repo = SQLiteWebSourceSyncRepository(tmp_path / "test-jobs.sqlite")
    owner_id = "user-123"

    # Querying a non-web KB returns empty list, never raises
    pairs = repo.list_pairings(owner_id, "marginnote-kb")
    assert pairs == []

    # Deleting non-existent returns 0
    deleted = repo.delete_pairings(owner_id, "marginnote-kb", "src-nonexistent")
    assert deleted == 0