DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
test: add tex_downloader/tex_chunker toolchain tests (#1748)
evan188199-tech614834dad0dd
diff --git a/tests/tools/test_tex_tools.py b/tests/tools/test_tex_tools.pynew file mode 100644index 000000000..e9f67b886--- /dev/null+++ b/tests/tools/test_tex_tools.py@@ -0,0 +1,529 @@+"""Tests for deeptutor.tools.tex_downloader and deeptutor.tools.tex_chunker.++Covers the happy paths (download + extract + locate main tex; section/paragraph+chunking), boundary inputs (empty, oversized, malformed archives, path-traversal+members) and the mocked network-failure branches. No real network access.+"""++from __future__ import annotations++import io+from pathlib import Path+import tarfile+from unittest.mock import MagicMock+import zipfile++import pytest+import requests++from deeptutor.tools.tex_chunker import TexChunker+import deeptutor.tools.tex_downloader as tex_downloader_module+from deeptutor.tools.tex_downloader import (+ TexDownloader,+ TexDownloadResult,+ read_tex_file,+)++MIN_TEX = "\\documentclass{article}\n\\begin{document}\nHello arxiv.\n\\end{document}\n"+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _make_tar_bytes(files: dict[str, bytes]) -> bytes:+ buf = io.BytesIO()+ with tarfile.open(fileobj=buf, mode="w") as tar:+ for name, data in files.items():+ info = tarfile.TarInfo(name=name)+ info.size = len(data)+ tar.addfile(info, io.BytesIO(data))+ return buf.getvalue()+++def _make_zip_bytes(files: dict[str, bytes]) -> bytes:+ buf = io.BytesIO()+ with zipfile.ZipFile(buf, "w") as zf:+ for name, data in files.items():+ zf.writestr(name, data)+ return buf.getvalue()+++def _mock_response(content: bytes) -> MagicMock:+ response = MagicMock()+ response.content = content+ response.raise_for_status.return_value = None+ return response+++def downloader(tmp_path: Path) -> TexDownloader:+ return TexDownloader(workspace_dir=str(tmp_path / "ws"))+++# ---------------------------------------------------------------------------+# TexDownloader — construction and result object+# ---------------------------------------------------------------------------+++class TestTexDownloaderBasics:+ def test_workspace_dir_created_recursively(self, tmp_path: Path) -> None:+ nested = tmp_path / "a" / "b" / "ws"+ TexDownloader(workspace_dir=str(nested))+ assert nested.is_dir()++ def test_existing_workspace_reused(self, tmp_path: Path) -> None:+ ws = tmp_path / "ws"+ ws.mkdir()+ marker = ws / "keep.txt"+ marker.write_text("keep", encoding="utf-8")+ TexDownloader(workspace_dir=str(ws))+ assert marker.read_text(encoding="utf-8") == "keep"++ def test_result_defaults(self) -> None:+ result = TexDownloadResult(success=False)+ assert result.success is False+ assert result.tex_path is None+ assert result.tex_content is None+ assert result.error is None+++# ---------------------------------------------------------------------------+# TexDownloader — ArXiv ID extraction+# ---------------------------------------------------------------------------+++class TestExtractArxivId:+ def test_abs_url(self, downloader: TexDownloader) -> None:+ assert downloader._extract_arxiv_id("https://arxiv.org/abs/1706.03762") == "1706.03762"++ def test_pdf_url_with_version(self, downloader: TexDownloader) -> None:+ assert downloader._extract_arxiv_id("https://arxiv.org/pdf/2401.12345v2") == "2401.12345"++ def test_unrelated_url_returns_none(self, downloader: TexDownloader) -> None:+ assert downloader._extract_arxiv_id("https://example.com/paper") is None++ def test_empty_url_returns_none(self, downloader: TexDownloader) -> None:+ assert downloader._extract_arxiv_id("") is None+++# ---------------------------------------------------------------------------+# TexDownloader — download + extract happy paths (network mocked)+# ---------------------------------------------------------------------------+++class TestDownloadHappyPaths:+ def test_tar_source_lands_in_paper_dir(self, downloader: TexDownloader, tmp_path: Path) -> None:+ payload = _make_tar_bytes({"main.tex": MIN_TEX.encode("utf-8")})++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests, "get", lambda *a, **k: _mock_response(payload)+ )+ result = downloader.download_arxiv_source(+ "https://arxiv.org/abs/1706.03762", arxiv_id="1706.03762"+ )++ assert result.success is True, result.error+ assert result.tex_path is not None+ assert Path(result.tex_path) == tmp_path / "ws" / "paper_1706.03762" / "main.tex"+ assert result.tex_content == MIN_TEX+ # temporary extraction dir is cleaned up, only the permanent paper dir remains+ assert [p.name for p in (tmp_path / "ws").iterdir()] == ["paper_1706.03762"]++ def test_zip_source_picks_paper_tex(self, downloader: TexDownloader, tmp_path: Path) -> None:+ payload = _make_zip_bytes({"paper.tex": MIN_TEX.encode("utf-8")})++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests, "get", lambda *a, **k: _mock_response(payload)+ )+ result = downloader.download_arxiv_source(+ "https://arxiv.org/pdf/2401.12345", arxiv_id="2401.12345"+ )++ assert result.success is True, result.error+ assert Path(result.tex_path).name == "main.tex" # always finalised to main.tex+ assert result.tex_content == MIN_TEX++ def test_plain_bytes_treated_as_single_tex_file(self, downloader: TexDownloader) -> None:+ # Not a tar, not a zip → stored as <arxiv_id>.tex and still found.+ payload = b"some raw tex body without documentclass"++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests, "get", lambda *a, **k: _mock_response(payload)+ )+ result = downloader.download_arxiv_source(+ "https://arxiv.org/abs/2101.00001", arxiv_id="2101.00001"+ )++ assert result.success is True, result.error+ assert result.tex_content == "some raw tex body without documentclass"++ def test_arxiv_id_taken_from_url_when_arg_missing(self, downloader: TexDownloader) -> None:+ payload = _make_tar_bytes({"main.tex": MIN_TEX.encode("utf-8")})++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests, "get", lambda *a, **k: _mock_response(payload)+ )+ result = downloader.download_arxiv_source("https://arxiv.org/abs/2401.12345v2")++ assert result.success is True, result.error+ assert "paper_2401.12345" in result.tex_path+++# ---------------------------------------------------------------------------+# TexDownloader — failure branches+# ---------------------------------------------------------------------------+++class TestDownloadFailureBranches:+ def test_unextractable_id_fails_fast(self, downloader: TexDownloader) -> None:+ result = downloader.download_arxiv_source("https://example.com/no-id")+ assert result.success is False+ assert result.error == "Unable to extract ArXiv ID"++ def test_network_error_maps_to_download_failed(self, downloader: TexDownloader) -> None:+ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests,+ "get",+ lambda *a, **k: (_ for _ in ()).throw(+ requests.exceptions.ConnectionError("conn refused")+ ),+ )+ result = downloader.download_arxiv_source(+ "https://arxiv.org/abs/1706.03762", arxiv_id="1706.03762"+ )++ assert result.success is False+ assert result.error is not None+ assert result.error.startswith("Download failed")+ assert "conn refused" in result.error++ def test_http_error_maps_to_download_failed(self, downloader: TexDownloader) -> None:+ response = MagicMock()+ response.raise_for_status.side_effect = requests.exceptions.HTTPError("404")++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(tex_downloader_module.requests, "get", lambda *a, **k: response)+ result = downloader.download_arxiv_source(+ "https://arxiv.org/abs/1706.03762", arxiv_id="1706.03762"+ )++ assert result.success is False+ assert result.error is not None+ assert result.error.startswith("Download failed")+ assert "404" in result.error++ def test_processing_error_maps_to_processing_failed(self, downloader: TexDownloader) -> None:+ payload = _make_tar_bytes({"main.tex": MIN_TEX.encode("utf-8")})++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests, "get", lambda *a, **k: _mock_response(payload)+ )+ mp.setattr(+ downloader,+ "_find_main_tex",+ lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")),+ )+ result = downloader.download_arxiv_source(+ "https://arxiv.org/abs/1706.03762", arxiv_id="1706.03762"+ )++ assert result.success is False+ assert result.error == "Processing failed: boom"++ def test_archive_without_tex_reports_main_not_found(self, downloader: TexDownloader) -> None:+ payload = _make_tar_bytes({"README.txt": b"no tex here"})++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests, "get", lambda *a, **k: _mock_response(payload)+ )+ result = downloader.download_arxiv_source(+ "https://arxiv.org/abs/1706.03762", arxiv_id="1706.03762"+ )++ assert result.success is False+ assert result.error == "Main tex file not found"++ def test_tar_path_traversal_member_is_skipped(+ self, downloader: TexDownloader, tmp_path: Path+ ) -> None:+ # TarSlip guard: "../evil.tex" must not be written anywhere in the workspace.+ payload = _make_tar_bytes({"../evil.tex": b"malicious"})++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ tex_downloader_module.requests, "get", lambda *a, **k: _mock_response(payload)+ )+ result = downloader.download_arxiv_source(+ "https://arxiv.org/abs/1706.03762", arxiv_id="1706.03762"+ )++ assert result.success is False+ assert result.error == "Main tex file not found"+ assert list(tmp_path.rglob("evil.tex")) == []+++# ---------------------------------------------------------------------------+# TexDownloader — main tex selection priority+# ---------------------------------------------------------------------------+++class TestFindMainTex:+ def test_prefers_main_tex_over_documentclass(+ self, downloader: TexDownloader, tmp_path: Path+ ) -> None:+ (tmp_path / " appendix.tex").write_text("\\documentclass{book}", encoding="utf-8")+ (tmp_path / "main.tex").write_text("main", encoding="utf-8")+ found = downloader._find_main_tex(tmp_path)+ assert found is not None and found.name == "main.tex"++ def test_falls_back_to_documentclass_file(+ self, downloader: TexDownloader, tmp_path: Path+ ) -> None:+ (tmp_path / "chapter.tex").write_text("no marker", encoding="utf-8")+ (tmp_path / "chapter-a.tex").write_text("\\documentclass{article}", encoding="utf-8")+ found = downloader._find_main_tex(tmp_path)+ assert found is not None and found.name == "chapter-a.tex"++ def test_falls_back_to_largest_tex(self, downloader: TexDownloader, tmp_path: Path) -> None:+ (tmp_path / "a.tex").write_text("x", encoding="utf-8")+ (tmp_path / "b.tex").write_text("x" * 500, encoding="utf-8")+ found = downloader._find_main_tex(tmp_path)+ assert found is not None and found.name == "b.tex"++ def test_no_tex_files_returns_none(self, downloader: TexDownloader, tmp_path: Path) -> None:+ (tmp_path / "notes.txt").write_text("nothing", encoding="utf-8")+ assert downloader._find_main_tex(tmp_path) is None++ def test_mixed_case_main_tex_matched(self, downloader: TexDownloader, tmp_path: Path) -> None:+ # Name comparison is case-insensitive for files that match the *.tex glob.+ (tmp_path / "Main.tex").write_text("mixed", encoding="utf-8")+ found = downloader._find_main_tex(tmp_path)+ assert found is not None and found.name == "Main.tex"++ def test_uppercase_extension_file_is_invisible(+ self, downloader: TexDownloader, tmp_path: Path+ ) -> None:+ # Known limitation: discovery globs "*.tex" case-sensitively, so a file+ # named MAIN.TEX is never even considered a candidate.+ (tmp_path / "MAIN.TEX").write_text("upper", encoding="utf-8")+ assert downloader._find_main_tex(tmp_path) is None+++# ---------------------------------------------------------------------------+# read_tex_file convenience function+# ---------------------------------------------------------------------------+++class TestReadTexFile:+ def test_reads_utf8_content(self, tmp_path: Path) -> None:+ tex = tmp_path / "paper.tex"+ tex.write_text("\\section{引言}\n内容", encoding="utf-8")+ assert read_tex_file(str(tex)) == "\\section{引言}\n内容"++ def test_invalid_utf8_bytes_are_ignored_not_raised(self, tmp_path: Path) -> None:+ tex = tmp_path / "broken.tex"+ tex.write_bytes(b"ok \\documentclass \xff\xfe end")+ content = read_tex_file(str(tex))+ assert content.startswith("ok \\documentclass")+++# ---------------------------------------------------------------------------+# TexChunker — construction+# ---------------------------------------------------------------------------+++def _cl100k_chunker(monkeypatch: pytest.MonkeyPatch) -> TexChunker:+ """Chunker forced onto the cl100k_base fallback, independent of local LLM config."""+ import deeptutor.tools.tex_chunker as chunker_module++ def _raise() -> None:+ raise RuntimeError("no llm config in tests")++ monkeypatch.setattr(chunker_module, "resolve_llm_runtime_config", _raise)+ return TexChunker()+++class TestTexChunkerInit:+ def test_no_model_resolves_fallback_encoding(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ assert chunker.encoder.name == "cl100k_base"++ def test_unsupported_model_falls_back_to_cl100k(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ "deeptutor.tools.tex_chunker.resolve_llm_runtime_config",+ lambda: MagicMock(model="definitely-not-a-real-model"),+ )+ chunker = TexChunker(model="definitely-not-a-real-model")+ assert chunker.encoder.name == "cl100k_base"++ def test_supported_model_uses_its_encoding(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ "deeptutor.tools.tex_chunker.resolve_llm_runtime_config",+ lambda: MagicMock(model="gpt-4o"),+ )+ chunker = TexChunker(model="gpt-4o")+ assert chunker.encoder.name in {"o200k_base", "cl100k_base"}+++# ---------------------------------------------------------------------------+# TexChunker — token estimation and text cleaning+# ---------------------------------------------------------------------------+++class TestEstimateTokens:+ def test_empty_string_is_zero_tokens(self, monkeypatch: pytest.MonkeyPatch) -> None:+ assert _cl100k_chunker(monkeypatch).estimate_tokens("") == 0++ def test_estimation_is_positive_and_monotonic(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ short = chunker.estimate_tokens("one two three")+ longer = chunker.estimate_tokens("one two three four five six seven eight")+ assert short > 0+ assert longer > short++ def test_encoder_failure_falls_back_to_char_estimate(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ chunker = _cl100k_chunker(monkeypatch)++ def _boom(text: str) -> list[int]:+ raise RuntimeError("encode failed")++ monkeypatch.setattr(chunker.encoder, "encode", _boom)+ text = "x" * 40+ assert chunker.estimate_tokens(text) == len(text) // 4+++class TestCleanText:+ def test_collapses_repeated_whitespace(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ cleaned = chunker._clean_text("a" + "\n" * 200 + "b")+ assert "\n" * 10 in cleaned+ assert "\n" * 11 not in cleaned++ def test_truncates_overlong_single_line(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ cleaned = chunker._clean_text("x" * 15000)+ assert cleaned.endswith("...[truncated]")+ assert len(cleaned) == 10000 + len("...[truncated]")++ def test_normal_lines_pass_through(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ text = "line one\nline two\nline three"+ assert chunker._clean_text(text) == text+++# ---------------------------------------------------------------------------+# TexChunker — chunking+# ---------------------------------------------------------------------------++SECTION_DOC = (+ "\\documentclass{article}\n\\begin{document}\n"+ + "\n\n".join(+ f"\\section{{Section {i}}}\nParagraph {i} explains topic {i} with words.\n"+ f"Second paragraph of section {i} adds more detail words here."+ for i in range(12)+ )+ + "\n\\end{document}\n"+)+++class TestSplitTexIntoChunks:+ def test_short_content_passes_through_verbatim(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ content = "\\section{Intro}\nShort body."+ assert chunker.split_tex_into_chunks(content, max_tokens=10_000) == [content]++ def test_empty_content_returns_single_empty_chunk(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ result = chunker.split_tex_into_chunks("", max_tokens=100)+ assert result == [""]++ def test_sectioned_document_splits_into_bounded_chunks(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ max_tokens = 120+ chunks = chunker.split_tex_into_chunks(SECTION_DOC, max_tokens=max_tokens, overlap=0)+ assert len(chunks) > 1+ for chunk in chunks:+ assert chunker.estimate_tokens(chunk) <= max_tokens+ joined = "\n".join(chunks)+ for i in range(12):+ assert f"\\section{{Section {i}}}" in joined++ def test_single_oversized_section_splits_by_paragraphs(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ body = "\n\n".join(+ f"Paragraph {i} contains several words of body text for testing." for i in range(40)+ )+ doc = f"\\section{{Huge}}\n{body}"+ max_tokens = 60+ chunks = chunker.split_tex_into_chunks(doc, max_tokens=max_tokens, overlap=0)+ assert len(chunks) > 1+ for chunk in chunks:+ assert chunker.estimate_tokens(chunk) <= max_tokens++ def test_document_without_sections_still_chunks(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ doc = "\n\n".join(+ f"Plain paragraph number {i} with a handful of ordinary words." for i in range(40)+ )+ chunks = chunker.split_tex_into_chunks(doc, max_tokens=80, overlap=0)+ assert len(chunks) > 1+ for chunk in chunks:+ assert chunker.estimate_tokens(chunk) <= 80++ def test_overlap_connects_adjacent_chunks(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ chunks = chunker.split_tex_into_chunks(SECTION_DOC, max_tokens=120, overlap=50)+ assert len(chunks) > 1+ # The second chunk must open with text taken from the tail of the first.+ assert chunks[1][:40] in chunks[0]++ def test_degenerate_single_sentence_cannot_be_split(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ # One huge paragraph without any sentence-ending punctuation cannot be+ # split below max_tokens by the current strategy: the blob is emitted+ # oversized (known limitation), possibly next to a blank trailing chunk.+ chunker = _cl100k_chunker(monkeypatch)+ doc = "a" * 6000+ chunks = chunker.split_tex_into_chunks(doc, max_tokens=50, overlap=0)+ assert chunks+ assert chunker.estimate_tokens(chunks[0]) > 50+ assert "".join(chunks).strip() == doc++ def test_tiny_max_tokens_never_hangs_or_crashes(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ chunks = chunker.split_tex_into_chunks(SECTION_DOC, max_tokens=1, overlap=0)+ assert chunks # produces something instead of failing+++class TestGetOverlapText:+ def test_shorter_chunk_returned_whole(self, monkeypatch: pytest.MonkeyPatch) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ assert chunker._get_overlap_text("tiny", 100) == "tiny"++ def test_tail_overlap_is_suffix_of_previous_chunk(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ chunker = _cl100k_chunker(monkeypatch)+ text = " ".join(f"word{i}" for i in range(200))+ overlap = chunker._get_overlap_text(text, 10)+ assert overlap+ assert overlap in text+ assert text.endswith(overlap)