Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

13.2 KB
Raw Blame
"""Concurrency, per-image timeout, and progress for image description.

``_load_image_nodes`` describes each extracted image with the configured
multimodal LLM. With ~92 images per KB this is the slowest indexing step, so it
now fans out (bounded by a Semaphore), enforces a per-image timeout, and
reports progress via ``image_progress_callback``. These tests pin that behavior.
"""

from __future__ import annotations

import asyncio
import json
from pathlib import Path
import time

import pytest


@pytest.mark.asyncio
async def test_opt_in_batches_preserve_node_alignment_and_progress(tmp_path, monkeypatch):
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    calls = []

    async def complete(prompt, **kwargs):
        calls.append(kwargs)
        if "image_filename" in kwargs:
            return "last image"
        return json.dumps(
            {
                "captions": [
                    {"image_id": "IMAGE_1", "caption": "second image"},
                    {"image_id": "IMAGE_0", "caption": "first image"},
                ]
            }
        )

    _install_multimodal_clients(monkeypatch, complete_fn=complete)
    monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
    paths = _make_images(tmp_path, ["a.png", "b.png", "c.png"])
    progress = []
    docs = await loader_module.LlamaIndexDocumentLoader().load(
        [str(p) for p in paths],
        image_progress_callback=lambda n, total: progress.append((n, total)),
    )
    assert len(calls) == 2
    assert [d.metadata["file_name"] for d in docs] == ["a.png", "b.png", "c.png"]
    assert [d.metadata["image_description"] for d in docs] == [
        "first image",
        "second image",
        "last image",
    ]
    assert progress == [(1, 3), (2, 3), (3, 3)]


@pytest.mark.asyncio
async def test_opt_in_auth_failure_stops_waiting_groups(tmp_path, monkeypatch):
    from deeptutor.services.llm.exceptions import LLMAuthenticationError
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    calls = []

    async def complete(prompt, **kwargs):
        calls.append(kwargs)
        raise LLMAuthenticationError()

    _install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(1, 2.0))
    monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
    paths = _make_images(tmp_path, [f"{i}.png" for i in range(6)])
    progress = []
    docs = await loader_module.LlamaIndexDocumentLoader().load(
        [str(p) for p in paths], image_progress_callback=lambda n, total: progress.append(n)
    )
    assert docs == []
    assert len(calls) == 1
    assert progress == list(range(1, 7))


def _make_images(tmp_path: Path, names: list[str]) -> list[Path]:
    paths: list[Path] = []
    for name in names:
        p = tmp_path / name
        p.write_bytes(b"\x89PNG\r\n")
        paths.append(p)
    return paths


def _install_multimodal_clients(
    monkeypatch: pytest.MonkeyPatch,
    *,
    complete_fn,
    limits: tuple[int, float] = (4, 60.0),
) -> None:
    """Wire fake multimodal embedding + vision clients so images are described."""
    from types import SimpleNamespace

    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    monkeypatch.setattr(loader_module, "image_description_limits", lambda: limits)

    class _EmbeddingClient:
        config = SimpleNamespace(binding="siliconflow", model="qwen3-vl")

        def supports_multimodal_contents(self) -> bool:
            return True

        async def embed_contents(self, contents):
            return [[0.1, 0.2, 0.3] for _ in contents]

    class _VisionClient:
        config = SimpleNamespace(binding="openai", model="gpt-4o")

        def supports_multimodal_images(self) -> bool:
            return True

        async def complete(self, prompt, **kwargs):
            return await complete_fn(prompt, **kwargs)

    monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _EmbeddingClient())
    monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _VisionClient())


@pytest.mark.asyncio
async def test_image_description_runs_bounded_concurrently(tmp_path, monkeypatch):
    pytest.importorskip("llama_index.core")
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    state = {"inflight": 0, "max": 0}

    async def complete(prompt, **kwargs):
        state["inflight"] += 1
        state["max"] = max(state["max"], state["inflight"])
        await asyncio.sleep(0.05)  # let multiple calls overlap
        state["inflight"] -= 1
        return "desc"

    _install_multimodal_clients(monkeypatch, complete_fn=complete)

    paths = _make_images(tmp_path, [f"img{i}.png" for i in range(10)])
    docs = await loader_module.LlamaIndexDocumentLoader().load([str(p) for p in paths])

    assert 2 <= state["max"] <= 4  # concurrent, but capped by the semaphore
    assert len(docs) == 10


@pytest.mark.asyncio
async def test_image_description_per_image_timeout_skips_and_keeps_order(tmp_path, monkeypatch):
    pytest.importorskip("llama_index.core")
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    async def complete(prompt, image_filename=None, **kwargs):
        if "slow" in (image_filename or ""):
            await asyncio.sleep(1.0)  # exceeds the 0.2s timeout -> skipped
        return "ok"

    _install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(4, 0.2))

    names = ["slow0.png", "fast0.png", "slow1.png", "fast1.png", "slow2.png", "fast2.png"]
    paths = _make_images(tmp_path, names)

    start = time.monotonic()
    docs = await loader_module.LlamaIndexDocumentLoader().load([str(p) for p in paths])
    elapsed = time.monotonic() - start

    # Slow images time out and are skipped; fast ones are kept in original order.
    assert [d.metadata["file_name"] for d in docs] == ["fast0.png", "fast1.png", "fast2.png"]
    assert elapsed < 2.0  # did not hang


@pytest.mark.asyncio
async def test_image_description_reports_progress(tmp_path, monkeypatch):
    pytest.importorskip("llama_index.core")
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    async def complete(prompt, **kwargs):
        return "desc"

    _install_multimodal_clients(monkeypatch, complete_fn=complete)

    callbacks: list[tuple[int, int]] = []
    paths = _make_images(tmp_path, [f"img{i}.png" for i in range(5)])
    await loader_module.LlamaIndexDocumentLoader().load(
        [str(p) for p in paths],
        image_progress_callback=lambda c, t: callbacks.append((c, t)),
    )

    assert len(callbacks) == 5
    # `current` is a completion counter, so it climbs 1..N regardless of order.
    assert [c for c, _ in callbacks] == [1, 2, 3, 4, 5]
    assert all(t == 5 for _, t in callbacks)
    assert callbacks[-1] == (5, 5)


@pytest.mark.asyncio
async def test_image_description_skips_failed_and_empty(tmp_path, monkeypatch):
    pytest.importorskip("llama_index.core")
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    async def complete(prompt, image_filename=None, **kwargs):
        name = image_filename or ""
        if "raise" in name:
            raise RuntimeError("boom")
        if "empty" in name:
            return ""
        return "ok"

    _install_multimodal_clients(monkeypatch, complete_fn=complete)

    names = ["ok0.png", "raise0.png", "empty0.png", "ok1.png"]
    paths = _make_images(tmp_path, names)
    docs = await loader_module.LlamaIndexDocumentLoader().load([str(p) for p in paths])

    # Only the two "ok" images survive; order preserved.
    assert [d.metadata["file_name"] for d in docs] == ["ok0.png", "ok1.png"]


@pytest.mark.asyncio
async def test_batches_bound_concurrency_and_finish_progress(tmp_path, monkeypatch):
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    inflight = 0
    peak = 0

    async def complete(prompt, **kwargs):
        nonlocal inflight, peak
        inflight += 1
        peak = max(peak, inflight)
        await asyncio.sleep(0.03)
        inflight -= 1
        return json.dumps(
            {
                "captions": [
                    {"image_id": "IMAGE_0", "caption": "first"},
                    {"image_id": "IMAGE_1", "caption": "second"},
                ]
            }
        )

    _install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(2, 2.0))
    monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
    paths = _make_images(tmp_path, [f"{i}.png" for i in range(6)])
    progress = []
    docs = await loader_module.LlamaIndexDocumentLoader().load(
        [str(p) for p in paths],
        image_progress_callback=lambda n, total: progress.append((n, total)),
    )
    assert peak == 2
    assert [d.metadata["file_name"] for d in docs] == [p.name for p in paths]
    assert progress == [(i, 6) for i in range(1, 7)]


@pytest.mark.asyncio
async def test_batch_deadline_covers_split_fallback(tmp_path, monkeypatch):
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    calls = []

    async def complete(prompt, **kwargs):
        calls.append(kwargs)
        if "image_filename" in kwargs:
            await asyncio.sleep(1.0)
            return "late"
        return "malformed response"

    _install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(1, 0.1))
    monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
    paths = _make_images(tmp_path, ["a.png", "b.png"])
    progress = []
    start = time.monotonic()
    docs = await loader_module.LlamaIndexDocumentLoader().load(
        [str(p) for p in paths], image_progress_callback=lambda n, total: progress.append(n)
    )
    assert docs == []
    assert len(calls) == 2  # canceled first single fallback; second is never sent
    assert time.monotonic() - start < 0.8
    assert progress == [1, 2]


@pytest.mark.asyncio
async def test_cached_batch_ingestion_keeps_source_alignment_and_reports_all_progress(
    tmp_path, monkeypatch
):
    from types import SimpleNamespace

    from deeptutor.services.llm import image_caption_cache
    from deeptutor.services.llm.config import LLMConfig
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    calls = []

    async def complete(prompt, **kwargs):
        calls.append(kwargs)
        return json.dumps(
            {
                "captions": [
                    {"image_id": "IMAGE_1", "caption": "second"},
                    {"image_id": "IMAGE_0", "caption": "first"},
                ]
            }
        )

    _install_multimodal_clients(monkeypatch, complete_fn=complete)
    client = loader_module.get_image_description_client()
    client.config = LLMConfig(model="vision", api_key="test", base_url="https://example.test/v1")
    monkeypatch.setattr(loader_module, "get_image_description_client", lambda: client)
    monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
    monkeypatch.setattr(
        image_caption_cache,
        "get_path_service",
        lambda: SimpleNamespace(get_parse_cache_root=lambda: tmp_path / "cache"),
    )
    paths = _make_images(tmp_path, ["a.png", "b.png"])
    for _ in range(2):
        progress = []
        docs = await loader_module.LlamaIndexDocumentLoader().load(
            [str(p) for p in paths],
            image_progress_callback=lambda n, total: progress.append((n, total)),
        )
        assert [d.metadata["file_name"] for d in docs] == ["a.png", "b.png"]
        assert [d.text for d in docs] == ["[Image] a.png\n\nfirst", "[Image] b.png\n\nsecond"]
        assert progress == [(1, 2), (2, 2)]
    assert len(calls) == 1


@pytest.mark.asyncio
async def test_a_raising_caption_group_does_not_abort_the_ingest(tmp_path, monkeypatch):
    """Captions are best-effort: every other image failure skips and logs.

    The batch group was the one seam that re-raised, so an unexpected error
    from the batcher (or from a future client wrapper) aborted the whole
    ingest instead of dropping that group's captions.
    """
    pytest.importorskip("llama_index.core")
    from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module

    async def never_called(*args, **kwargs):
        raise AssertionError("the batcher stub owns this call")

    groups: list[int] = []

    class _RaisingBatcher:
        halted = False

        def __init__(self, *args, **kwargs):
            pass

        async def describe(self, images):
            groups.append(len(images))
            raise RuntimeError("provider contract broke")

    _install_multimodal_clients(monkeypatch, complete_fn=never_called)
    monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
    monkeypatch.setattr(loader_module, "ImageCaptionBatcher", _RaisingBatcher)
    paths = _make_images(tmp_path, ["a.png", "b.png"])
    text = tmp_path / "notes.txt"
    text.write_text("text that must survive", encoding="utf-8")

    docs = await loader_module.LlamaIndexDocumentLoader().load(
        [str(p) for p in paths] + [str(text)]
    )

    assert groups == [2], "the failing group must actually have run"
    assert [d.metadata["file_name"] for d in docs] == ["notes.txt"]