DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""Concurrency, per-image timeout, and progress for image description.
``_load_image_nodes`` describes each extracted image with the configured
multimodal LLM. With ~92 images per KB this is the slowest indexing step, so it
now fans out (bounded by a Semaphore), enforces a per-image timeout, and
reports progress via ``image_progress_callback``. These tests pin that behavior.
"""
from __future__ import annotations
import asyncio
import json
from pathlib import Path
import time
import pytest
@pytest.mark.asyncio
async def test_opt_in_batches_preserve_node_alignment_and_progress(tmp_path, monkeypatch):
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
calls = []
async def complete(prompt, **kwargs):
calls.append(kwargs)
if "image_filename" in kwargs:
return "last image"
return json.dumps(
{
"captions": [
{"image_id": "IMAGE_1", "caption": "second image"},
{"image_id": "IMAGE_0", "caption": "first image"},
]
}
)
_install_multimodal_clients(monkeypatch, complete_fn=complete)
monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
paths = _make_images(tmp_path, ["a.png", "b.png", "c.png"])
progress = []
docs = await loader_module.LlamaIndexDocumentLoader().load(
[str(p) for p in paths],
image_progress_callback=lambda n, total: progress.append((n, total)),
)
assert len(calls) == 2
assert [d.metadata["file_name"] for d in docs] == ["a.png", "b.png", "c.png"]
assert [d.metadata["image_description"] for d in docs] == [
"first image",
"second image",
"last image",
]
assert progress == [(1, 3), (2, 3), (3, 3)]
@pytest.mark.asyncio
async def test_opt_in_auth_failure_stops_waiting_groups(tmp_path, monkeypatch):
from deeptutor.services.llm.exceptions import LLMAuthenticationError
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
calls = []
async def complete(prompt, **kwargs):
calls.append(kwargs)
raise LLMAuthenticationError()
_install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(1, 2.0))
monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
paths = _make_images(tmp_path, [f"{i}.png" for i in range(6)])
progress = []
docs = await loader_module.LlamaIndexDocumentLoader().load(
[str(p) for p in paths], image_progress_callback=lambda n, total: progress.append(n)
)
assert docs == []
assert len(calls) == 1
assert progress == list(range(1, 7))
def _make_images(tmp_path: Path, names: list[str]) -> list[Path]:
paths: list[Path] = []
for name in names:
p = tmp_path / name
p.write_bytes(b"\x89PNG\r\n")
paths.append(p)
return paths
def _install_multimodal_clients(
monkeypatch: pytest.MonkeyPatch,
*,
complete_fn,
limits: tuple[int, float] = (4, 60.0),
) -> None:
"""Wire fake multimodal embedding + vision clients so images are described."""
from types import SimpleNamespace
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
monkeypatch.setattr(loader_module, "image_description_limits", lambda: limits)
class _EmbeddingClient:
config = SimpleNamespace(binding="siliconflow", model="qwen3-vl")
def supports_multimodal_contents(self) -> bool:
return True
async def embed_contents(self, contents):
return [[0.1, 0.2, 0.3] for _ in contents]
class _VisionClient:
config = SimpleNamespace(binding="openai", model="gpt-4o")
def supports_multimodal_images(self) -> bool:
return True
async def complete(self, prompt, **kwargs):
return await complete_fn(prompt, **kwargs)
monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _EmbeddingClient())
monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _VisionClient())
@pytest.mark.asyncio
async def test_image_description_runs_bounded_concurrently(tmp_path, monkeypatch):
pytest.importorskip("llama_index.core")
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
state = {"inflight": 0, "max": 0}
async def complete(prompt, **kwargs):
state["inflight"] += 1
state["max"] = max(state["max"], state["inflight"])
await asyncio.sleep(0.05) # let multiple calls overlap
state["inflight"] -= 1
return "desc"
_install_multimodal_clients(monkeypatch, complete_fn=complete)
paths = _make_images(tmp_path, [f"img{i}.png" for i in range(10)])
docs = await loader_module.LlamaIndexDocumentLoader().load([str(p) for p in paths])
assert 2 <= state["max"] <= 4 # concurrent, but capped by the semaphore
assert len(docs) == 10
@pytest.mark.asyncio
async def test_image_description_per_image_timeout_skips_and_keeps_order(tmp_path, monkeypatch):
pytest.importorskip("llama_index.core")
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
async def complete(prompt, image_filename=None, **kwargs):
if "slow" in (image_filename or ""):
await asyncio.sleep(1.0) # exceeds the 0.2s timeout -> skipped
return "ok"
_install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(4, 0.2))
names = ["slow0.png", "fast0.png", "slow1.png", "fast1.png", "slow2.png", "fast2.png"]
paths = _make_images(tmp_path, names)
start = time.monotonic()
docs = await loader_module.LlamaIndexDocumentLoader().load([str(p) for p in paths])
elapsed = time.monotonic() - start
# Slow images time out and are skipped; fast ones are kept in original order.
assert [d.metadata["file_name"] for d in docs] == ["fast0.png", "fast1.png", "fast2.png"]
assert elapsed < 2.0 # did not hang
@pytest.mark.asyncio
async def test_image_description_reports_progress(tmp_path, monkeypatch):
pytest.importorskip("llama_index.core")
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
async def complete(prompt, **kwargs):
return "desc"
_install_multimodal_clients(monkeypatch, complete_fn=complete)
callbacks: list[tuple[int, int]] = []
paths = _make_images(tmp_path, [f"img{i}.png" for i in range(5)])
await loader_module.LlamaIndexDocumentLoader().load(
[str(p) for p in paths],
image_progress_callback=lambda c, t: callbacks.append((c, t)),
)
assert len(callbacks) == 5
# `current` is a completion counter, so it climbs 1..N regardless of order.
assert [c for c, _ in callbacks] == [1, 2, 3, 4, 5]
assert all(t == 5 for _, t in callbacks)
assert callbacks[-1] == (5, 5)
@pytest.mark.asyncio
async def test_image_description_skips_failed_and_empty(tmp_path, monkeypatch):
pytest.importorskip("llama_index.core")
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
async def complete(prompt, image_filename=None, **kwargs):
name = image_filename or ""
if "raise" in name:
raise RuntimeError("boom")
if "empty" in name:
return ""
return "ok"
_install_multimodal_clients(monkeypatch, complete_fn=complete)
names = ["ok0.png", "raise0.png", "empty0.png", "ok1.png"]
paths = _make_images(tmp_path, names)
docs = await loader_module.LlamaIndexDocumentLoader().load([str(p) for p in paths])
# Only the two "ok" images survive; order preserved.
assert [d.metadata["file_name"] for d in docs] == ["ok0.png", "ok1.png"]
@pytest.mark.asyncio
async def test_batches_bound_concurrency_and_finish_progress(tmp_path, monkeypatch):
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
inflight = 0
peak = 0
async def complete(prompt, **kwargs):
nonlocal inflight, peak
inflight += 1
peak = max(peak, inflight)
await asyncio.sleep(0.03)
inflight -= 1
return json.dumps(
{
"captions": [
{"image_id": "IMAGE_0", "caption": "first"},
{"image_id": "IMAGE_1", "caption": "second"},
]
}
)
_install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(2, 2.0))
monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
paths = _make_images(tmp_path, [f"{i}.png" for i in range(6)])
progress = []
docs = await loader_module.LlamaIndexDocumentLoader().load(
[str(p) for p in paths],
image_progress_callback=lambda n, total: progress.append((n, total)),
)
assert peak == 2
assert [d.metadata["file_name"] for d in docs] == [p.name for p in paths]
assert progress == [(i, 6) for i in range(1, 7)]
@pytest.mark.asyncio
async def test_batch_deadline_covers_split_fallback(tmp_path, monkeypatch):
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
calls = []
async def complete(prompt, **kwargs):
calls.append(kwargs)
if "image_filename" in kwargs:
await asyncio.sleep(1.0)
return "late"
return "malformed response"
_install_multimodal_clients(monkeypatch, complete_fn=complete, limits=(1, 0.1))
monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
paths = _make_images(tmp_path, ["a.png", "b.png"])
progress = []
start = time.monotonic()
docs = await loader_module.LlamaIndexDocumentLoader().load(
[str(p) for p in paths], image_progress_callback=lambda n, total: progress.append(n)
)
assert docs == []
assert len(calls) == 2 # canceled first single fallback; second is never sent
assert time.monotonic() - start < 0.8
assert progress == [1, 2]
@pytest.mark.asyncio
async def test_cached_batch_ingestion_keeps_source_alignment_and_reports_all_progress(
tmp_path, monkeypatch
):
from types import SimpleNamespace
from deeptutor.services.llm import image_caption_cache
from deeptutor.services.llm.config import LLMConfig
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
calls = []
async def complete(prompt, **kwargs):
calls.append(kwargs)
return json.dumps(
{
"captions": [
{"image_id": "IMAGE_1", "caption": "second"},
{"image_id": "IMAGE_0", "caption": "first"},
]
}
)
_install_multimodal_clients(monkeypatch, complete_fn=complete)
client = loader_module.get_image_description_client()
client.config = LLMConfig(model="vision", api_key="test", base_url="https://example.test/v1")
monkeypatch.setattr(loader_module, "get_image_description_client", lambda: client)
monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
monkeypatch.setattr(
image_caption_cache,
"get_path_service",
lambda: SimpleNamespace(get_parse_cache_root=lambda: tmp_path / "cache"),
)
paths = _make_images(tmp_path, ["a.png", "b.png"])
for _ in range(2):
progress = []
docs = await loader_module.LlamaIndexDocumentLoader().load(
[str(p) for p in paths],
image_progress_callback=lambda n, total: progress.append((n, total)),
)
assert [d.metadata["file_name"] for d in docs] == ["a.png", "b.png"]
assert [d.text for d in docs] == ["[Image] a.png\n\nfirst", "[Image] b.png\n\nsecond"]
assert progress == [(1, 2), (2, 2)]
assert len(calls) == 1
@pytest.mark.asyncio
async def test_a_raising_caption_group_does_not_abort_the_ingest(tmp_path, monkeypatch):
"""Captions are best-effort: every other image failure skips and logs.
The batch group was the one seam that re-raised, so an unexpected error
from the batcher (or from a future client wrapper) aborted the whole
ingest instead of dropping that group's captions.
"""
pytest.importorskip("llama_index.core")
from deeptutor.services.rag.pipelines.llamaindex import document_loader as loader_module
async def never_called(*args, **kwargs):
raise AssertionError("the batcher stub owns this call")
groups: list[int] = []
class _RaisingBatcher:
halted = False
def __init__(self, *args, **kwargs):
pass
async def describe(self, images):
groups.append(len(images))
raise RuntimeError("provider contract broke")
_install_multimodal_clients(monkeypatch, complete_fn=never_called)
monkeypatch.setattr(loader_module, "image_description_batch_size", lambda: 2)
monkeypatch.setattr(loader_module, "ImageCaptionBatcher", _RaisingBatcher)
paths = _make_images(tmp_path, ["a.png", "b.png"])
text = tmp_path / "notes.txt"
text.write_text("text that must survive", encoding="utf-8")
docs = await loader_module.LlamaIndexDocumentLoader().load(
[str(p) for p in paths] + [str(text)]
)
assert groups == [2], "the failing group must actually have run"
assert [d.metadata["file_name"] for d in docs] == ["notes.txt"]