Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

30.3 KB
Raw Blame
"""Source visual provenance, KB scope, and model pixel delivery."""

from __future__ import annotations

import asyncio
import base64
import json
from pathlib import Path
from types import SimpleNamespace
import zipfile

from PIL import Image
import pytest

from deeptutor.services.parsing.types import ParsedDocument
from deeptutor.services.rag.visual_assets import (
    VisualAssetStore,
    collect_visual_assets,
)


def _fixture(tmp_path: Path, *, suffix: str = ".pdf", engine: str = "mineru"):
    kb_dir = tmp_path / "kb"
    raw = kb_dir / "raw"
    raw.mkdir(parents=True)
    source = raw / f"lesson{suffix}"
    source.write_bytes(b"source document with a figure")
    assets = tmp_path / "parse-cache" / "images"
    assets.mkdir(parents=True)
    image = assets / "figure.png"
    Image.new("RGB", (3, 2), color=(17, 99, 211)).save(image)
    parsed = ParsedDocument(
        markdown="The learning curve is shown below.\n\n![Learning curve](images/figure.png)",
        blocks=(
            [
                {
                    "type": "image",
                    "img_path": str(image),
                    "page_idx": 2,
                    "bbox": [10, 20, 100, 200],
                    "image_caption": ["Learning curve"],
                }
            ]
            if engine == "mineru"
            else None
        ),
        asset_dir=assets,
        source_hash="source-hash",
        parser_signature=f"{engine}-signature",
        engine=engine,
    )
    return kb_dir, source, image, parsed


@pytest.mark.parametrize(
    ("suffix", "engine"),
    [(".pdf", "mineru"), (".epub", "pymupdf4llm")],
)
def test_verified_source_visual_contract_and_lifecycle(tmp_path: Path, suffix: str, engine: str):
    kb_dir, source, image, parsed = _fixture(tmp_path, suffix=suffix, engine=engine)
    candidates = collect_visual_assets(parsed, source, kb_dir)
    assert len(candidates) == 1
    record = candidates[0].record
    assert record["asset_id"] == collect_visual_assets(parsed, source, kb_dir)[0].record["asset_id"]
    assert record["source_path"] == source.name
    assert record["source_document_id"]
    assert record["parser_engine"] == engine
    assert record["caption"] == "Learning curve"
    if engine == "mineru":
        assert record["page_index"] == 2
        assert record["page_number"] == 3
        assert record["bbox"] == [10, 20, 100, 200]
        assert record["source_locator"] == "blocks.json#/0"
    else:
        assert record["page_index"] is None
        assert record["source_locator"].startswith("markdown:")

    store = VisualAssetStore(kb_dir)
    store.publish(candidates, replace=True)
    assert store.read(record["asset_id"]) == (record, image.read_bytes())
    assert store.read("../manifest.json") is None
    persisted = store.root / (record["asset_id"] + ".png")
    persisted.write_bytes(b"\x89PNG\r\n\x1a\ncorrupted")
    assert store.read(record["asset_id"]) is None
    store.publish(candidates, replace=True)
    newer = ParsedDocument(
        markdown=parsed.markdown,
        blocks=parsed.blocks,
        asset_dir=parsed.asset_dir,
        parser_signature="new-parser-signature",
        engine=parsed.engine,
    )
    newer_candidate = collect_visual_assets(newer, source, kb_dir)[0]
    store.publish([newer_candidate])
    assert store.read(record["asset_id"]) is not None  # older index version remains valid
    assert store.read(newer_candidate.record["asset_id"]) is not None
    moved = kb_dir / "raw" / "folder" / source.name
    moved.parent.mkdir()
    source.rename(moved)
    store.move_source(source.name, f"folder/{source.name}")
    assert store.records()[record["asset_id"]]["source_path"] == f"folder/{source.name}"
    store.remove_source(f"folder/{source.name}")
    assert store.read(record["asset_id"]) is None
    assert store.read(newer_candidate.record["asset_id"]) is None
    assert not persisted.exists()


def test_large_existing_manifest_remains_readable_and_accepts_more_figures(tmp_path: Path):
    """#1802: a corpus beyond the old 16 MiB limit must retain every reference."""
    import deeptutor.services.rag.visual_assets as assets_module

    kb_dir, source, image, parsed = _fixture(tmp_path)
    original = collect_visual_assets(parsed, source, kb_dir)[0]
    store = VisualAssetStore(kb_dir)
    store.publish([original])
    # Model a legacy manifest from a figure-heavy KB without 15,000 image copies.
    records = {
        f"{index:064x}": {**original.record, "asset_id": f"{index:064x}", "context": "x" * 1000}
        for index in range(11000)
    }
    records[original.record["asset_id"]] = original.record
    store.manifest_path.write_text(json.dumps({"version": 1, "assets": records}), encoding="utf-8")
    assert store.manifest_path.stat().st_size > 16 * 1024 * 1024
    assert store.read(original.record["asset_id"]) is not None
    second = assets_module.VisualAssetCandidate(
        path=image,
        record={**original.record, "asset_id": "a" * 64},
    )
    store.publish([second])
    assert len(store.records()) == len(records) + 1
    assert store.read(original.record["asset_id"]) is not None
    assert store.read(second.record["asset_id"]) is not None


@pytest.mark.parametrize("operation", ["publish", "remove_source", "move_source"])
def test_unreadable_manifest_is_reported_and_never_overwritten(tmp_path: Path, caplog, operation):
    kb_dir, source, _image, parsed = _fixture(tmp_path)
    candidate = collect_visual_assets(parsed, source, kb_dir)[0]
    store = VisualAssetStore(kb_dir)
    store.publish([candidate])
    store.manifest_path.write_bytes(b"{broken manifest")
    before = store.manifest_path.read_bytes()
    assert store.read(candidate.record["asset_id"]) is None
    assert "Cannot read source figures for KB 'kb'" in caplog.text
    with pytest.raises(OSError, match="Cannot read visual asset manifest"):
        if operation == "publish":
            store.publish([candidate])
        elif operation == "remove_source":
            store.remove_source(source.name)
        else:
            store.move_source(source.name, "moved.pdf")
    assert store.manifest_path.read_bytes() == before


def test_changed_image_does_not_replace_existing_manifest(tmp_path: Path):
    kb_dir, source, image, parsed = _fixture(tmp_path)
    candidate = collect_visual_assets(parsed, source, kb_dir)[0]
    store = VisualAssetStore(kb_dir)
    store.publish([candidate])
    before = store.manifest_path.read_bytes()
    image.write_bytes(b"not an image anymore")
    with pytest.raises(OSError, match="Source image changed"):
        store.publish([candidate])
    assert store.manifest_path.read_bytes() == before
    assert store.read(candidate.record["asset_id"]) is not None


def test_size_count_and_rebuild_cleanup(tmp_path: Path, monkeypatch):
    import deeptutor.services.rag.visual_assets as assets_module

    kb_dir, source, image, parsed = _fixture(tmp_path)
    store = VisualAssetStore(kb_dir)
    first = collect_visual_assets(parsed, source, kb_dir)[0]
    store.publish([first])
    source.unlink()
    store.publish([], prune_missing=True)
    assert store.read(first.record["asset_id"]) is None

    source.write_bytes(b"source document with a figure")
    monkeypatch.setattr(assets_module, "MAX_IMAGE_BYTES", 10)
    assert collect_visual_assets(parsed, source, kb_dir) == []
    monkeypatch.setattr(assets_module, "MAX_IMAGE_BYTES", 5 * 1024 * 1024)
    for index in range(2):
        Image.new("RGB", (3, 2), color=(index * 60, 0, 0)).save(image.parent / f"extra-{index}.png")
    assert len(collect_visual_assets(parsed, source, kb_dir)) == 3


def test_hundred_figure_textbook_keeps_late_figures_searchable_and_delivers_pixels(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
):
    """#1610: document retention must not be truncated by the model image budget.

    A local deterministic embedding tests the storage/retrieval path; it does
    not measure a provider's semantic or cross-language retrieval quality.
    """
    from dataclasses import replace
    import re
    import shutil

    from llama_index.core import VectorStoreIndex
    from llama_index.core.base.embeddings.base import BaseEmbedding

    from deeptutor.multi_user import knowledge_access
    import deeptutor.services.parsing as parsing
    from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader
    from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline
    from deeptutor.tools import rag_tool
    from deeptutor.tools.builtin import RAGTool

    kb_dir, source, image, parsed = _fixture(tmp_path)
    # Page-grounded indexing now opens the physical PDF as well as parser output.
    import pymupdf

    with pymupdf.open() as pdf:
        for _ in range(100):
            pdf.new_page()
        pdf.save(source)
    image.unlink()
    parsed = replace(
        parsed,
        markdown="Synthetic textbook with one hundred figures.",
        blocks=[],
    )
    for number in range(1, 101):
        asset = image.parent / f"figure-{number:03}.png"
        Image.new("RGB", (8, 6), color=(number, 100, 200)).save(asset)
        parsed.blocks.append(
            {
                "type": "image",
                "img_path": str(asset),
                "page_idx": number - 1,
                "bbox": [10, 20, 80, 60],
                "image_caption": [f"Figure {number}: learning curve"],
            }
        )

    class Parser:
        def parse(self, _source, **_kwargs):
            return parsed

    class FigureEmbedding(BaseEmbedding):
        def _get_text_embedding(self, text):
            match = re.search(r"Figure (\d+)", text)
            vector = [0.0] * 101
            vector[int(match.group(1)) if match else 0] = 1.0
            return vector

        def _get_query_embedding(self, query):
            return self._get_text_embedding(query)

        async def _aget_query_embedding(self, query):
            return self._get_query_embedding(query)

    monkeypatch.setattr(parsing, "get_parse_service", lambda: Parser())
    candidates = []
    documents = asyncio.run(
        LlamaIndexDocumentLoader().load([str(source)], kb_dir=kb_dir, visual_candidates=candidates)
    )
    assert len(candidates) == 100
    visuals = [doc for doc in documents if doc.metadata.get("content_type") == "source_visual"]
    assert len(visuals) == 100
    VisualAssetStore(kb_dir).publish(candidates)
    index = VectorStoreIndex.from_documents(documents, embed_model=FigureEmbedding())
    # The page text and its figure caption share the deterministic embedding.
    retriever = index.as_retriever(similarity_top_k=2)
    for number in (1, 50, 100):
        nodes = retriever.retrieve(f"Figure {number}")
        retrieved = LlamaIndexPipeline._nodes_to_result(None, f"Figure {number}", nodes)
        record = next(item for item in retrieved["sources"] if item.get("visual_asset_id"))
        assert record["page"] == number
        assert record["visual_asset_id"] == candidates[number - 1].record["asset_id"]

    late = candidates[-1]
    expected_pixels = late.path.read_bytes()
    # The KB-owned copy survives cache removal and a fresh store instance.
    shutil.rmtree(image.parent.parent)
    assert VisualAssetStore(kb_dir).read(late.record["asset_id"])[1] == expected_pixels
    monkeypatch.setattr(
        knowledge_access,
        "resolve_for_rag",
        lambda _name: SimpleNamespace(name="kb", base_dir=tmp_path),
    )

    async def search(**_kwargs):
        return {**retrieved, "answer": "Figure 100 is on page 100."}

    monkeypatch.setattr(rag_tool, "rag_search", search)
    result = asyncio.run(
        RAGTool().execute(query="Figure 100", kb_name="kb", _vision_supported=True)
    )
    parts = result.model_message["content"]
    assert sum(part["type"] == "image_url" for part in parts) == 1
    assert base64.b64decode(parts[1]["image_url"]["url"].split(",", 1)[1]) == expected_pixels
    assert any(
        late.record["asset_id"] in item.get("visual_asset_url", "") for item in result.sources
    )


@pytest.mark.parametrize("suffix", [".pdf", ".epub"])
def test_real_pymupdf_parser_extracts_pdf_and_epub_pixels(tmp_path: Path, suffix: str):
    fitz = pytest.importorskip("fitz")
    pytest.importorskip("pymupdf4llm")
    from deeptutor.services.parsing.cache import load_ir
    from deeptutor.services.parsing.engines.pymupdf4llm.config import PyMuPDF4LLMConfig
    from deeptutor.services.parsing.engines.pymupdf4llm.engine import PyMuPDF4LLMParser

    kb_dir = tmp_path / "kb"
    raw = kb_dir / "raw"
    raw.mkdir(parents=True)
    source = raw / f"lesson{suffix}"
    image = tmp_path / "figure.png"
    Image.new("RGB", (100, 80), color="blue").save(image)
    if suffix == ".pdf":
        pdf = fitz.open()
        page = pdf.new_page()
        page.insert_text((50, 50), "Learning curve")
        page.insert_image(fitz.Rect(50, 80, 250, 240), filename=str(image))
        pdf.save(source)
        pdf.close()
    else:
        with zipfile.ZipFile(source, "w") as epub:
            epub.writestr("mimetype", "application/epub+zip", compress_type=zipfile.ZIP_STORED)
            epub.writestr(
                "META-INF/container.xml",
                '<?xml version="1.0"?><container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>',
            )
            epub.writestr(
                "OEBPS/content.opf",
                '<?xml version="1.0"?><package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="id"><metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>Lesson</dc:title><dc:language>en</dc:language><dc:identifier id="id">lesson</dc:identifier></metadata><manifest><item id="page" href="page.xhtml" media-type="application/xhtml+xml"/><item id="fig" href="figure.png" media-type="image/png"/></manifest><spine><itemref idref="page"/></spine></package>',
            )
            epub.writestr(
                "OEBPS/page.xhtml",
                '<html xmlns="http://www.w3.org/1999/xhtml"><head><title>Lesson</title></head><body><p>Learning curve</p><img src="figure.png" alt="Learning curve"/></body></html>',
            )
            epub.write(image, "OEBPS/figure.png")

    workdir = tmp_path / "parse"
    workdir.mkdir()
    PyMuPDF4LLMParser().parse(source, workdir, config=PyMuPDF4LLMConfig())
    markdown, blocks, asset_dir = load_ir(workdir)
    parsed = ParsedDocument(
        markdown=markdown,
        blocks=blocks,
        asset_dir=asset_dir,
        parser_signature="pymupdf-test",
        engine="pymupdf4llm",
    )
    candidates = collect_visual_assets(parsed, source, kb_dir)
    assert len(candidates) == 1
    assert candidates[0].record["mime_type"] == "image/png"
    assert candidates[0].record["source_locator"].startswith("markdown:")
    assert candidates[0].record["context"].startswith("Learning curve")


def test_visual_document_is_searchable_with_text_embeddings(tmp_path: Path, monkeypatch):
    pytest.importorskip("llama_index.core")
    import deeptutor.services.parsing as parsing
    from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader

    kb_dir, source, _image, parsed = _fixture(tmp_path, suffix=".epub", engine="pymupdf4llm")

    class Parser:
        def parse(self, _source, **_kwargs):
            return parsed

    monkeypatch.setattr(parsing, "get_parse_service", lambda: Parser())
    candidates = []
    documents = asyncio.run(
        LlamaIndexDocumentLoader().load([str(source)], kb_dir=kb_dir, visual_candidates=candidates)
    )
    visual = [doc for doc in documents if doc.metadata.get("content_type") == "source_visual"]
    assert len(visual) == 1
    assert len(candidates) == 1
    assert visual[0].metadata["visual_asset_id"] == candidates[0].record["asset_id"]
    assert "Learning curve" in visual[0].text
    assert "The learning curve" in visual[0].text
    from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline

    retrieved = SimpleNamespace(
        node=SimpleNamespace(
            text=visual[0].text,
            metadata=visual[0].metadata,
            node_id="visual-node",
        ),
        score=0.9,
    )
    result = LlamaIndexPipeline._nodes_to_result(None, "learning curve", [retrieved])
    assert result["sources"][0]["visual_asset_id"] == candidates[0].record["asset_id"]
    assert result["sources"][0]["chunk_id"] == "visual-node"


def test_visual_preview_enforces_kb_scope_and_verified_bytes(tmp_path: Path, monkeypatch):
    import importlib

    from fastapi import HTTPException

    import deeptutor.services.config as config_module

    monkeypatch.setattr(config_module, "load_config_with_main", lambda *_args: {})
    knowledge = importlib.import_module("deeptutor.api.routers.knowledge")

    kb_dir, source, image, parsed = _fixture(tmp_path)
    candidate = collect_visual_assets(parsed, source, kb_dir)[0]
    VisualAssetStore(kb_dir).publish([candidate])
    monkeypatch.setattr(knowledge, "_resolve_kb_raw_dir", lambda _kb_name: kb_dir / "raw")
    response = asyncio.run(knowledge.serve_kb_visual_asset("kb", candidate.record["asset_id"]))
    assert response.body == image.read_bytes()
    assert response.headers["cache-control"] == "private, no-store"
    assert response.headers["x-content-type-options"] == "nosniff"

    def denied(_kb_name):
        raise HTTPException(status_code=403, detail="Knowledge base is not assigned to you")

    monkeypatch.setattr(knowledge, "_resolve_kb_raw_dir", denied)
    with pytest.raises(HTTPException) as exc:
        asyncio.run(knowledge.serve_kb_visual_asset("kb", candidate.record["asset_id"]))
    assert exc.value.status_code == 403


def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, monkeypatch):
    from deeptutor.multi_user import knowledge_access
    from deeptutor.services.workspace import context as workspace_context
    from deeptutor.tools import rag_tool
    from deeptutor.tools.builtin import RAGTool

    kb_dir, source, image, parsed = _fixture(tmp_path)
    record = collect_visual_assets(parsed, source, kb_dir)[0]
    VisualAssetStore(kb_dir).publish([record], replace=True)
    monkeypatch.setattr(
        knowledge_access,
        "resolve_for_rag",
        lambda kb_name: SimpleNamespace(name="kb", base_dir=tmp_path) if kb_name == "kb" else None,
    )
    monkeypatch.setattr(workspace_context, "current_workspace_id", lambda: "study")

    async def search(**_kwargs):
        return {
            "answer": "A learning curve is cited.",
            "sources": [
                {
                    "title": "lesson.pdf",
                    "visual_asset_id": record.record["asset_id"],
                    "caption": "Learning curve",
                    "source": "/etc/passwd",
                }
            ],
        }

    monkeypatch.setattr(rag_tool, "rag_search", search)
    result = asyncio.run(
        RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)
    )
    assert result.model_message is not None
    parts = result.model_message["content"]
    assert parts[0]["type"] == "text"
    assert record.record["asset_id"] in parts[0]["text"]
    assert "Learning curve" not in parts[0]["text"]
    assert parts[1]["type"] == "image_url"
    data_uri = parts[1]["image_url"]["url"]
    assert data_uri.startswith("data:image/png;base64,")
    assert base64.b64decode(data_uri.split(",", 1)[1]) == image.read_bytes()
    assert result.sources[0]["visual_asset_url"].endswith(
        f"{record.record['asset_id']}?dt_workspace=study"
    )
    assert "base64" not in str(result.metadata)

    text_only = asyncio.run(
        RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=False)
    )
    assert text_only.model_message is None
    assert "cannot inspect source image pixels" in text_only.content

    monkeypatch.setattr(knowledge_access, "resolve_for_rag", lambda _kb_name: None)
    inaccessible = asyncio.run(
        RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)
    )
    assert inaccessible.model_message is None

    monkeypatch.setattr(
        knowledge_access,
        "resolve_for_rag",
        lambda _kb_name: SimpleNamespace(name="different-kb", base_dir=tmp_path),
    )
    cross_kb = asyncio.run(
        RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)
    )
    assert cross_kb.model_message is None


def test_rag_model_payload_is_bounded_to_two_images(tmp_path: Path, monkeypatch):
    from deeptutor.multi_user import knowledge_access
    from deeptutor.tools import rag_tool
    from deeptutor.tools.builtin import RAGTool

    kb_dir, source, image, parsed = _fixture(tmp_path)
    for index in range(2):
        Image.new("RGB", (3, 2), color=(0, index * 60, 10)).save(
            image.parent / f"extra-{index}.png"
        )
    candidates = collect_visual_assets(parsed, source, kb_dir)
    assert len(candidates) == 3
    VisualAssetStore(kb_dir).publish(candidates)
    monkeypatch.setattr(
        knowledge_access,
        "resolve_for_rag",
        lambda _kb_name: SimpleNamespace(name="kb", base_dir=tmp_path),
    )

    async def search(**_kwargs):
        return {
            "answer": "Three figures found.",
            "sources": [{"visual_asset_id": item.record["asset_id"]} for item in candidates],
        }

    monkeypatch.setattr(rag_tool, "rag_search", search)
    result = asyncio.run(RAGTool().execute(query="figures", kb_name="kb", _vision_supported=True))
    assert sum(part["type"] == "image_url" for part in result.model_message["content"]) == 2


def test_visual_message_reaches_next_model_request_without_stream_leak(tmp_path: Path, monkeypatch):
    from deeptutor.runtime.agentic import loop as agent_loop
    from deeptutor.runtime.agentic.labeled_step import LabeledStepResult
    from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome, _collect_outcome

    kb_dir, source, image, parsed = _fixture(tmp_path)
    candidate = collect_visual_assets(parsed, source, kb_dir)[0]
    VisualAssetStore(kb_dir).publish([candidate], replace=True)
    record, pixels = VisualAssetStore(kb_dir).read(candidate.record["asset_id"])
    private_message = {
        "role": "user",
        "content": [
            {"type": "text", "text": f"Inspect {record['asset_id']}"},
            {
                "type": "image_url",
                "image_url": {"url": "data:image/png;base64," + base64.b64encode(pixels).decode()},
            },
        ],
    }

    class Stream:
        def __init__(self):
            self.events = []

        async def tool_result(self, **kwargs):
            self.events.append(kwargs)

    stream = Stream()
    outcome = asyncio.run(
        _collect_outcome(
            prepared=[("tool-1", "rag", {})],
            results=[
                {
                    "result_text": "Source visual found",
                    "sources": [],
                    "metadata": {},
                    "model_message": private_message,
                }
            ],
            per_tool_trace_meta=[{}],
            stream=stream,
            source="chat",
            stage="exploring",
        )
    )
    assert outcome.model_messages == [{**private_message, "_after_tool_call_id": "tool-1"}]
    assert "base64" not in str(stream.events)
    capped = asyncio.run(
        _collect_outcome(
            prepared=[(f"tool-{index}", "rag", {}) for index in range(3)],
            results=[
                {
                    "result_text": "Source visual found",
                    "sources": [],
                    "metadata": {},
                    "model_message": private_message,
                }
                for _ in range(3)
            ],
            per_tool_trace_meta=[{} for _ in range(3)],
            stream=stream,
            source="chat",
            stage="exploring",
        )
    )
    assert len(capped.model_messages) == 2

    captured = []

    async def fake_step(**kwargs):
        captured.append(list(kwargs["messages"]))
        if len(captured) == 1:
            return LabeledStepResult(
                label="CALL_TOOLS",
                text="",
                tool_calls=[{"id": "tool-1", "name": "rag", "arguments": "{}"}],
            )
        return LabeledStepResult(label="FINISH", text="Done")

    monkeypatch.setattr(agent_loop, "run_labeled_step", fake_step)

    class Host:
        async def guard_context_window(self, _messages):
            pass

        def build_iteration_trace_meta(self, _iteration):
            return {}, {}

        async def dispatch_tools(self, **_kwargs):
            return outcome

    protocol = agent_loop.LabelProtocol(
        allowed=("CALL_TOOLS", "FINISH"),
        terminal=frozenset({"FINISH"}),
        intermediate=frozenset(),
        final=frozenset(),
        tool_label="CALL_TOOLS",
    )
    loop_result = asyncio.run(
        agent_loop.run_agentic_loop(
            initial_messages=[{"role": "user", "content": "What does the figure show?"}],
            protocol=protocol,
            client=object(),
            model="gpt-4o",
            completion_kwargs={},
            binding="openai",
            tool_schemas=[],
            stream=stream,
            source="chat",
            stage="exploring",
            max_iterations=2,
            host=Host(),
        )
    )
    assert loop_result.completed
    assert "base64" not in str(loop_result.messages)
    assert all(message.get("role") != "user" for message in loop_result.messages[1:])
    next_request = captured[1]
    assert next_request[-2]["role"] == "tool"
    assert next_request[-1] == private_message
    assert (
        base64.b64decode(next_request[-1]["content"][1]["image_url"]["url"].split(",", 1)[1])
        == image.read_bytes()
    )
    from deeptutor.services.llm.provider_core.anthropic_provider import AnthropicProvider
    from deeptutor.services.llm.provider_core.openai_responses.converters import convert_messages

    _system, responses_input = convert_messages(next_request)
    response_image = responses_input[-1]["content"][1]
    assert response_image["type"] == "input_image"
    assert base64.b64decode(response_image["image_url"].split(",", 1)[1]) == image.read_bytes()
    anthropic_image = AnthropicProvider._convert_image_block(next_request[-1]["content"][1])
    assert anthropic_image["type"] == "image"
    assert base64.b64decode(anthropic_image["source"]["data"]) == image.read_bytes()
    _system, anthropic_messages = AnthropicProvider.__new__(AnthropicProvider)._convert_messages(
        next_request
    )
    assert anthropic_messages[-1]["role"] == "user"
    assert anthropic_messages[-1]["content"][0]["type"] == "tool_result"
    assert anthropic_messages[-1]["content"][-1]["type"] == "image"


def test_visual_messages_follow_complete_tool_reply_batch():
    from deeptutor.runtime.agentic.messages import with_transient_model_messages

    messages = [
        {"role": "assistant", "tool_calls": [{"id": "rag"}, {"id": "other"}]},
        {"role": "tool", "tool_call_id": "rag", "content": "Figure found"},
        {"role": "tool", "tool_call_id": "other", "content": "Other result"},
    ]
    transient = [
        {
            "role": "user",
            "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}],
            "_after_tool_call_id": "rag",
        }
    ]
    request = with_transient_model_messages(messages, transient)
    assert [item["role"] for item in request] == ["assistant", "tool", "tool", "user"]
    assert request[-1]["content"] == transient[0]["content"]
    assert messages[-1]["role"] == "tool"  # canonical history was not changed


def test_visual_images_are_bounded_across_tool_rounds(monkeypatch):
    from deeptutor.agents.loop.pipeline import IMAGE_TOKEN_GUARD_RESERVE, AgenticLoopPipeline
    from deeptutor.runtime.agentic import loop as agent_loop
    from deeptutor.runtime.agentic.labeled_step import LabeledStepResult
    from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome

    requests = []
    guard_estimates = []

    async def fake_step(**kwargs):
        requests.append(kwargs["messages"])
        if len(requests) <= 2:
            tool_id = f"tool-{len(requests)}"
            return LabeledStepResult(
                label="CALL_TOOLS",
                text="",
                tool_calls=[{"id": tool_id, "name": "rag", "arguments": "{}"}],
            )
        return LabeledStepResult(label="FINISH", text="Done")

    monkeypatch.setattr(agent_loop, "run_labeled_step", fake_step)

    class Host:
        async def guard_context_window(self, messages):
            guard_estimates.append(AgenticLoopPipeline._estimate_messages_tokens(messages))

        def build_iteration_trace_meta(self, _iteration):
            return {}, {}

        async def dispatch_tools(self, *, iteration, **_kwargs):
            tool_id = f"tool-{iteration + 1}"
            return DispatchOutcome(
                tool_messages=[
                    {"role": "tool", "tool_call_id": tool_id, "content": "Figure found"}
                ],
                model_messages=[
                    {
                        "role": "user",
                        "_after_tool_call_id": tool_id,
                        "content": [
                            {
                                "type": "image_url",
                                "image_url": {"url": f"data:image/png;base64,{iteration}{image}"},
                            }
                            for image in range(2)
                        ],
                    }
                ],
            )

    protocol = agent_loop.LabelProtocol(
        allowed=("CALL_TOOLS", "FINISH"),
        terminal=frozenset({"FINISH"}),
        intermediate=frozenset(),
        final=frozenset(),
        tool_label="CALL_TOOLS",
    )
    result = asyncio.run(
        agent_loop.run_agentic_loop(
            initial_messages=[{"role": "user", "content": "Explain the figures"}],
            protocol=protocol,
            client=object(),
            model="vision-model",
            completion_kwargs={},
            binding="openai",
            tool_schemas=[],
            stream=object(),
            source="chat",
            stage="exploring",
            max_iterations=3,
            host=Host(),
        )
    )
    assert result.completed
    images = [
        part["image_url"]["url"]
        for message in requests[2]
        for part in (message.get("content") or [])
        if isinstance(part, dict) and part.get("type") == "image_url"
    ]
    assert images == ["data:image/png;base64,10", "data:image/png;base64,11"]
    assert guard_estimates[2] >= 2 * IMAGE_TOKEN_GUARD_RESERVE
    assert "base64" not in str(result.messages)