Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

47.6 KB
Raw Blame
"""Four-role calls, task freezing, legacy policies, and write ownership contracts."""

from __future__ import annotations

import asyncio
from copy import deepcopy
from dataclasses import replace
import json
from pathlib import Path
from types import SimpleNamespace

from pydantic import ValidationError
import pytest

from deeptutor.multi_user.context import reset_current_user, set_current_user
from deeptutor.multi_user.models import CurrentUser, UserScope
from deeptutor.services.config.lightrag_roles import LightRagRoleModels
from deeptutor.services.embedding.config import EmbeddingConfig
from deeptutor.services.llm.config import LLMConfig
from deeptutor.services.model_selection.llm import LLMSelection
from deeptutor.services.rag.embedding_signature import embedding_meta_fields
from deeptutor.services.rag.pipelines.lightrag import engine, roles, storage
from deeptutor.services.rag.pipelines.lightrag import indexing_policy as policy
from deeptutor.services.rag.pipelines.lightrag.pipeline import BatchOutcome, LightRagPipeline
from deeptutor.services.rag.pipelines.lightrag.write_lock import write_ownership


def choice(number: int, effort: str | None = None) -> dict:
    result = {"profile_id": f"p{number}", "model_id": f"m{number}"}
    if effort is not None:
        result["reasoning_effort"] = effort
    return result


@pytest.fixture
def role_environment(tmp_path, monkeypatch):
    user = CurrentUser(
        id="role-user",
        username="tester",
        role="user",
        scope=UserScope(kind="user", user_id="role-user", root=tmp_path / "owner"),
    )
    token = set_current_user(user)
    models = LightRagRoleModels.model_validate(
        {
            "base": choice(0),
            "extract": {
                "mode": "inherit",
                "reasoning_effort": "none",
                "max_async": 2,
                "timeout": 101,
            },
            "keyword": {
                "mode": "model",
                "selection": choice(1),
                "reasoning_effort": "low",
                "max_async": 3,
                "timeout": 102,
            },
            "query": {
                "mode": "model",
                "selection": choice(2),
                "reasoning_effort": "high",
                "max_async": 4,
                "timeout": 103,
            },
            "vlm": {
                "mode": "model",
                "selection": choice(3),
                "reasoning_effort": "none",
                "max_async": 5,
                "timeout": 104,
            },
        }
    )
    state = {
        "embedding": EmbeddingConfig(
            model="embed-one", api_key="fake", dim=3, base_url="https://embed.test/v1"
        ),
        "settings": {"role_models": models.model_dump()},
        "allowed": {f"p{i}" for i in range(4)},
        "configs": {
            f"p{i}": LLMConfig(
                model=f"model-{i}",
                binding="custom",
                provider_name="custom",
                api_key="private-key",
                base_url=f"https://name:password@provider-{i}.test/v1?key=private",
                reasoning_effort="medium",
            )
            for i in range(4)
        },
        "vision": {f"model-{i}": i == 3 for i in range(4)},
    }

    def allowed(value):
        if value["profile_id"] not in state["allowed"]:
            raise PermissionError("Model access was revoked.")
        return value

    def resolve(value):
        selection = LLMSelection.from_payload(value)
        config = state["configs"][selection.profile_id]
        return config.model_copy(
            update={"reasoning_effort": selection.reasoning_effort or config.reasoning_effort}
        )

    def options():
        return {
            "active": choice(0),
            "options": [
                {
                    **choice(i),
                    "model": f"model-{i}",
                    "provider": "custom",
                    "model_name": f"Model {i}",
                    "profile_name": f"Provider {i}",
                    "is_active_default": i == 0,
                    "supported_reasoning_efforts": ["none", "low", "medium", "high"],
                }
                for i in range(4)
                if f"p{i}" in state["allowed"]
            ],
        }

    monkeypatch.setattr(
        "deeptutor.services.config.load_lightrag_settings", lambda: deepcopy(state["settings"])
    )
    monkeypatch.setattr(roles, "allowed_llm_options", options)
    monkeypatch.setattr(roles, "apply_allowed_llm_selection", allowed)
    monkeypatch.setattr(roles, "resolve_llm_config_for_selection", resolve)
    monkeypatch.setattr(
        roles, "supports_vision", lambda _binding, model: state["vision"].get(model, False)
    )
    monkeypatch.setattr(policy, "apply_allowed_llm_selection", allowed)
    monkeypatch.setattr(policy, "resolve_llm_config_for_selection", resolve)
    monkeypatch.setattr(
        policy, "supports_vision", lambda _binding, model: state["vision"].get(model, False)
    )
    monkeypatch.setattr(policy, "_active_catalog_selection", lambda: choice(0))
    from deeptutor.services.embedding.config import scoped_embedding_config

    monkeypatch.setattr(
        "deeptutor.services.embedding.get_embedding_config",
        lambda: scoped_embedding_config() or state["embedding"],
    )
    # The knowledge routes resolve a *selected* embedding through the module
    # binding, not the package-level zero-arg alias above, and the selection
    # itself comes from the real catalog. Patching only the alias left the route
    # reading whatever embedding the developer has configured and scoping it for
    # everything downstream, so this fixture's default never reached it.
    monkeypatch.setattr(
        "deeptutor.services.embedding.config.get_embedding_config",
        lambda selection=None, **_kwargs: state["embedding"],
    )
    yield state
    reset_current_user(token)


def test_role_contract_distinguishes_inherit_explicit_disabled_and_none():
    models = LightRagRoleModels.model_validate(
        {"base": choice(0, "high"), "extract": {"reasoning_effort": "none"}}
    )
    assert models.selection_for("query").reasoning_effort == "high"
    assert models.selection_for("extract").reasoning_effort == "none"
    assert models.selection_for("vlm") is None
    for invalid in (
        {"base": choice(0), "query": {"mode": "disabled"}},
        {"base": choice(0), "vlm": {"mode": "disabled", "selection": choice(3)}},
        {"base": choice(0), "query": {"mode": "model"}},
        {"base": choice(0), "query": {"mode": "inherit", "selection": choice(1)}},
        {"base": choice(0), "query": {"max_async": 0}},
        {"base": choice(0), "query": {"timeout": True}},
        {"base": {**choice(0), "api_key": "private"}},
    ):
        with pytest.raises(ValidationError):
            LightRagRoleModels.model_validate(invalid)


def test_new_settings_do_not_fall_back_to_chat(role_environment, monkeypatch):
    monkeypatch.setattr(
        "deeptutor.services.rag.pipelines.lightrag.config.resolve_lightrag_query_llm_config",
        lambda: pytest.fail("new settings must not resolve chat fallback"),
    )
    first = roles.resolve_query_roles()
    assert first["keyword"].config.model == "model-1"
    assert first["query"].config.model == "model-2"
    role_environment["settings"]["role_models"]["query"] = {"mode": "inherit"}
    role_environment["settings"]["role_models"]["base"] = choice(3)
    second = roles.resolve_query_roles()
    assert second["query"].config.model == "model-3"
    assert first["query"].config.model == "model-2"
    assert (first["query"].max_async, first["query"].timeout) == (4, 103)


def test_all_four_native_roles_call_their_frozen_models(role_environment, monkeypatch, tmp_path):
    pytest.importorskip("lightrag")
    calls = []

    def build(config, *, allow_multimodal):
        async def complete(prompt, **kwargs):
            calls.append((config.model, config.reasoning_effort, allow_multimodal, kwargs))
            return config.model

        return complete

    monkeypatch.setattr("deeptutor.services.llm.client.build_model_func_for_config", build)
    monkeypatch.setattr(engine, "_require_exact_version", lambda: None)
    monkeypatch.setattr(engine, "_register_parser", lambda: None)
    monkeypatch.setattr(engine, "_controlled_class", lambda: lambda **kwargs: kwargs)
    monkeypatch.setattr(engine, "build_embedding_func", lambda **kwargs: "independent embedding")
    monkeypatch.setattr(engine, "constructor_kwargs_from_settings", dict)
    monkeypatch.setattr(engine, "indexing_kwargs_from_settings", dict)
    snapshot = policy.freeze_roles()
    queries = roles.resolve_query_roles()
    indexed = engine.build_rag(tmp_path, indexing_snapshot=snapshot, enable_vlm=True)
    queried = engine.build_rag(tmp_path, query_roles=queries)

    async def exercise():
        await indexed["role_llm_configs"]["extract"].func("extract")
        await indexed["role_llm_configs"]["vlm"].func("vision", image_inputs=[{"base64": "image"}])
        await queried["role_llm_configs"]["keyword"].func("keyword")
        await queried["role_llm_configs"]["query"].func("query")

    asyncio.run(exercise())
    assert [(model, effort, vision) for model, effort, vision, _ in calls] == [
        ("model-0", "none", False),
        ("model-3", "none", True),
        ("model-1", "low", False),
        ("model-2", "high", False),
    ]
    assert all(kwargs["allow_image_fallback"] is False for *_, kwargs in calls)
    all_roles = {**indexed["role_llm_configs"], **queried["role_llm_configs"]}
    assert {name: (value.max_async, value.timeout) for name, value in all_roles.items()} == {
        "extract": (2, 101),
        "vlm": (5, 104),
        "keyword": (3, 102),
        "query": (4, 103),
    }
    assert len({value.metadata["model"] for value in all_roles.values()}) == 4


def test_index_policy_freezes_defaults_and_limits_but_refreshes_credentials(role_environment):
    snapshot = policy.freeze_roles()
    persisted = snapshot.persisted_policy()
    role_environment["settings"]["role_models"]["extract"]["max_async"] = 12
    role_environment["settings"]["role_models"]["extract"]["timeout"] = 900
    role_environment["settings"]["role_models"]["base"] = choice(2)
    role_environment["configs"]["p0"].api_key = "rotated-secret"
    fresh = policy.revalidate_snapshot(snapshot)
    assert fresh.extract.config.api_key == "rotated-secret"
    assert fresh.extract.config.model == "model-0"
    assert fresh.limits["extract"] == {"max_async": 2, "timeout": 101}
    assert fresh.persisted_policy() == persisted
    assert policy.freeze_roles().extract.config.model == "model-2"
    serialized = json.dumps(persisted)
    for forbidden in ("private", "password", "api_key", "role-user"):
        assert forbidden not in serialized
    assert "endpoint" not in json.dumps(policy.public_policy(persisted))


@pytest.mark.parametrize(
    "change", ["endpoint", "model", "wire_api", "context_window", "access", "vision"]
)
def test_invalid_pinned_identity_blocks_before_writes(role_environment, change):
    snapshot = policy.freeze_roles()
    if change == "endpoint":
        role_environment["configs"]["p0"].effective_url = "https://other.test/v1"
    elif change == "model":
        role_environment["configs"]["p0"].model = "different"
    elif change == "wire_api":
        role_environment["configs"]["p0"].api_format = "openai_responses"
    elif change == "context_window":
        role_environment["configs"]["p0"].context_window = 100000
    elif change == "access":
        role_environment["allowed"].remove("p3")
    else:
        role_environment["vision"]["model-3"] = False
    with pytest.raises(policy.IndexingPolicyError):
        policy.revalidate_snapshot(snapshot)


def test_unsupported_reasoning_and_nonvision_inheritance_fail(role_environment):
    models = deepcopy(role_environment["settings"]["role_models"])
    models["query"]["reasoning_effort"] = "max"
    with pytest.raises(ValueError, match="reasoning"):
        roles.validate_models(LightRagRoleModels.model_validate(models))
    models["query"]["reasoning_effort"] = "high"
    models["vlm"] = {"mode": "inherit"}
    with pytest.raises(ValueError, match="image inputs"):
        roles.validate_models(LightRagRoleModels.model_validate(models))


def test_explicit_catalog_vision_capability_is_used_for_custom_model(role_environment, monkeypatch):
    role_environment["vision"]["model-0"] = False
    monkeypatch.setattr(
        roles,
        "allowed_llm_options",
        lambda: {
            "active": choice(0),
            "options": [
                {
                    **choice(0),
                    "model": "model-0",
                    "provider": "custom",
                    "model_name": "Custom vision model",
                    "profile_name": "Custom provider",
                    "is_active_default": True,
                    "declared_vision": True,
                    "supported_reasoning_efforts": ["none", "low", "medium", "high"],
                }
            ],
        },
    )

    config = roles.resolve_selection(LLMSelection(**choice(0)), vision=True)

    assert config.model == "model-0"


def test_legacy_single_policy_keeps_its_original_algorithm_and_vision(role_environment):
    legacy = policy.freeze_snapshot(choice(3, "none")).persisted_policy()
    adapted = policy.snapshot_from_persisted(legacy)
    assert adapted.vision_available
    assert adapted.extract.fingerprint == legacy["fingerprint"]
    assert adapted.vlm.fingerprint == legacy["fingerprint"]
    assert adapted.persisted_policy() == legacy
    assert "schema_version" not in adapted.persisted_policy()
    del legacy["vision_available"]
    with pytest.raises(policy.IndexingModelChangedError, match="historical vision"):
        policy.snapshot_from_persisted(legacy)


@pytest.mark.parametrize(
    "bad",
    [
        {},
        {"policy": "legacy_unpinned"},
        {"policy": "pinned", "schema_version": 99},
        {"policy": "pinned", "schema_version": 2, "extract": {}, "vlm": {"mode": "enabled"}},
        {
            "policy": "pinned",
            "schema_version": 2,
            "extract": {},
            "vlm": {"mode": "disabled", "snapshot": {}},
        },
    ],
)
def test_malformed_policies_never_acquire_current_defaults(role_environment, bad):
    with pytest.raises(policy.IndexingPolicyError):
        policy.snapshot_from_persisted(bad)


def test_disabled_vlm_requires_rebuild_for_explicit_analysis(role_environment):
    disabled = policy.freeze_roles({"extract": choice(0), "vlm": {"mode": "disabled"}})
    assert not disabled.vision_available
    assert policy.with_image_analysis(disabled, False).vlm is None
    with pytest.raises(policy.IndexingModelChangedError, match="full re-index"):
        policy.with_image_analysis(disabled, True)
    enabled = policy.freeze_roles(
        {"extract": choice(0), "vlm": {"mode": "enabled", "selection": choice(3)}}
    )
    assert policy.with_image_analysis(enabled, True).vision_available
    assert (
        policy.with_image_analysis(enabled, False).persisted_policy() == enabled.persisted_policy()
    )


def _published(path: Path, value: dict):
    path.mkdir(parents=True, exist_ok=True)
    (path / "kv_store_doc_status.json").write_text('{"doc": {"status": "processed"}}')
    (path / "meta.json").write_text(
        json.dumps(
            {
                "provider": "lightrag",
                "signature": "lightrag",
                "lightrag_adapter_schema": storage.ADAPTER_SCHEMA,
                "parser_bridge_schema": 1,
                "state": "published",
                "indexing_policy": value,
                **embedding_meta_fields(),
            }
        )
    )


def test_stale_queued_target_is_rejected_before_graph_write(
    role_environment, tmp_path, monkeypatch
):
    kb = tmp_path / "kb"
    first = policy.freeze_roles()
    _published(kb / "version-1", first.persisted_policy())
    accepted = policy.bind_target(first, kb)
    _published(kb / "version-2", first.persisted_policy())
    pipeline = LightRagPipeline(str(tmp_path))
    monkeypatch.setattr(
        pipeline, "_run_indexing", lambda *_args: pytest.fail("stale job wrote graph")
    )
    with pytest.raises(policy.IndexingPolicyError, match="target index"):
        asyncio.run(pipeline.add_documents("kb", ["doc.md"], accepted_indexing_snapshot=accepted))
    assert storage.latest_published_root(kb) == kb / "version-2"


def test_two_writers_cannot_own_one_knowledge_base(tmp_path):
    kb = tmp_path / "kb"
    with write_ownership(kb):
        with pytest.raises(policy.IndexingPolicyError, match="Another indexing"):
            with write_ownership(kb):
                pytest.fail("both writers acquired ownership")
    with write_ownership(kb):
        pass


def test_failed_terminal_persistence_does_not_publish_candidate(
    role_environment, tmp_path, monkeypatch
):
    snapshot = policy.freeze_roles()
    kb = tmp_path / "kb"
    _published(kb / "version-1", snapshot.persisted_policy())
    pipeline = LightRagPipeline(str(tmp_path))
    monkeypatch.setattr(pipeline, "_ensure_available", lambda: None)
    monkeypatch.setattr(storage, "has_output", lambda _root: True)

    async def indexed(root, *_args, **_kwargs):
        (root / "kv_store_doc_status.json").write_text('{"doc": {"status": "processed"}}')
        return BatchOutcome(
            1, accepted=1, processed=("doc.md",), indexing_policy=snapshot.persisted_policy()
        )

    def fail_terminal(_root):
        raise OSError("terminal persistence failed")

    monkeypatch.setattr(pipeline, "_run_indexing", indexed)
    with pytest.raises(OSError, match="terminal persistence"):
        asyncio.run(
            pipeline.initialize(
                "kb", ["doc.md"], indexing_snapshot=snapshot, before_publish=fail_terminal
            )
        )
    assert storage.latest_published_root(kb) == kb / "version-1"
    assert not (kb / "version-2" / "meta.json").exists()


def test_prepared_terminal_is_not_announced_or_read_as_completed(tmp_path, monkeypatch):
    from deeptutor.knowledge.progress_tracker import ProgressStage, ProgressTracker

    entry = {"status": "processing"}

    class Manager:
        def __init__(self, **_kwargs):
            pass

        def update_kb_status(self, *, status, progress, **_kwargs):
            entry.update(status=status, progress=progress)

        def get_kb_entry(self, _name):
            return entry

    monkeypatch.setattr("deeptutor.knowledge.manager.KnowledgeBaseManager", Manager)
    monkeypatch.setattr(
        "deeptutor.knowledge.progress_events.emit_task_progress", lambda *_args: None
    )
    tracker = ProgressTracker("kb", tmp_path)
    tracker.task_id = "rebuild-1"
    seen = []
    tracker.set_callback(lambda progress: seen.append(progress["stage"]))
    tracker.update(
        ProgressStage.COMPLETED,
        "Rebuild complete",
        current=1,
        total=1,
        indexed_count=1,
        index_changed=True,
        index_action="reindex",
        publication_version="version-2",
    )
    tracker.verify_terminal(
        current=1, total=1, indexed_count=1, index_action="reindex", publication_version="version-2"
    )
    assert entry["status"] == "processing"
    assert seen == ["processing_documents"]
    assert tracker.get_progress()["stage"] == "processing_documents"
    # A process interrupted here has a prepared record, not a published index.
    _published(tmp_path / "kb" / "version-2", {"policy": "pinned"})
    assert tracker.get_progress()["stage"] == "completed"


@pytest.mark.parametrize("lost_write", ["progress_file", "kb_status"])
def test_lost_terminal_write_is_detected_before_publication(tmp_path, monkeypatch, lost_write):
    from deeptutor.knowledge.progress_tracker import ProgressStage, ProgressTracker

    entry = {"status": "ready"}

    class Manager:
        def __init__(self, **_kwargs):
            pass

        def update_kb_status(self, *, status, progress, **_kwargs):
            if lost_write == "kb_status":
                raise OSError("status storage unavailable")
            entry.update(status=status, progress=progress)

        def get_kb_entry(self, _name):
            return entry

    monkeypatch.setattr("deeptutor.knowledge.manager.KnowledgeBaseManager", Manager)
    if lost_write == "progress_file":

        def fail(*_args):
            raise OSError("progress storage unavailable")

        monkeypatch.setattr("deeptutor.knowledge.progress_tracker.atomic_write_json", fail)
    tracker = ProgressTracker("kb", tmp_path)
    tracker.task_id = "new-task"
    tracker.update(
        ProgressStage.COMPLETED,
        current=1,
        total=1,
        indexed_count=1,
        index_changed=True,
        index_action="reindex",
        publication_version="version-2",
    )
    with pytest.raises(RuntimeError, match="terminal state was not persisted"):
        tracker.verify_terminal(
            current=1,
            total=1,
            indexed_count=1,
            index_action="reindex",
            publication_version="version-2",
        )


def test_concurrent_first_writers_do_not_publish_different_policies(
    role_environment, tmp_path, monkeypatch
):
    snapshot = policy.bind_target(policy.freeze_roles(), tmp_path / "kb")
    pipeline = LightRagPipeline(str(tmp_path))
    monkeypatch.setattr(pipeline, "_ensure_available", lambda: None)
    monkeypatch.setattr(pipeline, "_clear_pending_policy", lambda _name: None)
    monkeypatch.setattr(
        storage,
        "write_meta",
        lambda root, *, indexing_policy, embedding_config: _published(root, indexing_policy),
    )

    async def exercise():
        entered = asyncio.Event()
        release = asyncio.Event()

        async def indexed(root, *_args, **_kwargs):
            (root / "kv_store_doc_status.json").write_text('{"doc": {"status": "processed"}}')
            entered.set()
            await release.wait()
            return BatchOutcome(
                1, accepted=1, processed=("doc.md",), indexing_policy=snapshot.persisted_policy()
            )

        monkeypatch.setattr(pipeline, "_run_indexing", indexed)
        first = asyncio.create_task(
            pipeline.add_documents("kb", ["doc.md"], accepted_indexing_snapshot=snapshot)
        )
        await entered.wait()
        try:
            with pytest.raises(policy.IndexingPolicyError, match="Another indexing"):
                await pipeline.add_documents(
                    "kb", ["other.md"], accepted_indexing_snapshot=snapshot
                )
        finally:
            release.set()
        assert await first
        with pytest.raises(policy.IndexingPolicyError, match="target index"):
            await pipeline.add_documents("kb", ["other.md"], accepted_indexing_snapshot=snapshot)

    asyncio.run(exercise())
    published = storage.latest_published_root(tmp_path / "kb")
    assert published.name == "version-1"
    assert (
        storage.read_published_policy(published)["extract"]["fingerprint"]
        == snapshot.extract.fingerprint
    )


def test_pending_policy_change_invalidates_a_queued_first_write(role_environment, tmp_path):
    from deeptutor.knowledge.manager import KnowledgeBaseManager

    kb = tmp_path / "kb"
    kb.mkdir()
    manager = KnowledgeBaseManager(base_dir=str(tmp_path))
    original = policy.freeze_roles(policy=policy.POLICY_PENDING)
    manager.config["knowledge_bases"]["kb"] = {
        "path": "kb",
        "rag_provider": "lightrag",
        "pending_indexing_policy": original.persisted_policy(),
    }
    manager._save_config()
    accepted = policy.bind_target(original, kb)
    replacement = policy.freeze_roles(
        {"extract": choice(1), "vlm": {"mode": "disabled"}}, policy=policy.POLICY_PENDING
    )
    manager.config["knowledge_bases"]["kb"]["pending_indexing_policy"] = (
        replacement.persisted_policy()
    )
    manager._save_config()
    with pytest.raises(policy.IndexingPolicyError, match="pending policy changed"):
        policy.validate_target(accepted, kb)


def test_queued_rebuild_rejects_same_version_append_but_append_remains_valid(
    role_environment, tmp_path
):
    kb = tmp_path / "kb"
    snapshot = policy.freeze_roles()
    root = kb / "version-1"
    _published(root, snapshot.persisted_policy())
    rebuild = policy.bind_target(snapshot, kb, protect_contents=True)
    append = policy.bind_target(snapshot, kb)
    (root / "kv_store_doc_status.json").write_text(
        '{"doc": {"status": "processed"}, "later": {"status": "processed"}}'
    )
    with pytest.raises(policy.IndexingPolicyError, match="target index"):
        policy.validate_target(rebuild, kb)
    policy.validate_target(append, kb)


def test_disabled_vision_keeps_table_and_equation_processing(tmp_path, monkeypatch):
    pipeline = LightRagPipeline(str(tmp_path))
    from dataclasses import dataclass

    from deeptutor.services.rag.pipelines.lightrag import ingress

    @dataclass
    class Staged:
        process_options: str = "ite"

    monkeypatch.setattr(
        "deeptutor.services.parsing.get_parse_service",
        lambda: SimpleNamespace(parse=lambda path: object()),
    )
    monkeypatch.setattr(ingress, "freeze_document", lambda *args: Staged())
    disabled, failures = pipeline._stage_documents(tmp_path, ["doc.md"], vision_available=False)
    enabled, _ = pipeline._stage_documents(tmp_path, ["doc.md"], vision_available=True)
    assert not failures
    assert disabled[0].process_options == "te"
    assert enabled[0].process_options == "ite"


@pytest.mark.asyncio
async def test_github_sync_freezes_policy_before_download(role_environment, tmp_path, monkeypatch):
    from deeptutor.services.github_source import sync

    kb = tmp_path / "kb"
    pinned = policy.freeze_roles()
    _published(kb / "version-1", pinned.persisted_policy())
    monkeypatch.setattr(
        "deeptutor.services.rag.provider_binding.resolve_bound_provider", lambda *args: "lightrag"
    )
    recorded = []
    monkeypatch.setattr(
        "deeptutor.knowledge.manager.KnowledgeBaseManager",
        lambda **kwargs: SimpleNamespace(
            update_github_source_state=lambda **data: recorded.append(data)
        ),
    )
    accepted = []

    async def add_documents(**kwargs):
        accepted.append(kwargs["accepted_indexing_snapshot"])
        return 1

    monkeypatch.setattr("deeptutor.knowledge.add_documents.add_documents", add_documents)

    class Client:
        async def get_latest_commit_sha(self, *args):
            role_environment["settings"]["role_models"]["base"] = choice(2)
            return "new-sha"

        async def get_tree(self, *args, **kwargs):
            return [SimpleNamespace(path="doc.md", type="blob")]

        async def download_file(self, *args):
            return b"test source"

    result = await sync.sync_source(
        "kb", {"id": "src", "repo": "owner/repo"}, base_dir=str(tmp_path), client=Client()
    )
    assert result.ok
    assert accepted[0].extract.fingerprint == pinned.extract.fingerprint
    assert accepted[0].vlm.fingerprint == pinned.vlm.fingerprint
    assert accepted[0].target_bound
    assert recorded[0]["last_synced_sha"] == "new-sha"


@pytest.mark.parametrize("outcome_kind", ["partial_append", "metadata_refresh_failure"])
def test_native_workspace_append_invalidates_queued_rebuild_without_meta_change(
    role_environment, tmp_path, monkeypatch, outcome_kind
):
    from deeptutor.services.rag.pipelines.lightrag.pipeline import LightRagBatchError

    kb = tmp_path / "kb"
    root = kb / "version-1"
    snapshot = policy.freeze_roles()
    _published(root, snapshot.persisted_policy())
    workspace = root / engine.workspace_for(root)
    workspace.mkdir()
    status = workspace / "kv_store_doc_status.json"
    (root / status.name).rename(status)
    before_meta = (root / "meta.json").read_bytes()
    rebuild = policy.bind_target(snapshot, kb, protect_contents=True)
    append = policy.bind_target(snapshot, kb)
    pipeline = LightRagPipeline(str(tmp_path))
    monkeypatch.setattr(pipeline, "_ensure_available", lambda: None)

    async def append_native(*args, **_kwargs):
        status.write_text('{"doc":{"status":"processed"},"later":{"status":"processed"}}')
        result = BatchOutcome(
            requested=2 if outcome_kind == "partial_append" else 1,
            accepted=2 if outcome_kind == "partial_append" else 1,
            processed=("later.md",),
            failed={"bad.md": "provider rejected"} if outcome_kind == "partial_append" else {},
            indexing_policy=snapshot.persisted_policy(),
        )
        if outcome_kind == "partial_append":
            raise LightRagBatchError(result)
        return result

    def fail_metadata(*args, **kwargs):
        raise OSError("metadata refresh failed")

    monkeypatch.setattr(pipeline, "_run_indexing", append_native)
    monkeypatch.setattr(storage, "write_meta", fail_metadata)
    if outcome_kind == "partial_append":
        with pytest.raises(LightRagBatchError):
            asyncio.run(
                pipeline.add_documents(
                    "kb", ["later.md", "bad.md"], accepted_indexing_snapshot=append
                )
            )
    else:
        assert asyncio.run(
            pipeline.add_documents("kb", ["later.md"], accepted_indexing_snapshot=append)
        )
    assert (root / "meta.json").read_bytes() == before_meta
    assert storage.latest_published_root(kb) == root
    with pytest.raises(policy.IndexingPolicyError, match="target index"):
        policy.validate_target(rebuild, kb)
    policy.validate_target(append, kb)


@pytest.mark.parametrize("model", ["qwen3-32b", "deepseek-r1"])
def test_custom_binary_reasoning_choices_match_actual_request(model):
    from deeptutor.services.llm.provider_core.openai_compat_provider import OpenAICompatProvider
    from deeptutor.services.model_selection.reasoning import supported_reasoning_efforts
    from deeptutor.services.provider_registry import find_by_name

    assert supported_reasoning_efforts("custom", model) == ["minimal", "high"]
    assert supported_reasoning_efforts(
        "custom", model, metadata={"codex_supported_reasoning_levels": ["none", "low", "high"]}
    ) == ["high"]
    provider = OpenAICompatProvider(
        api_key="synthetic", default_model=model, spec=find_by_name("custom"), configure_env=False
    )
    disabled = provider._build_kwargs(
        [{"role": "user", "content": "test"}], None, model, 100, 0, "minimal", None
    )
    enabled = provider._build_kwargs(
        [{"role": "user", "content": "test"}], None, model, 100, 0, "high", None
    )
    if "qwen" in model:
        assert disabled["extra_body"]["enable_thinking"] is False
        assert enabled["extra_body"]["enable_thinking"] is True
    else:
        assert disabled["extra_body"]["thinking"]["type"] == "disabled"
        assert enabled["extra_body"]["thinking"]["type"] == "enabled"


def test_binary_provider_none_is_rejected_before_snapshot(role_environment):
    role_environment["configs"]["p0"].model = "qwen3-32b"
    # Use the real capability mapper with the same catalog model name.
    options = roles.allowed_llm_options

    def catalog():
        value = options()
        value["options"][0]["model"] = "qwen3-32b"
        return value

    from unittest.mock import patch

    with patch.object(roles, "allowed_llm_options", catalog):
        with pytest.raises(policy.IndexingPolicyError, match="reasoning effort"):
            policy.freeze_roles()


def test_fresh_defaults_do_not_enter_legacy_model_or_vision_fallback(role_environment, tmp_path):
    from deeptutor.services.config.runtime_settings import RuntimeSettingsService

    service = RuntimeSettingsService(tmp_path / "settings", process_env={})
    fresh = service.load_lightrag()
    assert fresh["version"] == 2
    role_environment["settings"] = fresh
    with pytest.raises(policy.IndexingPolicyError, match="base model"):
        policy.freeze_roles()
    with pytest.raises(ValueError, match="base model"):
        roles.resolve_query_roles()
    # An old-form explicit model input on a fresh configuration still defaults VLM off.
    created = policy.freeze_roles(choice(3))
    assert created.vlm is None
    legacy_path = service.path_for("lightrag")
    legacy_path.write_text('{"llm_profile_id":"p3","llm_model_id":"m3"}')
    legacy = service.load_lightrag()
    assert legacy["version"] == 1
    role_environment["settings"] = legacy
    # The compatibility path retains historical vision semantics.
    assert policy.freeze_roles(choice(3)).vlm is not None


@pytest.mark.parametrize("version", [3, "2", True])
def test_invalid_settings_version_never_becomes_legacy_fallback(tmp_path, version):
    from deeptutor.services.config.runtime_settings import RuntimeSettingsService

    service = RuntimeSettingsService(tmp_path, process_env={})
    path = service.path_for("lightrag")
    path.write_text(json.dumps({"version": version}))
    before = path.read_bytes()
    with pytest.raises(ValueError, match="settings version"):
        service.load_lightrag()
    assert path.read_bytes() == before


def test_provider_auto_ignores_saved_effort_and_survives_catalog_changes(role_environment):
    """Automatic indexing keeps provider defaults across catalog edits and reloads."""
    snapshot = policy.freeze_roles({"extract": choice(0), "vlm": {"mode": "disabled"}})
    assert snapshot.extract.config.reasoning_effort == ""
    persisted = snapshot.persisted_policy()
    assert persisted["extract"]["descriptor"]["reasoning_effort"] == ""
    role_environment["configs"]["p0"].reasoning_effort = "high"
    restored = policy.snapshot_from_persisted(persisted)
    assert restored.extract.config.reasoning_effort == ""
    assert restored.extract.fingerprint == snapshot.extract.fingerprint
    explicit = roles.resolve_selection(LLMSelection(**choice(0, "none")))
    assert explicit.reasoning_effort == "none"


def test_restoring_pre_auto_policy_preserves_legacy_resolution(role_environment):
    """Old pinned policies do not silently acquire the new automatic semantics."""
    role_environment["configs"]["p0"].reasoning_effort = None
    original = policy._freeze_role(choice(0), provider_default=False)
    persisted = policy.IndexingPolicySnapshot(original, None).persisted_policy()
    assert persisted["extract"]["descriptor"]["reasoning_effort"] is None
    restored = policy.snapshot_from_persisted(persisted)
    assert restored.extract.config.reasoning_effort is None
    assert restored.extract.fingerprint == original.fingerprint
    role_environment["configs"]["p0"].reasoning_effort = "high"
    with pytest.raises(policy.IndexingModelChangedError):
        policy.snapshot_from_persisted(persisted)


@pytest.mark.parametrize(
    "binding,model",
    [
        ("gemini", "gemini-3.5-flash-lite"),
        ("custom", "qwen3-32b"),
        ("custom", "deepseek-v4-pro"),
        ("openai", "gpt-5"),
    ],
)
def test_provider_auto_omits_actual_reasoning_request_fields(role_environment, binding, model):
    """Automatic roles produce no explicit reasoning or thinking request controls."""
    from deeptutor.services.llm.provider_core.openai_compat_provider import OpenAICompatProvider
    from deeptutor.services.provider_registry import find_by_name

    role_environment["configs"]["p0"].binding = binding
    role_environment["configs"]["p0"].model = model
    config = roles.resolve_selection(LLMSelection(**choice(0)))
    provider = OpenAICompatProvider(
        api_key="synthetic", default_model=model, spec=find_by_name(binding), configure_env=False
    )
    body = provider._build_kwargs(
        [{"role": "user", "content": "test"}],
        None,
        model,
        100,
        0,
        config.reasoning_effort,
        None,
    )
    assert "reasoning_effort" not in body
    assert "reasoning" not in body
    assert "thinking" not in body.get("extra_body", {})
    assert "enable_thinking" not in body.get("extra_body", {})


def test_empty_existing_kb_ignores_old_pending_and_freezes_current_defaults(
    role_environment, tmp_path
):
    from deeptutor.knowledge.manager import KnowledgeBaseManager

    manager = KnowledgeBaseManager(base_dir=str(tmp_path))
    kb = tmp_path / "empty"
    kb.mkdir()
    old = policy.freeze_roles()
    manager.config["knowledge_bases"]["empty"] = {
        "rag_provider": "lightrag",
        "pending_indexing_policy": old.persisted_policy(),
    }
    manager._save_config()
    role_environment["settings"]["role_models"]["extract"]["reasoning_effort"] = "high"
    current = policy.resolve_write_snapshot(kb, base_dir=str(tmp_path), kb_name="empty")
    accepted = policy.bind_target(current, kb)
    assert accepted.extract.config.reasoning_effort == "high"
    role_environment["settings"]["role_models"]["extract"]["reasoning_effort"] = "low"
    role_environment["embedding"].model = "new-global-embedding"
    fresh = policy.revalidate_snapshot(accepted)
    assert fresh.extract.config.reasoning_effort == "high"
    assert fresh.embedding_config.model == "embed-one"


def test_task_publishes_actual_frozen_embedding_after_defaults_change(
    role_environment, tmp_path, monkeypatch
):
    monkeypatch.setattr(engine, "installed_version", lambda: engine.LIGHTRAG_VERSION)
    kb = tmp_path / "kb"
    accepted = policy.bind_target(policy.freeze_roles(), kb, protect_contents=True)
    role_environment["embedding"].model = "later-default"
    pipeline = LightRagPipeline(str(tmp_path))
    monkeypatch.setattr(pipeline, "_ensure_available", lambda: None)

    async def index(root, _files, _progress, snapshot, **_kwargs):
        assert snapshot.embedding_config.model == "embed-one"
        (root / "kv_store_doc_status.json").write_text('{"doc": {"status": "processed"}}')
        return BatchOutcome(
            1, accepted=1, processed=("doc.md",), indexing_policy=snapshot.persisted_policy()
        )

    monkeypatch.setattr(pipeline, "_run_indexing", index)
    receipts: list[list[str]] = []
    assert asyncio.run(
        pipeline.initialize(
            "kb",
            ["doc.md"],
            indexing_snapshot=accepted,
            indexed_file_callback=receipts.append,
        )
    )
    assert receipts == [["doc.md"]]
    published = storage.latest_published_root(kb)
    meta = json.loads((published / "meta.json").read_text())
    assert meta["embedding_model"] == "embed-one"
    assert meta["embedding_dim"] == 3
    assert "fake" not in json.dumps(meta)


@pytest.mark.parametrize("status", ["ready", "error"])
def test_rebuild_confirmation_rejects_changed_defaults_without_queueing(
    role_environment, tmp_path, monkeypatch, status
):
    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from deeptutor.api.routers import knowledge

    entry = {"rag_provider": "lightrag", "status": status}
    monkeypatch.setattr(knowledge, "_writable_kb", lambda _name: (object(), "kb", tmp_path))
    monkeypatch.setattr(knowledge, "_load_kb_entry_or_404", lambda *_a: entry)
    monkeypatch.setattr(knowledge, "_assert_provider_ready", lambda _provider: None)
    queued = []
    monkeypatch.setattr(
        knowledge, "_mark_kb_queued_for_processing", lambda *_a, **_k: queued.append(True)
    )
    captured = []

    async def task(**kwargs):
        captured.append(kwargs["indexing_snapshot"])

    monkeypatch.setattr(knowledge, "run_reindex_task", task)
    app = FastAPI()
    app.include_router(knowledge.router, prefix="/api")
    with TestClient(app) as client:
        for data in ({}, {"config_fingerprint": ""}):
            response = client.post("/api/knowledge-bases/kb/reindex", data=data)
            assert response.status_code == 409
            assert "confirm" in response.json()["detail"]
            assert not queued and not captured
        preview = client.get("/api/knowledge-bases/kb/reindex-config")
        assert preview.status_code == 200
        payload = preview.json()
        assert "private-key" not in preview.text and "password" not in preview.text
        role_environment["embedding"].model = "new-default"
        response = client.post(
            "/api/knowledge-bases/kb/reindex", data={"config_fingerprint": payload["fingerprint"]}
        )
        assert response.status_code == 409
        assert not queued and not captured
        refreshed = client.get("/api/knowledge-bases/kb/reindex-config").json()
        assert refreshed["embedding"]["model"] == "new-default"
        # Credential rotation is not an identity change.
        role_environment["embedding"].api_key = "rotated-secret"
        response = client.post(
            "/api/knowledge-bases/kb/reindex", data={"config_fingerprint": refreshed["fingerprint"]}
        )
        assert response.status_code == 200
        assert len(queued) == 1 and len(captured) == 1
        assert captured[0].embedding_config.model == "new-default"


def test_rebuild_confirmation_preserves_binding_and_freezes_explicit_selection(
    role_environment, tmp_path, monkeypatch
):
    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from deeptutor.api.routers import knowledge

    selections = {name: {"profile_id": "embedding", "model_id": name} for name in ("a", "b")}
    configs = {
        name: EmbeddingConfig(
            model=f"embed-{name}", dim=3, api_key="secret", base_url="https://embed.test"
        )
        for name in selections
    }
    entry = {"rag_provider": "lightrag", "status": "ready", "embedding_selection": selections["b"]}
    monkeypatch.setattr(knowledge, "_writable_kb", lambda _name: (object(), "kb", tmp_path))
    monkeypatch.setattr(knowledge, "_load_kb_entry_or_404", lambda *_a: entry)
    monkeypatch.setattr(knowledge, "_assert_provider_ready", lambda _provider: None)

    def resolve(value, provider, saved=None):
        selection = json.loads(value) if value else saved["embedding_selection"]
        return selection, configs[selection["model_id"]]

    monkeypatch.setattr(knowledge, "_resolve_embedding_form", resolve)
    monkeypatch.setattr(knowledge, "_mark_kb_queued_for_processing", lambda *_a, **_k: None)
    (tmp_path / "kb" / "raw").mkdir(parents=True)
    (tmp_path / "kb" / "raw" / "doc.md").write_text("document")
    captured = []

    async def task(**kwargs):
        role_environment["embedding"].model = "later-global"
        captured.append(kwargs)

    monkeypatch.setattr(knowledge, "run_reindex_task", task)
    app = FastAPI()
    app.include_router(knowledge.router, prefix="/api")
    with TestClient(app) as client:
        bound = client.get("/api/knowledge-bases/kb/reindex-config").json()
        assert bound["embedding"]["model"] == "embed-b"
        assert bound["embedding_selection"] == selections["b"]
        role_environment["embedding"].model = "changed-global"
        assert (
            client.get("/api/knowledge-bases/kb/reindex-config").json()["fingerprint"]
            == bound["fingerprint"]
        )
        data = {
            "embedding_model": json.dumps(selections["a"]),
            "config_fingerprint": bound["fingerprint"],
        }
        assert client.post("/api/knowledge-bases/kb/reindex", data=data).status_code == 409
        assert not captured
        selected = client.get(
            "/api/knowledge-bases/kb/reindex-config",
            params={"embedding_model": data["embedding_model"]},
        ).json()
        data["config_fingerprint"] = selected["fingerprint"]
        assert client.post("/api/knowledge-bases/kb/reindex", data=data).status_code == 200
    assert captured[0]["embedding_selection"] == selections["a"]
    assert captured[0]["embedding_config"].model == "embed-a"
    assert captured[0]["indexing_snapshot"].embedding_config.model == "embed-a"
    assert entry["embedding_selection"] == selections["b"]


def test_append_policy_and_target_follow_older_bound_version(
    role_environment, monkeypatch, tmp_path
):
    from deeptutor.services.embedding.config import embedding_config_scope
    from deeptutor.services.rag import embedding_binding

    monkeypatch.setattr(engine, "installed_version", lambda: engine.LIGHTRAG_VERSION)
    kb = tmp_path / "kb"
    original = deepcopy(role_environment["embedding"])
    first_policy = policy.freeze_roles().persisted_policy()
    first = kb / "version-1"
    _published(first, first_policy)
    role_environment["embedding"].model = "other-vector-space"
    role_environment["settings"]["role_models"]["extract"] = {
        "mode": "model",
        "selection": choice(1),
    }
    second = kb / "version-2"
    _published(second, policy.freeze_roles().persisted_policy())
    for root in (first, second):
        workspace = root / engine.workspace_for(root)
        workspace.mkdir()
        (workspace / "kv_store_doc_status.json").write_text('{"doc":{"status":"processed"}}')
    second_before = (second / "meta.json").read_bytes()
    (tmp_path / "kb_config.json").write_text(
        json.dumps(
            {
                "knowledge_bases": {
                    "kb": {
                        "rag_provider": "lightrag",
                        **embedding_binding.binding_fields(
                            {"profile_id": "embedding", "model_id": "a"}, original
                        ),
                    }
                }
            }
        )
    )
    monkeypatch.setattr(embedding_binding, "binding_status", lambda _entry: ("ready", original))
    accepted = policy.bind_target(
        policy.resolve_write_snapshot(kb, base_dir=str(tmp_path), kb_name="kb"), kb
    )
    assert accepted.target_version == str(first)
    assert accepted.extract.config.model == "model-0"
    assert accepted.embedding_config.model == original.model
    pipeline = LightRagPipeline(str(tmp_path))
    monkeypatch.setattr(pipeline, "_ensure_available", lambda: None)

    async def index(root, _files, _progress, snapshot, **_kwargs):
        assert root == first
        assert snapshot.extract.config.model == "model-0"
        return BatchOutcome(
            1, accepted=1, processed=("new.md",), indexing_policy=snapshot.persisted_policy()
        )

    monkeypatch.setattr(pipeline, "_run_indexing", index)
    with embedding_config_scope(original):
        assert asyncio.run(
            pipeline.add_documents("kb", ["new.md"], accepted_indexing_snapshot=accepted)
        )
    assert (second / "meta.json").read_bytes() == second_before


def test_query_does_not_require_old_extract_or_vlm_access(role_environment, monkeypatch, tmp_path):
    kb = tmp_path / "kb"
    _published(kb / "version-1", policy.freeze_roles().persisted_policy())
    role_environment["allowed"].remove("p3")
    role_environment["allowed"].remove("p0")
    pipeline = LightRagPipeline(str(tmp_path))
    monkeypatch.setattr(pipeline, "_ensure_available", lambda: None)
    monkeypatch.setattr(engine, "build_rag", lambda *_a, **_k: object())

    async def noop(*_a, **_k):
        return None

    async def query(*_a, **_k):
        return "answer", []

    monkeypatch.setattr(engine, "initialize", noop)
    monkeypatch.setattr(engine, "finalize", noop)
    monkeypatch.setattr(engine, "query_with_sources", query)
    assert asyncio.run(pipeline.search("q", "kb"))["answer"] == "answer"
    with pytest.raises(policy.IndexingPolicyError, match="revoked"):
        policy.resolve_write_snapshot(kb, base_dir=str(tmp_path), kb_name="kb")


def test_embedding_adapter_calls_accepted_config_after_default_switch(
    role_environment, monkeypatch
):
    pytest.importorskip("lightrag.utils", reason="requires the optional rag-lightrag extra")
    from deeptutor.services.rag.pipelines.lightrag.config import build_embedding_func

    accepted = policy.with_embedding(policy.freeze_roles())
    role_environment["embedding"].model = "later-default"
    calls = []

    class Client:
        def __init__(self, *, config):
            self.config = config

        async def embed(self, texts, *, input_type):
            calls.append((self.config.model, texts, input_type))
            return [[1.0, 2.0, 3.0]]

    monkeypatch.setattr("deeptutor.services.embedding.client.EmbeddingClient", Client)
    adapter = build_embedding_func(embedding_config=accepted.embedding_config)
    result = asyncio.run(adapter.func(["fixture"], context="document"))
    assert adapter.embedding_dim == 3
    assert result.tolist() == [[1.0, 2.0, 3.0]]
    assert calls == [("embed-one", ["fixture"], "search_document")]