Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

22.7 KB
Raw Blame
from __future__ import annotations

from copy import deepcopy
import json
from types import SimpleNamespace

import httpx
from openai import AsyncOpenAI
import pytest

from deeptutor.services.llm.provider_core import github_copilot_provider as module
from deeptutor.services.session.context_builder import ContextBuilder
from deeptutor.services.session.provider_response_state import normalize_provider_response_state
from deeptutor.services.session.sqlite_store import SQLiteSessionStore


@pytest.mark.parametrize("model", ["gpt-6-astra", "github-copilot/gpt-6-astra", "GPT-6-ASTRA"])
@pytest.mark.parametrize("effort", [None, "none", "high"])
def test_astra_never_supports_temperature(model, effort):
    assert module.GitHubCopilotProvider._supports_temperature(model, effort) is False


@pytest.mark.asyncio
async def test_provider_exchanges_stored_token_before_every_uncached_request(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    exchanges = 0
    seen_keys: list[str] = []

    async def exchange(github_token):
        assert github_token == "owner-token"
        nonlocal exchanges
        exchanges += 1
        return SimpleNamespace(
            token="copilot-access",
            expires_at=2_000_000_000,
            api_base="https://tenant.example",
        )

    async def models(_access):
        return [module.CopilotModel("gpt-4.1")]

    async def chat(self, **_kwargs):
        seen_keys.append(self.api_key)
        return SimpleNamespace(content="ok")

    monkeypatch.setattr(module, "exchange_copilot_token", exchange)
    monkeypatch.setattr(module, "fetch_github_copilot_models", models)
    monkeypatch.setattr(
        module,
        "get_github_copilot_storage",
        lambda: SimpleNamespace(load=lambda: SimpleNamespace(access="owner-token")),
    )
    monkeypatch.setattr(module.OpenAICompatProvider, "chat", chat)

    provider = module.GitHubCopilotProvider()
    assert provider._key_pool is None
    await provider.chat(messages=[{"role": "user", "content": "hello"}])
    await provider.chat(messages=[{"role": "user", "content": "again"}])

    assert exchanges == 1
    assert seen_keys == ["copilot-access", "copilot-access"]
    assert provider.api_base == "https://tenant.example"
    assert provider._effective_base == "https://tenant.example"
    assert str(provider._client.base_url) == "https://tenant.example/"
    await provider.aclose()


async def _provider_with_transport(monkeypatch, handler, catalog):
    async def exchange(_token):
        return SimpleNamespace(
            token="copilot-access",
            expires_at=2_000_000_000,
            api_base="https://tenant.example",
        )

    async def models(_access):
        return catalog

    monkeypatch.setattr(module, "exchange_copilot_token", exchange)
    monkeypatch.setattr(module, "fetch_github_copilot_models", models)
    monkeypatch.setattr(
        module,
        "get_github_copilot_storage",
        lambda: SimpleNamespace(load=lambda: SimpleNamespace(access="owner-token")),
    )
    provider = module.GitHubCopilotProvider()
    await provider.aclose()
    provider._client = AsyncOpenAI(
        api_key="no-key",
        max_retries=0,
        http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
    )
    return provider


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize(
    "model,endpoints,path",
    [
        ("gpt-6-astra", ("/responses",), "/responses"),
        ("gpt-6-astra", ("/chat/completions",), "/chat/completions"),
        ("claude-sonnet", ("/responses",), "/responses"),
        ("gpt-5-chat-only", ("/chat/completions",), "/chat/completions"),
        ("gpt-5", ("/responses", "/chat/completions"), "/responses"),
        ("claude-both", ("/responses", "/chat/completions"), "/chat/completions"),
    ],
)
async def test_model_protocol_metadata_routes_actual_sdk_requests(
    monkeypatch, stream, model, endpoints, path
):
    calls = []

    def handler(request):
        calls.append(request)
        assert request.url.host == "tenant.example"
        assert request.url.path == path
        assert request.headers["authorization"] == "Bearer copilot-access"
        body = json.loads(request.content)
        assert body["model"] == model
        if model == "gpt-6-astra":
            assert "temperature" not in body
        elif model.startswith("claude"):
            assert body["temperature"] == 0.7
        if stream:
            if path == "/responses":
                events = [
                    {"type": "response.output_text.delta", "delta": "OK"},
                    {
                        "type": "response.completed",
                        "response": {
                            "id": "resp_1",
                            "status": "completed",
                            "output": [],
                        },
                    },
                ]
            else:
                events = [
                    {
                        "id": "chat_1",
                        "choices": [
                            {"index": 0, "delta": {"content": "OK"}, "finish_reason": "stop"}
                        ],
                    }
                ]
            content = "".join(f"data: {json.dumps(event)}\n\n" for event in events)
            return httpx.Response(
                200,
                headers={"content-type": "text/event-stream"},
                text=content + "data: [DONE]\n\n",
            )
        if path == "/responses":
            return httpx.Response(
                200,
                json={
                    "id": "resp_1",
                    "status": "completed",
                    "output": [
                        {
                            "type": "message",
                            "role": "assistant",
                            "content": [{"type": "output_text", "text": "OK"}],
                        }
                    ],
                },
            )
        return httpx.Response(
            200,
            json={
                "id": "chat_1",
                "choices": [
                    {
                        "index": 0,
                        "message": {"role": "assistant", "content": "OK"},
                        "finish_reason": "stop",
                    }
                ],
            },
        )

    provider = await _provider_with_transport(
        monkeypatch, handler, [module.CopilotModel(model, endpoints)]
    )
    # A Responses-only model must not be redirected by the heuristic circuit breaker.
    provider._responses_circuit_allows = lambda *args: False if len(endpoints) == 1 else True
    try:
        method = provider.chat_stream if stream else provider.chat
        response = await method(
            messages=[{"role": "user", "content": "hi"}], model=f"github-copilot/{model}"
        )
        assert response.finish_reason != "error", response.content
        assert response.content == "OK"
        assert len(calls) == 1
    finally:
        await provider.aclose()


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_responses_only_error_never_falls_back_to_chat(monkeypatch, stream):
    calls = []

    def handler(request):
        calls.append(request.url.path)
        return httpx.Response(400, json={"error": {"message": "responses unavailable"}})

    provider = await _provider_with_transport(
        monkeypatch, handler, [module.CopilotModel("claude", ("/responses",))]
    )
    try:
        method = provider.chat_stream if stream else provider.chat
        response = await method(messages=[], model="claude")
        assert response.finish_reason == "error"
        assert calls == ["/responses"]
    finally:
        await provider.aclose()


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("replay", [False, True])
async def test_multiturn_responses_omits_output_message_metadata(monkeypatch, stream, replay):
    messages = [
        {"role": "user", "content": "Hello"},
        {"role": "assistant", "content": "Hi"},
        {"role": "user", "content": "Continue"},
    ]
    if replay:
        messages[1]["_provider_response_state"] = {
            "responses_output_items": [
                {
                    "type": "message",
                    "role": "assistant",
                    "id": "msg_previous",
                    "status": "completed",
                    "content": [{"type": "output_text", "text": "Hi"}],
                },
                {
                    "type": "function_call",
                    "id": "fc_previous",
                    "call_id": "call_previous",
                    "name": "lookup",
                    "arguments": "{}",
                },
            ]
        }
        messages.insert(2, {"role": "tool", "tool_call_id": "call_previous", "content": "result"})
    original = deepcopy(messages)
    calls = []

    def handler(request):
        body = json.loads(request.content)
        calls.append(body)
        assert request.url.path == "/responses"
        assistant = body["input"][1]
        assert assistant["role"] == "assistant"
        assert assistant["content"][0]["text"] == "Hi"
        assert "status" not in assistant
        assert "id" not in assistant
        if replay:
            assert body["input"][2]["id"] == "fc_previous"
            assert body["input"][2]["call_id"] == body["input"][3]["call_id"]
        if stream:
            events = [
                {"type": "response.output_text.delta", "delta": "OK"},
                {
                    "type": "response.completed",
                    "response": {"id": "resp_2", "status": "completed", "output": []},
                },
            ]
            return httpx.Response(
                200,
                headers={"content-type": "text/event-stream"},
                text="".join(f"data: {json.dumps(event)}\n\n" for event in events),
            )
        return httpx.Response(
            200,
            json={
                "id": "resp_2",
                "status": "completed",
                "output": [
                    {
                        "type": "message",
                        "role": "assistant",
                        "content": [{"type": "output_text", "text": "OK"}],
                    }
                ],
            },
        )

    provider = await _provider_with_transport(
        monkeypatch, handler, [module.CopilotModel("gpt-5", ("/responses",))]
    )
    try:
        method = provider.chat_stream if stream else provider.chat
        response = await method(messages=messages, model="gpt-5")
        assert response.finish_reason != "error", response.content
        assert response.content == "OK"
        assert len(calls) == 1
        assert messages == original
    finally:
        await provider.aclose()


def _reasoning_output(index):
    # Shape observed on Copilot's wire: no status. SDK model_dump adds status=None.
    return {
        "type": "reasoning",
        "id": f"rs_{index}",
        "summary": [],
        "content": [],
        "encrypted_content": f"opaque-test-reasoning-{index}",
    }


def _message_output(index):
    return {
        "type": "message",
        "id": f"msg_{index}",
        "role": "assistant",
        "status": "completed",
        "phase": "final_answer",
        "content": [{"type": "output_text", "text": "OK", "annotations": []}],
    }


def _responses_reply(output, stream):
    response = {"id": "resp_test", "status": "completed", "output": output}
    if not stream:
        return httpx.Response(200, json=response)
    events = []
    for index, item in enumerate(output):
        events.append({"type": "response.output_item.added", "output_index": index, "item": item})
        if item["type"] == "message":
            events.append({"type": "response.output_text.delta", "delta": "OK"})
        events.append({"type": "response.output_item.done", "output_index": index, "item": item})
    events.append({"type": "response.completed", "response": response})
    return httpx.Response(
        200,
        headers={"content-type": "text/event-stream"},
        text="".join(f"data: {json.dumps(event)}\n\n" for event in events),
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_four_turn_sqlite_history_replays_sdk_reasoning_without_status(
    monkeypatch, tmp_path, stream
):
    bodies = []

    def handler(request):
        body = json.loads(request.content)
        bodies.append(body)
        for index, item in enumerate(body["input"]):
            if "status" in item:
                return httpx.Response(
                    400,
                    json={"error": {"message": f"Unknown parameter: 'input[{index}].status'."}},
                )
        turn = len(bodies)
        output = ([] if turn == 1 else [_reasoning_output(turn)]) + [_message_output(turn)]
        return _responses_reply(output, stream)

    store = SQLiteSessionStore(db_path=tmp_path / "copilot-replay.db")
    session = await store.create_session()
    provider = await _provider_with_transport(
        monkeypatch, handler, [module.CopilotModel("gpt-5.6-sol-fast", ("/responses",))]
    )
    try:
        method = provider.chat_stream if stream else provider.chat
        for turn in range(1, 5):
            await store.add_message(session["id"], "user", f"Synthetic turn {turn}")
            stored = await store.get_messages(session["id"])
            history = ContextBuilder(store)._build_history("", stored)
            original = deepcopy(history)
            result = await method(messages=history, model="gpt-5.6-sol-fast")
            assert result.finish_reason == "stop", result.content
            assert history == original
            items = result.provider_specific_fields.get("native_output_items", [])
            if turn >= 2:
                assert items[0]["type"] == "reasoning"
                assert "status" in items[0] and items[0]["status"] is None
                assert items[0]["encrypted_content"] == f"opaque-test-reasoning-{turn}"
            state = normalize_provider_response_state({"responses_output_items": items})
            await store.add_message(
                session["id"],
                "assistant",
                result.content,
                metadata={"provider_response_state": state},
            )
            if turn >= 3:
                reasoning = bodies[-1]["input"][3]
                assert reasoning["type"] == "reasoning"
                assert reasoning["id"] == "rs_2"
                assert reasoning["encrypted_content"] == "opaque-test-reasoning-2"
                assert reasoning["summary"] == []
                assert "status" not in reasoning
                assistant = bodies[-1]["input"][4]
                assert assistant["phase"] == "final_answer"
                assert "id" not in assistant
            persisted = await store.get_messages(session["id"])
            assert persisted[:-1] == stored
            assert persisted[-1]["metadata"]["provider_response_state"] == state
        assert len(bodies) == 4
    finally:
        await provider.aclose()


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_replayed_reasoning_and_tool_rounds_preserve_call_pairing(monkeypatch, stream):
    bodies = []

    def handler(request):
        body = json.loads(request.content)
        bodies.append(body)
        assert all(
            "status" not in item
            for item in body["input"]
            if item.get("type") in {"message", "reasoning"}
        )
        for item in body["input"]:
            if item.get("type") == "function_call":
                assert item["status"] == "completed"
                assert item["id"] == f"fc_{item['call_id'].split('_')[-1]}"
                assert item["arguments"] == '{"status":"keep nested argument"}'
                assert any(
                    output.get("type") == "function_call_output"
                    and output["call_id"] == item["call_id"]
                    for output in body["input"]
                )
        turn = len(bodies)
        output = [
            _reasoning_output(turn),
            {
                "type": "function_call",
                "id": f"fc_{turn}",
                "call_id": f"call_{turn}",
                "name": "lookup",
                "arguments": '{"status":"keep nested argument"}',
                "status": "completed",
            },
        ]
        return _responses_reply(output, stream)

    provider = await _provider_with_transport(
        monkeypatch, handler, [module.CopilotModel("gpt-5", ("/responses",))]
    )
    try:
        history = [{"role": "user", "content": "Use lookup"}]
        method = provider.chat_stream if stream else provider.chat
        for turn in range(1, 5):
            original = deepcopy(history)
            result = await method(messages=history, model="gpt-5")
            assert result.finish_reason != "error", result.content
            assert history == original
            items = result.provider_specific_fields["native_output_items"]
            assert items[1]["status"] == "completed"
            call = result.tool_calls[0]
            assert call.id == f"call_{turn}|fc_{turn}"
            history.extend(
                [
                    {
                        "role": "assistant",
                        "content": "",
                        "_provider_response_state": {"responses_output_items": items},
                    },
                    {"role": "tool", "tool_call_id": call.id, "content": "Synthetic result"},
                ]
            )
            history = json.loads(json.dumps(history))
        assert len(bodies) == 4
    finally:
        await provider.aclose()


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("override", ["input", "extra_body"])
@pytest.mark.parametrize("reasoning_status", [None, "completed"])
async def test_responses_input_overrides_are_normalized_after_kwargs_merge(
    monkeypatch, stream, override, reasoning_status
):
    items = [
        {"role": "assistant", "id": "msg_raw", "status": "completed", "content": "OK"},
        {**_reasoning_output(1), "status": reasoning_status},
        {
            "type": "function_call",
            "id": "fc_1",
            "call_id": "call_1",
            "name": "lookup",
            "arguments": "{}",
            "status": "completed",
        },
        {"type": "function_call_output", "call_id": "call_1", "output": "OK", "status": None},
        {"type": "web_search_call", "id": "ws_1", "status": "completed"},
    ]
    kwargs = {"input": items} if override == "input" else {"extra_body": {"input": items}}
    original = deepcopy(kwargs)

    def handler(request):
        body = json.loads(request.content)
        expected = deepcopy(items)
        for item in expected[:2]:
            item.pop("status")
        expected[0].pop("id")
        assert body["input"] == expected
        return _responses_reply([_message_output(1)], stream)

    provider = await _provider_with_transport(
        monkeypatch, handler, [module.CopilotModel("gpt-5", ("/responses",))]
    )
    try:
        method = provider.chat_stream if stream else provider.chat
        result = await method(
            messages=[{"role": "user", "content": "Ignored"}], model="gpt-5", **kwargs
        )
        assert result.finish_reason == "stop", result.content
        assert kwargs == original
    finally:
        await provider.aclose()


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_copilot_normalization_does_not_change_openai_requests(stream):
    items = [{**_reasoning_output(1), "status": None}, _message_output(1)]
    history = [
        {
            "role": "assistant",
            "content": "OK",
            "_provider_response_state": {"responses_output_items": items},
        },
        {"role": "user", "content": "Continue"},
    ]
    original = deepcopy(history)
    bodies = []

    def handler(request):
        body = json.loads(request.content)
        bodies.append(body)
        assert body["input"][:2] == items
        return _responses_reply([_message_output(2)], stream)

    provider = module.OpenAICompatProvider(
        api_key="test-key", wire_api="responses", configure_env=False
    )
    await provider.aclose()
    provider._client = AsyncOpenAI(
        api_key="test-key",
        max_retries=0,
        http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
    )
    try:
        method = provider.chat_stream if stream else provider.chat
        result = await method(messages=history, model="gpt-5")
        assert result.finish_reason == "stop", result.content
        assert len(bodies) == 1
        assert history == original
    finally:
        await provider.aclose()


@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_unavailable_model_does_not_send_inference(monkeypatch, stream):
    provider = await _provider_with_transport(
        monkeypatch, lambda request: pytest.fail("unavailable model was requested"), []
    )
    try:
        method = provider.chat_stream if stream else provider.chat
        response = await method(messages=[], model="removed-model")
        assert response.finish_reason == "error"
        assert "not currently available" in response.content
    finally:
        await provider.aclose()


@pytest.mark.asyncio
async def test_refresh_changes_endpoint_and_keeps_original_owner(monkeypatch):
    seen = []

    def handler(request):
        seen.append((request.url.host, request.headers["authorization"]))
        assert request.url.path == "/api/chat/completions"
        return httpx.Response(
            200, json={"choices": [{"message": {"content": "OK"}, "finish_reason": "stop"}]}
        )

    provider = await _provider_with_transport(
        monkeypatch, handler, [module.CopilotModel("gpt-4.1", ("/chat/completions",))]
    )
    count = 0

    async def exchange(token):
        nonlocal count
        assert token == "owner-token"
        count += 1
        return SimpleNamespace(
            token=f"token-{count}",
            expires_at=2_000_000_000,
            api_base=f"https://tenant-{count}.example/api",
        )

    monkeypatch.setattr(module, "exchange_copilot_token", exchange)
    try:
        await provider.chat(messages=[])
        # A later request context must not change the owner attached to the provider.
        monkeypatch.setattr(
            module, "get_github_copilot_storage", lambda: pytest.fail("wrong owner")
        )
        provider._copilot_expires_at = 0
        await provider.chat(messages=[])
        assert seen == [
            ("tenant-1.example", "Bearer token-1"),
            ("tenant-2.example", "Bearer token-2"),
        ]
        assert provider.api_base == provider._effective_base == "https://tenant-2.example/api"
        assert str(provider._client.base_url) == "https://tenant-2.example/api/"
    finally:
        await provider.aclose()