Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

7.8 KB
Raw Blame
"""Measured behaviour of ``llm.request_timeout`` against a local server.

These tests exercise a real ``ChatOpenAI`` built by the provider against a
loopback HTTP server that misbehaves in specific ways, and assert on
elapsed wall-clock time and on the number of request attempts the server
actually saw — not on constructor kwargs.

They pin the semantics the setting documents:

* it bounds each individual network operation, so a silent peer is
  abandoned at the configured value;
* it is *not* a total-response cap, so a peer that keeps dripping bytes
  legitimately outlives it (this is why the setting is named and
  documented as an inactivity bound);
* SDK retries multiply the effective wait, which is why
  ``llm.max_retries`` exists and is bounded.
"""

import json
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

import pytest
from openai import APITimeoutError

from local_deep_research.llm.providers.implementations.custom_openai_endpoint import (
    CustomOpenAIEndpointProvider,
)
from local_deep_research.settings.exceptions import SettingsValueError

_COMPLETION_BODY = json.dumps(
    {
        "id": "chatcmpl-test",
        "object": "chat.completion",
        "created": 0,
        "model": "test-model",
        "choices": [
            {
                "index": 0,
                "message": {"role": "assistant", "content": "hi"},
                "finish_reason": "stop",
            }
        ],
        "usage": {
            "prompt_tokens": 1,
            "completion_tokens": 1,
            "total_tokens": 2,
        },
    }
).encode()

# Long enough that the client always gives up first, short enough that a
# leaked handler thread cannot outlive the test session by much.
_SERVER_HANG_SECONDS = 30.0

# Small enough that the dripped body is split into ~17 writes, so the drip
# test can keep a wide margin between the inter-chunk gap and the read bound
# and still have the whole exchange outlast that bound.
_DRIP_CHUNK_BYTES = 16


class _Handler(BaseHTTPRequestHandler):
    protocol_version = "HTTP/1.1"

    def do_POST(self):  # noqa: N802 - stdlib naming
        length = int(self.headers.get("Content-Length") or 0)
        self.rfile.read(length)
        self.server.attempts.append(time.monotonic())

        mode = self.server.mode
        if mode == "silent":
            # Accept the connection, then never answer.
            self.server.release.wait(_SERVER_HANG_SECONDS)
            return

        self.send_response(200)
        self.send_header("Content-Type", "application/json")
        self.send_header("Content-Length", str(len(_COMPLETION_BODY)))
        self.end_headers()

        if mode == "drip":
            for start in range(0, len(_COMPLETION_BODY), _DRIP_CHUNK_BYTES):
                self.wfile.write(
                    _COMPLETION_BODY[start : start + _DRIP_CHUNK_BYTES]
                )
                self.wfile.flush()
                time.sleep(self.server.gap)
        elif mode == "stall":
            self.wfile.write(_COMPLETION_BODY[:16])
            self.wfile.flush()
            self.server.release.wait(_SERVER_HANG_SECONDS)
        else:
            self.wfile.write(_COMPLETION_BODY)

    def log_message(self, *args):  # pragma: no cover - silence stderr
        pass


@pytest.fixture
def slow_server():
    """A loopback OpenAI-compatible endpoint with configurable badness."""
    server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
    server.daemon_threads = True
    server.mode = "ok"
    server.gap = 0.0
    server.attempts = []
    server.release = threading.Event()
    thread = threading.Thread(target=server.serve_forever, daemon=True)
    thread.start()
    try:
        yield server
    finally:
        server.release.set()
        server.shutdown()
        server.server_close()
        thread.join(timeout=5)


def _make_llm(server, *, timeout, retries):
    snapshot = {
        "llm.openai_endpoint.api_key": "test-key",
        "llm.openai_endpoint.url": (
            f"http://127.0.0.1:{server.server_address[1]}/v1"
        ),
        "llm.supports_max_tokens": False,
        "llm.request_timeout": timeout,
        "llm.max_retries": retries,
    }
    return CustomOpenAIEndpointProvider.create_llm(
        model_name="test-model", settings_snapshot=snapshot
    )


def _elapsed_failure(llm):
    started = time.monotonic()
    with pytest.raises(APITimeoutError):
        llm.invoke("hello")
    return time.monotonic() - started


class TestSilentServer:
    def test_aborts_at_the_configured_timeout_when_retries_are_disabled(
        self, slow_server
    ):
        """A peer that accepts and never answers must not hold the worker."""
        # Given
        slow_server.mode = "silent"
        llm = _make_llm(slow_server, timeout=1, retries=0)

        # When
        elapsed = _elapsed_failure(llm)

        # Then
        assert len(slow_server.attempts) == 1
        assert 0.5 <= elapsed < 3.0, elapsed

    def test_sdk_retries_multiply_the_effective_wait(self, slow_server):
        """One logical call costs (max_retries + 1) x the timeout.

        With ``llm.max_retries`` unwired, both SDKs silently used their own
        default of 2, so a "1 second" bound cost ~3 timeouts plus backoff
        with nothing capping a user-supplied value.
        """
        # Given
        slow_server.mode = "silent"
        no_retry = _make_llm(slow_server, timeout=1, retries=0)
        with_retries = _make_llm(slow_server, timeout=1, retries=2)

        # When
        baseline = _elapsed_failure(no_retry)
        attempts_without_retries = len(slow_server.attempts)
        slow_server.attempts.clear()
        retried = _elapsed_failure(with_retries)

        # Then
        assert attempts_without_retries == 1
        assert len(slow_server.attempts) == 3
        assert retried > baseline * 2, (baseline, retried)

    @pytest.mark.parametrize(
        ("timeout", "retries", "invalid_key"),
        [(0, 0, "llm.request_timeout"), (1, 1000, "llm.max_retries")],
    )
    def test_invalid_settings_prevent_client_construction_and_requests(
        self, slow_server, timeout, retries, invalid_key
    ):
        with pytest.raises(SettingsValueError, match=invalid_key):
            _make_llm(slow_server, timeout=timeout, retries=retries)

        assert slow_server.attempts == []


class TestResponseBodyPacing:
    def test_a_stalled_response_body_aborts_at_the_timeout(self, slow_server):
        """A gap between chunks larger than the bound ends the request."""
        # Given
        slow_server.mode = "stall"
        llm = _make_llm(slow_server, timeout=1, retries=0)

        # When
        elapsed = _elapsed_failure(llm)

        # Then
        assert 0.5 <= elapsed < 5.0, elapsed

    def test_a_steady_drip_outlives_the_timeout_by_design(self, slow_server):
        """The bound is per-operation, not a cap on total response time.

        This is the documented behaviour: as long as bytes keep arriving
        inside the window, the response completes even though the whole
        exchange takes far longer than ``llm.request_timeout``. The
        setting name, description and docs/CONFIGURATION.md must keep
        saying so — if this test ever starts failing because a total
        deadline was added, those need updating with it.
        """
        # Given — a 10x margin between the inter-chunk gap and the read
        # bound, so a loaded CI runner stalling between writes cannot flip
        # the result into a spurious timeout.
        slow_server.mode = "drip"
        slow_server.gap = 0.2
        llm = _make_llm(slow_server, timeout=2, retries=0)

        # When
        started = time.monotonic()
        result = llm.invoke("hello")
        elapsed = time.monotonic() - started

        # Then — the whole exchange outlasts the 2s bound because no single
        # gap reaches it.
        assert result.content == "hi"
        assert elapsed > 2.0, elapsed