Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

8.0 KB
Raw Blame
"""Tests for the provider connectivity-test classifier + service.

The classifier turns a raw provider error message into an HONEST outcome status
that separates an *integration* problem (broken code / unreachable) from an
*account* state (invalid key / no credits / not configured). That distinction is
the whole point of the feature: a green "configured" badge says a key STRING is
stored, never that the provider answers — and a 401/403 proves the integration
reached the provider and only the credential/account was rejected.

Error strings below are VERBATIM from live probes against the real providers
(Anthropic 401, an OpenAI-compatible 403 out-of-credits, OpenAI missing-key
RuntimeError).
"""

from __future__ import annotations

import pytest

from dodaios.brain.provider_test import (
    PROVIDER_TEST_STATUSES,
    classify_provider_error,
)


def test_status_vocabulary_is_the_expected_closed_set() -> None:
    # Single source of truth — the Pydantic Literal + the TS union must mirror this.
    assert set(PROVIDER_TEST_STATUSES) == {
        "ok",
        "not_configured",
        "bad_key",
        "no_credits",
        "rate_limited",
        "model_unavailable",
        "unreachable",
        "error",
    }


def test_anthropic_401_invalid_key_is_bad_key() -> None:
    msg = (
        "AuthenticationError: Error code: 401 - {'type': 'error', 'error': "
        "{'type': 'authentication_error', 'message': 'invalid x-api-key'}}"
    )
    assert classify_provider_error(msg) == "bad_key"


def test_403_out_of_credits_is_no_credits() -> None:
    msg = (
        "PermissionDeniedError: Error code: 403 - {'code': 'permission-denied', "
        "'error': 'Your team e6d8 has either used all available credits or reached "
        "its monthly spending limit. To continue making API requests, please "
        "purchase more credits or raise your spending limit.'}"
    )
    assert classify_provider_error(msg) == "no_credits"


def test_missing_key_runtimeerror_is_not_configured() -> None:
    msg = "RuntimeError: Kein OpenAI-API-Key gefunden (openai_api_key / OPENAI_API_KEY)."
    assert classify_provider_error(msg) == "not_configured"


def test_english_missing_key_is_not_configured() -> None:
    assert classify_provider_error("No API key found for provider") == "not_configured"


def test_openrouter_key_spend_limit_403_is_no_credits() -> None:
    # Live probe 2026-06-30: an OpenRouter key with a per-key spend cap that is
    # used up returns 403 "Key limit exceeded (total limit)" for EVERY paid model,
    # while the SAME key still answers free models. It is a funded-account budget
    # block, NOT an invalid key — labeling it "Key invalid" (bad_key) sent the user
    # hunting the wrong problem. Must read as out-of-credits.
    msg = (
        "PermissionDeniedError: Error code: 403 - {'error': {'message': "
        "'Key limit exceeded (total limit). Manage it using "
        "https://openrouter.ai/settings/keys', 'code': 403}}"
    )
    assert classify_provider_error(msg) == "no_credits"


def test_generic_spend_limit_without_credit_word_is_no_credits() -> None:
    # Robustness across providers: a 403 whose wording is a budget/limit block but
    # does not contain the literal word "credit" must still read as no_credits, not
    # bad_key. (The old marker list keyed on "credit"/"spending limit" only.)
    msg = "PermissionDeniedError: Error code: 403 - monthly budget exceeded for this key"
    assert classify_provider_error(msg) == "no_credits"


def test_openai_insufficient_quota_429_is_no_credits() -> None:
    msg = (
        "RateLimitError: Error code: 429 - {'error': {'message': 'You exceeded your "
        "current quota, please check your plan and billing details.', "
        "'code': 'insufficient_quota'}}"
    )
    assert classify_provider_error(msg) == "no_credits"


@pytest.mark.parametrize(
    "msg",
    (
        (
            "ClientError: 429 Too Many Requests. Your project has exceeded "
            "its monthly spending cap. status: RESOURCE_EXHAUSTED"
        ),
        (
            "APIError: 1011 None. Your project has exceeded its monthly "
            "spending cap. Please go to AI Studio to manage your project"
        ),
    ),
)
def test_gemini_monthly_spending_cap_is_no_credits(msg: str) -> None:
    """REST 429 and Live 1011 are the same terminal account state."""
    assert classify_provider_error(msg) == "no_credits"


@pytest.mark.parametrize(
    "msg",
    (
        (
            "1011 None. Resource exhausted. Please try again later. Please "
            "refer to https://cloud.google.com/vertex-ai/generative-ai/docs/"
            "model-reference/gemini-live"
        ),
        "APIError: 1011 None. Resource exhausted. Please try again later.",
        'ClientError: status: "RESOURCE_EXHAUSTED"',
    ),
)
def test_google_live_resource_exhausted_without_billing_is_rate_limited(
    msg: str,
) -> None:
    """Vertex/Gemini Live 1011 capacity throttle is not a spending cap.

    Live 2026-08-20 21:15: vertex-live closed three times with this exact
    wording, the classifier returned ``error``, and the rebuild-capable
    transport treated it as a 1006-style wire death — silent reconnect
    storm, then listening, nothing said. Billing markers must still win
    when the same gRPC status carries a spending cap.
    """
    assert classify_provider_error(msg) == "rate_limited"


def test_plain_429_rate_limit_is_rate_limited() -> None:
    msg = "RateLimitError: Error code: 429 - rate limit exceeded, please slow down"
    assert classify_provider_error(msg) == "rate_limited"


def test_404_model_not_found_is_model_unavailable() -> None:
    msg = (
        "NotFoundError: Error code: 404 - {'error': {'message': "
        "'The model `gpt-9` does not exist or you do not have access to it.'}}"
    )
    assert classify_provider_error(msg) == "model_unavailable"


def test_connection_error_is_unreachable() -> None:
    assert classify_provider_error("APIConnectionError: Connection error.") == "unreachable"


def test_timeout_is_unreachable() -> None:
    assert classify_provider_error("timeout after 25.0s") == "unreachable"


def test_local_server_not_reachable_is_unreachable() -> None:
    """The local brains phrase connectivity honestly ("… not reachable at … —
    is it running?"); that must read as unreachable, and the port number must
    never be mistaken for an HTTP status code."""
    msg = (
        "RuntimeError: Ollama not reachable at http://localhost:11434 — is it "
        "running? Start it (or install from https://ollama.com/download), then retry."
    )
    assert classify_provider_error(msg) == "unreachable"


def test_local_server_without_models_is_model_unavailable() -> None:
    """Reachable local server, nothing pulled — the fix is `ollama pull`, so
    the chip must say model-unavailable, not integration error."""
    msg = (
        "RuntimeError: Ollama at http://localhost:11434 has no models installed "
        "— run: ollama pull qwen3.5, then retry."
    )
    assert classify_provider_error(msg) == "model_unavailable"


def test_local_openai_empty_model_list_is_model_unavailable() -> None:
    msg = (
        "RuntimeError: The server at http://localhost:8000 lists no models "
        "(/v1/models is empty) — load a model in the server, or set one explicitly."
    )
    assert classify_provider_error(msg) == "model_unavailable"


def test_local_openai_missing_base_url_is_not_configured() -> None:
    """No server URL on the card = the local analogue of "no key set" — an
    actionable amber, never a red integration error."""
    msg = (
        "RuntimeError: The local OpenAI-compatible provider needs a server URL "
        "— set it on the provider card."
    )
    assert classify_provider_error(msg) == "not_configured"


def test_unknown_error_is_error() -> None:
    assert classify_provider_error("ValueError: something structurally broke") == "error"


def test_none_or_empty_is_error() -> None:
    assert classify_provider_error(None) == "error"
    assert classify_provider_error("") == "error"