Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

22.5 KB
Raw Blame
"""User-visible failure copy consumes exact Hub facts, not native error text."""

import json
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock

import aiohttp
import pytest

from config.v2_config import (
    ModelHubBackendModelConfig,
    ModelHubRouteConfig,
    ModelHubRouteHopConfig,
    ModelHubSourceStateConfig,
)
from core.backend_failure import emit_backend_failure, emit_replayed_backend_failure
from core.controller import Controller
from core.handlers.model_hub.adapter import RawOutcomeKind
from core.handlers.model_hub.provenance import (
    ENGINE_DOWN_TURN_OUTCOME,
    TurnOutcomeProjectionInput,
    TurnSupplyFacts,
    TurnUpstreamFailure,
    render_turn_outcome_copy,
)
from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
from core.run_settlement import SETTLED_BY_TERMINAL_RESULT
from modules.agents.catalog import AGENT_BACKENDS
from modules.agents.model_hub import ModelHubRuntimeRouter, bind_launch
from modules.im import MessageContext
from tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _outcome
from tests.test_model_hub_resolution import _config, _service, _source
from tests.test_model_hub_retry_policy import clock_service
from vibe.i18n import t


def unavailable(tmp_path, backend="codex"):
    source = _source("src_copy0001", ("模型/alpha",))
    source.display_name = "Relay 服务"
    source.state = ModelHubSourceStateConfig(
        status="cooldown",
        detail_key="models.source.cooldown.server_error",
        retry_at="2026-09-09T04:30:00+00:00",
    )
    config = _config([source], model="模型/alpha")
    config.agents[backend].models.append(ModelHubBackendModelConfig(id="模型/alpha", origin="manual"))
    config.agents[backend].routes["模型/alpha"] = ModelHubRouteConfig(
        hops=(ModelHubRouteHopConfig(source.id, "模型/alpha"),),
    )
    service, _, _ = _service(tmp_path, config)
    config, resolution = service._inspect_terminal_chain(backend=backend, model_id="模型/alpha")
    return service._produce_exhausted_terminal_outcome(
        config=config, resolution=resolution,
    )


@pytest.mark.parametrize("backend", AGENT_BACKENDS)
@pytest.mark.parametrize("platform", ["avibe", "slack"])
@pytest.mark.parametrize("language", ["en", "zh"])
async def test_shared_failure_uses_exact_hub_copy_and_preserves_terminal_evidence(
    tmp_path, backend, platform, language,
):
    projection = unavailable(tmp_path, backend)
    project = Mock(return_value=projection)
    controller = SimpleNamespace(
        config=SimpleNamespace(language=language),
        model_hub_turn_gateway=SimpleNamespace(
            correlation=SimpleNamespace(terminal_projection=project),
        ),
        emit_agent_message=AsyncMock(return_value="msg-failed"),
    )
    context = MessageContext(
        user_id="user", channel_id="channel", platform=platform,
        platform_specific={"turn_token": "turn-exact"},
    )
    diagnostic = (
        "Codex turn failed: unexpected status 503 Service Unavailable; "
        "url: http://127.0.0.1:49800/codex/v1/responses"
    )

    await emit_backend_failure(controller, context, backend, diagnostic)

    project.assert_called_once_with("turn-exact", backend=backend)
    notify, terminal = controller.emit_agent_message.await_args_list
    assert notify.args[1] == "notify"
    assert notify.args[2] == render_turn_outcome_copy(projection, language)
    assert "127.0.0.1" not in notify.args[2]
    assert "2026-09-09" not in notify.args[2]
    assert "Relay 服务" in notify.args[2]
    assert notify.kwargs["output"].metadata["event"] == "backend_failure"
    assert terminal.args[1:] == ("result", "")
    assert terminal.kwargs["terminal_error"] == diagnostic
    assert terminal.kwargs["level"] == "silent"


@pytest.mark.parametrize("projection", [ENGINE_DOWN_TURN_OUTCOME, None])
@pytest.mark.parametrize("language", ["en", "zh"])
async def test_failure_after_shutdown_request_reports_the_restart(projection, language):
    """Shutdown stops the Hub engine first; the in-flight Turn's notice names the restart."""
    controller = Controller.__new__(Controller)
    controller.config = SimpleNamespace(language=language)
    controller.model_hub_turn_gateway = SimpleNamespace(
        correlation=SimpleNamespace(terminal_projection=Mock(return_value=projection)),
    )
    controller.emit_agent_message = AsyncMock(return_value="msg-failed")
    controller.request_shutdown("signal")
    context = MessageContext(
        user_id="user", channel_id="channel", platform="avibe",
        platform_specific={"turn_token": "turn-restart"},
    )
    diagnostic = "Claude turn failed: 502 engine_down"
    symptom = (
        render_turn_outcome_copy(projection, language)
        if projection is not None else "Claude process exited unexpectedly"
    )

    await emit_backend_failure(controller, context, "claude", diagnostic, display_text=symptom)

    notify, terminal = controller.emit_agent_message.await_args_list
    assert notify.args[2] == t("turn.interrupted.serviceRestart", language) != symptom
    assert terminal.kwargs["terminal_error"] == diagnostic


@pytest.mark.parametrize("identity", [None, "", 123, "different-turn"])
async def test_no_exact_projection_keeps_unrelated_native_failure(identity):
    project = Mock(return_value=None)
    controller = SimpleNamespace(
        model_hub_turn_gateway=SimpleNamespace(
            correlation=SimpleNamespace(terminal_projection=project),
        ),
        emit_agent_message=AsyncMock(),
    )
    context = MessageContext(
        user_id="user", channel_id="channel", platform="avibe",
        platform_specific={"turn_token": identity},
    )
    await emit_backend_failure(
        controller, context, "codex", "native failed",
        display_text="Native process exited unexpectedly",
    )
    assert controller.emit_agent_message.await_args_list[0].args[2] == (
        "Native process exited unexpectedly"
    )
    if identity == "different-turn":
        project.assert_called_once_with(identity, backend="codex")
    else:
        project.assert_not_called()


async def test_historical_notice_does_not_consume_a_live_turn_projection(tmp_path):
    project = Mock(return_value=unavailable(tmp_path))
    controller = SimpleNamespace(
        model_hub_turn_gateway=SimpleNamespace(
            correlation=SimpleNamespace(terminal_projection=project),
        ),
        emit_agent_message=AsyncMock(),
    )
    context = MessageContext(
        user_id="user", channel_id="channel", platform="avibe",
        platform_specific={"turn_token": "new-live-turn"},
    )
    await emit_replayed_backend_failure(
        controller, context, "codex", "old failure",
        display_text="Original failure notice", failure_id="old-failure",
    )
    project.assert_not_called()
    assert controller.emit_agent_message.await_count == 1
    assert controller.emit_agent_message.await_args.args[2] == "Original failure notice"


@pytest.mark.parametrize("backend,endpoint,status", [
    ("codex", "responses", 400), ("claude", "messages", 424),
])
@pytest.mark.parametrize("language", ["en", "zh"])
async def test_gateway_terminal_survives_native_recorder_before_shared_notice(
    tmp_path, backend, endpoint, status, language,
):
    """MH-RETRY-COPY-001: actual HTTP terminal -> native recorder -> one clean notice."""
    service, clock = clock_service(tmp_path, outcomes=[
        _outcome(RawOutcomeKind.NETWORK_ERROR, source_id="src_recovery01")
        for _ in range(20)
    ])
    service.store.load().sources[0].display_name = "Relay 服务"
    models = _canonicalize_fixed_test_routes(service)
    gateway = ModelHubTurnGateway(service)
    router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
    context = MessageContext(
        user_id="user", channel_id="channel", platform="avibe",
        platform_specific={"turn_token": "turn-copy"},
    )
    controller = SimpleNamespace(
        config=SimpleNamespace(language=language),
        model_hub_turn_gateway=gateway,
        emit_agent_message=AsyncMock(return_value="msg-failed"),
    )
    try:
        launch = await router.resolve(
            backend, models[backend], process_scope="fixture-copy", turn_id="turn-copy",
        )
        bind_launch(context, launch)
        async with aiohttp.ClientSession(trust_env=False, timeout=aiohttp.ClientTimeout(total=5)) as client:
            async with client.post(
                f"{launch.gateway_base_url}/v1/{endpoint}",
                headers={
                    "Authorization": f"Bearer {launch.gateway_token}",
                    **({"x-codex-turn-metadata": json.dumps(launch.gateway_request_metadata)}
                       if launch.gateway_request_metadata else {}),
                },
                json={"model": launch.runtime_model, "stream": False},
            ) as response:
                assert response.status == status
                assert (await response.json())["error"]["code"] == "model_hub_recovery_exhausted"
                assert "Retry-After" not in response.headers
        assert clock.elapsed == 91
        assert len(service.adapter.invocations) == 8
        controller.emit_agent_message.assert_not_awaited()
        projection = gateway.correlation.terminal_projection("turn-copy", backend=backend)
        assert projection is not None
        diagnostic = f"Native error {status}, URL http://127.0.0.1/model-request"
        assert await router.record_native_failure(context, diagnostic) is False
        assert gateway.correlation.terminal_projection("turn-copy", backend=backend) == projection
        await emit_backend_failure(controller, context, backend, diagnostic)
        notice, terminal = controller.emit_agent_message.await_args_list
        assert notice.args[2] == render_turn_outcome_copy(projection, language)
        assert "Relay 服务" in notice.args[2]
        assert "127.0.0.1" not in notice.args[2]
        assert ("已结束" if language == "zh" else "has ended") in notice.args[2]
        assert terminal.kwargs["terminal_error"] == diagnostic
        assert gateway.correlation.recovery_snapshot("turn-copy") == []
    finally:
        await gateway.close()


@pytest.mark.parametrize("last_said", ["text", "nothing", "connection"])
@pytest.mark.parametrize("backend,endpoint,status", [
    ("codex", "responses", 400), ("claude", "messages", 424),
])
@pytest.mark.parametrize("language", ["en", "zh"])
async def test_upstream_refusal_text_reaches_reply_record_and_log(
    tmp_path, caplog, backend, endpoint, status, language, last_said,
):
    """MH-UPSTREAM-DETAIL-001: what the upstream said is shown, kept, and logged, not only its class."""
    detail = "upstream request failed: read tcp 198.51.100.7:443: i/o timeout"
    if last_said == "connection":
        # The engine's own transport envelope: no upstream answered at all.
        texts = [detail] * 12
        failure = _outcome(RawOutcomeKind.NETWORK_ERROR, source_id="src_recovery01")
    else:
        # Only the first refusal carries text when the last one said nothing.
        texts = [detail] + [detail if last_said == "text" else None] * 4
        failure = _outcome(RawOutcomeKind.HTTP_ERROR, source_id="src_recovery01", status=500, code="server_error")
    service, _clock = clock_service(tmp_path, outcomes=[
        replace(failure, upstream_detail=text) for text in texts
    ])
    service.store.load().sources[0].display_name = "Relay 服务"
    models = _canonicalize_fixed_test_routes(service)
    gateway = ModelHubTurnGateway(service)
    router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
    context = MessageContext(
        user_id="user", channel_id="channel", platform="avibe",
        platform_specific={"turn_token": "turn-detail"},
    )
    controller = SimpleNamespace(
        config=SimpleNamespace(language=language),
        model_hub_turn_gateway=gateway,
        emit_agent_message=AsyncMock(return_value="msg-failed"),
    )
    try:
        launch = await router.resolve(
            backend, models[backend], process_scope="fixture-detail", turn_id="turn-detail",
        )
        bind_launch(context, launch)
        caplog.set_level("WARNING", logger="core.handlers.model_hub.provenance")
        async with aiohttp.ClientSession(trust_env=False, timeout=aiohttp.ClientTimeout(total=5)) as client:
            async with client.post(
                f"{launch.gateway_base_url}/v1/{endpoint}",
                headers={
                    "Authorization": f"Bearer {launch.gateway_token}",
                    **({"x-codex-turn-metadata": json.dumps(launch.gateway_request_metadata)}
                       if launch.gateway_request_metadata else {}),
                },
                json={"model": launch.runtime_model, "stream": False},
            ) as response:
                assert response.status == status
        attempts = len(service.adapter.invocations)
        assert attempts >= 2
        diagnostic = f"Native error {status}"
        assert await router.record_native_failure(context, diagnostic) is False
        await emit_backend_failure(controller, context, backend, diagnostic)
        notice, _terminal = controller.emit_agent_message.await_args_list
        projection = gateway.correlation.terminal_projection("turn-detail", backend=backend)
        assert notice.args[2] == render_turn_outcome_copy(projection, language)
        suffix = t(
            "modelHub.launch.last_upstream_failure", language,
            source="Relay 服务", status=500, detail=detail,
        )
        connection = t("modelHub.launch.last_connection_failure", language, source="Relay 服务", detail=detail)
        # An older message never stands in for a newer refusal that had none,
        # and a connection failure is never presented as an upstream answer.
        assert (suffix in notice.args[2]) is (last_said == "text")
        assert (connection in notice.args[2]) is (last_said == "connection")
        gateway.correlation.settle("turn-detail", settled_by=SETTLED_BY_TERMINAL_RESULT)
        record = service.provenance.get("turn-detail")
        assert [attempt.get("upstream_detail") for attempt in record["failed_attempts"]] == texts[:attempts]
        logged = [entry.getMessage() for entry in caplog.records if "Model Hub attempt failed" in entry.getMessage()]
        status_text = "http_status=None" if last_said == "connection" else "http_status=500"
        assert len(logged) == attempts and all(status_text in line for line in logged)
        assert detail in logged[0]
    finally:
        await gateway.close()


@pytest.mark.parametrize("language", ["en", "zh"])
def test_request_incompatible_copy_preserves_nonfallback_guidance(language):
    detail = "response protection is unavailable"
    projection = TurnOutcomeProjectionInput(
        outcome="failed_terminal",
        discriminator="request_nonfallback",
        upstream_detail=detail,
    )

    text = render_turn_outcome_copy(projection, language)

    assert text == t(
        "modelHub.launch.request_incompatible_detail",
        language,
        detail=detail,
    )
    assert (
        "switching Sources will not help" in text
        if language == "en"
        else "切换模型供应商也无法解决" in text
    )


@pytest.mark.parametrize("language", ["en", "zh"])
def test_http_200_sse_failure_is_presented_as_upstream_error(language):
    detail = "response protection is unavailable"
    projection = TurnOutcomeProjectionInput(
        outcome="exhausted",
        discriminator="final_supply_state",
        supply_facts=TurnSupplyFacts(
            backend="codex",
            model="gpt-6-astra",
            supply_state="waiting",
            source="Relay 服务",
        ),
        last_upstream_failure=TurnUpstreamFailure(
            source="Relay 服务",
            http_status=None,
            detail=detail,
        ),
    )

    text = render_turn_outcome_copy(projection, language)

    assert t(
        "modelHub.launch.last_upstream_failure_without_status",
        language,
        source="Relay 服务",
        detail=detail,
    ) in text
    assert t(
        "modelHub.launch.last_connection_failure",
        language,
        source="Relay 服务",
        detail=detail,
    ) not in text
    assert "HTTP 200" not in text
    assert (
        "switching Sources will not help" not in text
        if language == "en"
        else "切换模型供应商也无法解决" not in text
    )
    assert detail in text


@pytest.mark.parametrize("language", ["en", "zh"])
def test_terminal_copy_never_promises_automatic_recovery(language):
    """MH-RETRY-COPY-001: final failures never promise that a timer will recover a Source."""
    params = {"model": "test", "source": "relay", "retry_at": "2026-09-09T04:00:00Z"}
    for key in ("modelHub.launch.waiting", "modelHub.launch.waiting_without_retry"):
        text = t(key, language, **params)
        assert ("已结束" if language == "zh" else "has ended") in text
        assert "2026-09-09" not in text
        assert "自动恢复。" not in text
        assert "has recovered" not in text


def _next_step_controller(language: str, *, public_url: str | None):
    cloud = SimpleNamespace(enabled=public_url is not None, public_url=public_url or "")
    return SimpleNamespace(
        config=SimpleNamespace(language=language, remote_access=SimpleNamespace(vibe_cloud=cloud)),
        emit_agent_message=AsyncMock(return_value="msg-failed"),
    )


_WORKBENCH = MessageContext(user_id="local", channel_id="proj-1", platform="avibe", platform_specific={})
_IM = MessageContext(user_id="U1", channel_id="C1", platform="slack", platform_specific={})
# A scheduled or Harness Turn in a Workbench Session whose notices go to an IM channel.
_ROUTED_TO_IM = MessageContext(
    user_id="local", channel_id="proj-1", platform="avibe",
    platform_specific={"delivery_override": {"platform": "slack", "channel_id": "C1"}},
)


def _notice(controller):
    """The visible text and metadata of the notice a failure emitted."""
    notify = controller.emit_agent_message.await_args_list[0]
    return notify.args[2], notify.kwargs["output"].metadata


@pytest.mark.parametrize("language", ["en", "zh"])
@pytest.mark.parametrize(
    ("context", "public_url", "line_key", "url"),
    [
        # The Workbench draws notices as plain text: the step rides as data, and the body stays the refusal.
        (_WORKBENCH, "https://team.avibe.bot", None, None),
        # No IM identity maps to a Workbench principal: IM says who can fix it, with the public URL when there is one.
        (_IM, "https://team.avibe.bot", "errors.modelsNextStepImUrl", "https://team.avibe.bot/settings/models"),
        (_IM, None, "errors.modelsNextStepIm", None),
        # The form follows where the notice goes, not the Session it came from.
        (_ROUTED_TO_IM, "https://team.avibe.bot", "errors.modelsNextStepImUrl", "https://team.avibe.bot/settings/models"),
    ],
)
async def test_a_refusal_fixed_in_models_ends_with_where_to_fix_it(language, context, public_url, line_key, url):
    from modules.agents.model_hub import ModelSupplyRefusal

    controller = _next_step_controller(language, public_url=public_url)
    refusal = ModelSupplyRefusal("Vibey has no model source yet.", fixed_in_models=True)

    await emit_backend_failure(controller, context, "vibey", str(refusal), display_text=f"❌ {refusal}", cause=refusal)

    text, metadata = _notice(controller)
    if line_key is None:
        assert (text, metadata.get("failure_next_step")) == (f"❌ {refusal}", "models")
        assert "](" not in text
    else:
        line = t(line_key, language, url=url) if url else t(line_key, language)
        assert (text, metadata.get("failure_next_step")) == (f"❌ {refusal}\n{line}", None)


@pytest.mark.parametrize("context", [_WORKBENCH, _IM])
@pytest.mark.parametrize(
    "cause",
    [
        # The operator switched the Model Hub off, which takes Models away too.
        "hub_disabled",
        # Not a Model Hub refusal at all.
        "native",
    ],
)
async def test_a_failure_models_cannot_fix_gets_no_next_step(context, cause):
    from modules.agents.model_hub import ModelSupplyRefusal

    controller = _next_step_controller("en", public_url="https://team.avibe.bot")
    error = (
        ModelSupplyRefusal("Model Hub is disabled on this instance.", fixed_in_models=False)
        if cause == "hub_disabled" else RuntimeError("process exited")
    )

    await emit_backend_failure(controller, context, "vibey", str(error), display_text=f"❌ {error}", cause=error)

    text, metadata = _notice(controller)
    assert (text, metadata.get("failure_next_step")) == (f"❌ {error}", None)


@pytest.mark.parametrize("context", [_WORKBENCH, _IM])
async def test_a_restart_replaces_the_refusal_and_its_next_step(context):
    from modules.agents.model_hub import ModelSupplyRefusal

    controller = _next_step_controller("en", public_url="https://team.avibe.bot")
    controller.shutdown_requested = True
    refusal = ModelSupplyRefusal("Vibey has no model source yet.", fixed_in_models=True)

    await emit_backend_failure(controller, context, "vibey", str(refusal), display_text=f"❌ {refusal}", cause=refusal)

    text, metadata = _notice(controller)
    assert (text, metadata.get("failure_next_step")) == (t("turn.interrupted.serviceRestart", "en"), None)


@pytest.mark.parametrize(
    ("discriminator", "supply_state", "pointed"),
    [
        # Copy whose fix is in Models carries the way there: it sends the reader there, or names a model
        # list Models keeps, as when a person removed the Agent's model, even the list's last one...
        ("route_unconfigured", "interrupted", True),
        ("blocked_supply_state", "interrupted", True),
        ("model_unlisted", "interrupted", True),
        # ...and copy that does not, such as a source still cooling down, does not.
        ("blocked_supply_state", "waiting", False),
    ],
)
async def test_a_route_refusal_points_to_models_only_when_its_copy_sends_the_reader_there(
    discriminator, supply_state, pointed,
):
    from core.handlers.model_hub.service import ModelHubError

    outcome = TurnOutcomeProjectionInput(
        outcome="no_candidate",
        discriminator=discriminator,
        supply_facts=TurnSupplyFacts(
            backend="claude", model="claude-opus-5-5", supply_state=supply_state,
            retry_at="2026-10-12T08:00:00+00:00",
        ),
    )
    refused = ModelHubError("no_candidate", turn_outcome=outcome)
    controller = _next_step_controller("en", public_url=None)
    copy = render_turn_outcome_copy(outcome, "en")

    await emit_backend_failure(controller, _WORKBENCH, "claude", str(refused), display_text=copy, cause=refused)

    text, metadata = _notice(controller)
    assert (text, metadata.get("failure_next_step")) == (copy, "models" if pointed else None)