Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

21.9 KB
Raw Blame
"""Shared representation for authoritative backend terminal failures."""

from __future__ import annotations

import uuid
from dataclasses import replace
from inspect import Parameter, signature
from typing import Any

from core.agent_output import AgentError, TurnEnd, TurnOutcome, emit_output
from core.delivery_evidence import (
    ACK_EVIDENCE_DELIVERY_ONLY,
    ACK_EVIDENCE_RECEIPT,
    DeliveryEvidence,
)
from core.delivery_target import routed_delivery_context
from core.message_output import MessageOutput, terminal_output_for, terminal_turn_output
from vibe.i18n import t as i18n_t
from vibe.message_types import spec_for

BACKEND_FAILURE_EVENT = "backend_failure"
#: ``failure_next_step`` of a notice whose Agent cannot run (disabled, deleted, or its
#: backend unavailable): the Workbench offers to move the session to Vibey and run the
#: failed input there (``continue_on_backend`` on the notice's Retry).
CONTINUE_ON_VIBEY_NEXT_STEP = "continue_on_vibey"


def output_with_next_step(request: Any, next_step: str) -> MessageOutput:
    """The terminal output of a failure notice that names ``next_step`` as data."""

    terminal = terminal_output_for(request) if request is not None else terminal_turn_output()
    return replace(terminal, metadata={**terminal.metadata, "failure_next_step": next_step})


def is_backend_failure_notification(message_type: str | None, metadata: Any) -> bool:
    """Whether a persisted message is the visible half of a terminal failure."""

    normalized_type = str(message_type or "").strip()
    event = metadata.get("event") if isinstance(metadata, dict) else None
    return (
        event == BACKEND_FAILURE_EVENT
        and event in spec_for(normalized_type)["terminalWhenEvents"]
    )


def _terminal_output(request: Any, output: MessageOutput | None) -> MessageOutput:
    if output is not None:
        return output
    if request is not None:
        return terminal_output_for(request)
    return terminal_turn_output()


def harness_run_identity(context: Any, request: Any) -> str:
    """The Run id proving that this Turn was entered through Harness, if any."""

    for source in (getattr(request, "context", None), context):
        payload = getattr(source, "platform_specific", None) or {}
        identity = str(payload.get("task_execution_id") or "").strip()
        if identity:
            return identity
        accepted = payload.get("accepted_agent_run_ids")
        if isinstance(accepted, list):
            for value in accepted:
                identity = str(value or "").strip()
                if identity:
                    return identity
    return ""


def _turn_failure_identity(context: Any, request: Any) -> str:
    """Stable user-visible failure identity shared by every Run in one Turn."""

    for source in (getattr(request, "context", None), context):
        payload = getattr(source, "platform_specific", None) or {}
        identity = str(payload.get("turn_token") or "").strip()
        if identity:
            return f"turn:{identity}"
    return ""


def _failure_identity(
    context: Any,
    request: Any,
    explicit_id: str | None,
    *,
    authoritative: bool = False,
) -> str:
    identity = str(explicit_id or "").strip()
    # An AUTHORITATIVE id is one the caller derived from the durable run row — the
    # owed-notice drain's. It wins outright, which is what keeps a D1 interruption
    # notice (``interrupt:{run}:{reason}``) from collapsing into the ordinary
    # notice for the same execution.
    if identity and authoritative:
        return identity

    # A Harness failure is user-visible once per TURN, whichever backend reported
    # it and however many Runs that Turn accepted. Codex and OpenCode pass their
    # own turn/message/session ids at five call sites, so the durable turn token is
    # the shared identity both the live notification and every linked Run can carry.
    # Legacy callers without a turn token still fall back to the Harness Run id.
    #
    # Non-harness contexts have no run to align to, so a backend's explicit id
    # still wins there and that path is unchanged.
    harness_identity = harness_run_identity(context, request)
    if harness_identity:
        return _turn_failure_identity(context, request) or harness_identity

    if identity:
        return identity

    for source in (getattr(request, "context", None), context):
        payload = getattr(source, "platform_specific", None) or {}
        for key in (
            "task_execution_id",
            "turn_token",
            "agent_runtime_turn_token",
        ):
            identity = str(payload.get(key) or "").strip()
            if identity:
                return identity

    if request is not None:
        identity = str(getattr(request, "_backend_failure_id", "") or "").strip()
        if not identity:
            identity = uuid.uuid4().hex
            setattr(request, "_backend_failure_id", identity)
        return identity
    return uuid.uuid4().hex


def _failure_texts(backend_name: str, diagnostic: Any, display_text: Any) -> tuple[str, str]:
    """The diagnostic and the user-visible body, for the live path and the replay.

    Shared so a replayed notice cannot drift from the notice the live path would
    have shown for the same failure: same fallback when a backend supplies no
    display text, same fallback when it supplies no diagnostic either.
    """

    error = str(diagnostic or "").strip() or f"{backend_name} backend failed"
    visible = str(display_text or "").strip() or error
    return error, visible


def _controller_language(controller: Any) -> str:
    return str(getattr(getattr(controller, "config", None), "language", "en") or "en")


def _model_hub_failure(
    controller: Any, context: Any, request: Any, backend: str
) -> tuple[str | None, str | None, Any] | None:
    """Render exact live Hub evidence, never parse a native exception wrapper.

    The registry owns ambiguity, pending-request, and Stop guards. Without that
    evidence the original backend error remains authoritative; an earlier source
    outage must not disguise an unrelated native/tool failure.
    """

    from core.handlers.model_hub.provenance import (
        TurnOutcomeProjectionInput,
        render_turn_outcome_copy,
    )

    gateway = getattr(controller, "model_hub_turn_gateway", None)
    registry = getattr(gateway, "correlation", None)
    project = getattr(registry, "terminal_projection", None)
    if not callable(project):
        return None
    for source in (getattr(request, "context", None), context):
        payload = getattr(source, "platform_specific", None) or {}
        turn_id = payload.get("turn_token")
        if not isinstance(turn_id, str) or not turn_id.strip():
            continue
        projection = project(turn_id.strip(), backend=backend)
        if not isinstance(projection, TurnOutcomeProjectionInput):
            continue
        return (
            render_turn_outcome_copy(projection, _controller_language(controller)),
            projection.local_error_detail,
            projection,
        )
    return None


def backend_failure_notification_output(
    context: Any,
    backend: str,
    *,
    request: Any = None,
    output: MessageOutput | None = None,
    failure_id: str | None = None,
    failure_id_authoritative: bool = False,
) -> MessageOutput:
    """Build the durable, non-settling visible half of a backend failure."""

    backend_name = str(backend or "backend").strip() or "backend"
    terminal = _terminal_output(request, output)
    identity = _failure_identity(
        context, request, failure_id, authoritative=failure_id_authoritative
    )
    metadata = dict(terminal.metadata)
    metadata.update(
        {
            "backend": backend_name,
            "event": BACKEND_FAILURE_EVENT,
            "failure_id": identity,
        }
    )
    return MessageOutput(
        completes_turn=False,
        completes_run=False,
        detached=terminal.detached,
        idempotency_key=f"backend-failure:{identity}",
        activity_id=terminal.activity_id,
        causation_id=terminal.causation_id,
        run_id=terminal.run_id,
        metadata=metadata,
    )


def _acknowledges_target(
    context: Any,
    evidence: str | None,
) -> bool:
    if evidence == ACK_EVIDENCE_RECEIPT:
        return True
    target_platform = getattr(routed_delivery_context(context), "platform", None)
    return (
        evidence == ACK_EVIDENCE_DELIVERY_ONLY
        and bool(target_platform)
        and target_platform != "avibe"
    )


def terminal_backend_failure_output(
    context: Any,
    *,
    request: Any = None,
    output: MessageOutput | None = None,
    failure_id: str | None = None,
    delivery: DeliveryEvidence | None = None,
) -> MessageOutput:
    """Attach one monotonic Turn failure-delivery contract to a terminal output.

    Some failures use the ordinary ``notify`` dispatcher while others need a
    specialized visible path, such as an OAuth recovery button. Both must settle
    with the same evidence contract or a later Harness Run cannot distinguish an
    error the user already saw from an error that still needs a fallback notice.
    """

    terminal = _terminal_output(request, output)
    if not (_turn_failure_identity(context, request) or harness_run_identity(context, request)):
        return terminal

    metadata = dict(terminal.metadata)
    existing = metadata.get("turn_failure_notification")
    existing_notification = dict(existing) if isinstance(existing, dict) else {}
    local_detail = metadata.pop("local_error_detail", None)
    if local_detail:
        # The failure-delivery contract is already carried through immediate,
        # deferred, and late-attached Run settlement. Keep its snapshot there.
        existing_notification["local_error_detail"] = local_detail
    identity = str(existing_notification.get("failure_id") or "").strip()
    if not identity:
        identity = _failure_identity(context, request, failure_id)

    incoming_ack = delivery.ack_evidence if delivery is not None else None
    existing_ack = str(existing_notification.get("ack_evidence") or "").strip() or None
    ack_priority = {None: 0, ACK_EVIDENCE_DELIVERY_ONLY: 1, ACK_EVIDENCE_RECEIPT: 2}
    ack_evidence = (
        incoming_ack
        if ack_priority.get(incoming_ack, 1 if incoming_ack else 0)
        > ack_priority.get(existing_ack, 1 if existing_ack else 0)
        else existing_ack
    )
    existing_notification.update(
        {
            "failure_id": identity,
            "ack_evidence": ack_evidence,
            "delivered": bool(existing_notification.get("delivered"))
            or _acknowledges_target(context, ack_evidence),
        }
    )
    metadata["turn_failure_notification"] = existing_notification
    return replace(terminal, metadata=metadata)


async def emit_replayed_backend_failure(
    controller: Any,
    context: Any,
    backend: str,
    diagnostic: str,
    *,
    failure_id: str,
    turn_id: str | None = None,
    local_error_detail: str | None = None,
    display_text: str | None = None,
    delivery: DeliveryEvidence | None = None,
) -> None:
    """Deliver the visible half of a failure that ALREADY ended. Settles nothing.

    The owed-failure-notice drain replays a failure it read back from a durable
    run row, hours after the run itself ended. That is a different act from
    reporting a failure as it happens, and it used to be expressed as the live
    ``emit_backend_failure`` with its live behaviour switched off piece by piece:
    a non-settling ``output``, an auth-recovery bypass, an explicit identity. Four
    review rounds found settlement, auth and identity defects in the gaps between
    those switches, because the replay still entered the live lifecycle and every
    new live behaviour had to be neutralized again by hand.

    So the replay is its own emitter, and the neutralizations become facts of its
    construction rather than arguments:

    * ONE ``notify``. No ``result``, therefore no turn settlement, no status-bubble
      teardown, no runtime-turn release. A replayed context describes a run that
      ended long ago and carries no runtime-turn token, and
      ``emit_matches_runtime_turn`` fails OPEN for a tokenless context —
      deliberately, so a scheduled or watch run can settle its own turn — so a
      settling replay was adopted by whatever turn happened to be live on the
      target channel and finalized it. The run this notice is about is terminal by
      construction: a notice is owed only once the row is settled.
    * NO auth recovery. This is a report about a possibly hours-old 401, not an
      interactive "reset OAuth" affordance about the backend's state right now, and
      ``maybe_emit_auth_recovery_message`` cannot report into ``DeliveryEvidence``
      anyway, so a notice delivered through it would read as unacknowledged and walk
      on to the next delivery rung.
    * The identity is AUTHORITATIVE, with no way to say otherwise. It comes from the
      durable notice row, so two drain passes over that row produce one identity —
      and an interruption's ``interrupt:{run}:{reason}`` survives instead of
      collapsing into the ordinary notice for the same execution, which is what
      happens when a context-derived run id is allowed to win.

    ``delivery`` is filled in with what the notify attempt actually proved; the
    drain acks its durable notice on that evidence, never on a clean return.

    ``turn_id`` is the durable notice's failed Turn, supplied only when the
    delivery target is that Run's own Session. It is message provenance, never a
    live context token. Unlinked and cross-Session reports remain informational;
    all replays stay outside foreground Activity even when Retry is available.
    """

    backend_name = str(backend or "backend").strip() or "backend"
    _error, visible = _failure_texts(backend_name, diagnostic, display_text)
    # Non-settling by construction: ``backend_failure_notification_output`` hardcodes
    # ``completes_turn=False`` / ``completes_run=False``, and it is the only output
    # this function emits.
    notification = backend_failure_notification_output(
        context,
        backend_name,
        failure_id=failure_id,
        failure_id_authoritative=True,
        output=MessageOutput(
            metadata={
                "turn_id": str(turn_id or "").strip() or None,
                "replayed": True,
                **({"local_error_detail": local_error_detail} if local_error_detail else {}),
            },
        ),
    )
    # ``delivery`` is passed ONLY when a caller asked for it: controller-like objects
    # that implement ``emit_agent_message`` without the keyword must keep working.
    await emit_output(controller, context, AgentError(visible, output=notification, delivery=delivery))


async def emit_backend_failure(
    controller: Any,
    context: Any,
    backend: str,
    diagnostic: str,
    *,
    display_text: str | None = None,
    request: Any = None,
    output: MessageOutput | None = None,
    failure_id: str | None = None,
    delivery: DeliveryEvidence | None = None,
    cause: BaseException | None = None,
) -> bool:
    """Notify and settle one terminal backend failure.

    Backend adapters own structured failure recognition. This helper owns the
    shared representation and keeps visible delivery separate from lifecycle
    settlement. A live Agent Turn is the primary owner of its user-visible error,
    including a Turn entered through Harness. Linked Runs retain durable owed notices
    only as one Turn-scoped fallback when this notification cannot be acknowledged.
    Non-Harness callers retain immediate auth recovery. The return value is true when
    auth recovery supplied that immediate notification.

    ``cause`` is the exception the failure reports, when the adapter has one. A
    Model Hub refusal names its own cause, so it never reaches the text auth
    heuristic, which would misread a model id that says ``oauth``; when its fix
    is in Models, the notice ends with where to make it.

    ``delivery``, when supplied, is filled in with what the notify attempt actually
    proved. A durable Turn or legacy Harness Run supplies an evidence object
    automatically so Runs attached after settlement inherit the same proof. A clean
    return alone is not enough to distinguish a delivered message from a lost one.
    """

    from core.message_context import resolve_context_platform
    from modules.agents.model_hub import (
        MODELS_NEXT_STEP,
        is_model_hub_refusal,
        models_next_step_text,
        refusal_fixed_in_models,
    )

    backend_name = str(backend or "backend").strip() or "backend"
    error, visible = _failure_texts(backend_name, diagnostic, display_text)
    hub_failure = _model_hub_failure(controller, context, request, backend_name)
    if hub_failure is not None and hub_failure[0] is not None:
        visible = hub_failure[0]
    terminal = _terminal_output(request, output)
    if hub_failure is not None and hub_failure[1] and not getattr(controller, "shutdown_requested", False):
        terminal = replace(
            terminal, metadata={**terminal.metadata, "local_error_detail": hub_failure[1]},
        )
    # A Model Hub refusal whose fix is in Models ends with where to make it. The Workbench
    # draws notices as plain text, so it gets the step as data and renders its own link;
    # IM gets the line itself. The form follows where the notice goes, after a scheduled or
    # Harness override; a restart's copy below replaces the refusal, and its step with it.
    if getattr(controller, "shutdown_requested", False) is not True and refusal_fixed_in_models(
        cause=cause,
        outcome=hub_failure[2] if hub_failure is not None and hub_failure[0] is not None else None,
    ):
        platform = resolve_context_platform(
            routed_delivery_context(context), fallback_platform=getattr(controller, "primary_platform", None),
        )
        if platform == "avibe":
            terminal = replace(terminal, metadata={**terminal.metadata, "failure_next_step": MODELS_NEXT_STEP})
        else:
            visible = f"{visible}\n{models_next_step_text(controller)}"
    harness_run_id = harness_run_identity(context, request)
    owns_failure_contract = bool(
        _turn_failure_identity(context, request) or harness_run_id
    )
    notification = backend_failure_notification_output(
        context,
        backend_name,
        request=request,
        output=terminal,
        failure_id=failure_id,
    )
    notification_identity = str(notification.metadata.get("failure_id") or "").strip()
    live_delivery = delivery
    if owns_failure_contract and live_delivery is None:
        live_delivery = DeliveryEvidence()

    async def settle_terminal_failure() -> None:
        terminal_output = terminal_backend_failure_output(
            context,
            request=request,
            output=terminal,
            failure_id=notification_identity,
            delivery=live_delivery,
        )
        await emit_output(
            controller,
            context,
            TurnEnd(outcome=TurnOutcome.FAILED, silent=True, terminal_error=error, output=terminal_output),
        )

    # Auth recovery is unconditional for non-Harness live callers, and there is no
    # switch to turn it off. A real interactive 401 should offer its reset-OAuth
    # affordance immediately; a Harness Turn uses the ordinary backend failure
    # notification so the Run fallback remains backend-neutral.
    #
    # The one caller that must not do that — the owed-notice drain, replaying a
    # possibly hours-old failure it read back from a durable row — does not call this
    # function at all; see ``emit_replayed_backend_failure``. A bypass argument here
    # was the previous shape, and it left the replay inside this lifecycle with its
    # live behaviours switched off one at a time.
    auth_service = getattr(controller, "agent_auth_service", None)
    maybe_recover = getattr(auth_service, "maybe_emit_auth_recovery_message", None)
    if not harness_run_id and callable(maybe_recover) and not is_model_hub_refusal(cause):
        try:
            handled_auth = await maybe_recover(
                context,
                backend_name,
                visible,
                output=terminal,
                terminal_error=error,
            )
        except Exception:
            await settle_terminal_failure()
            raise
        if handled_auth:
            return True

    # Shutdown stops the Model Hub engine and backend processes before in-flight
    # Turns end, so a failure reported after it began names a symptom (gateway
    # down, process exited) instead of the cause. Say the restart interrupted it.
    if getattr(controller, "shutdown_requested", False) is True:
        visible = i18n_t("turn.interrupted.serviceRestart", _controller_language(controller))

    # ``delivery`` is passed only when the emitter supports it. Durable Turn and
    # Harness contexts create the object automatically, but controller-like test
    # and integration doubles with the legacy signature must keep working.
    accepts_delivery = False
    if live_delivery is not None:
        emit = controller.emit_agent_message
        try:
            parameters = signature(emit).parameters.values()
            accepts_delivery = any(
                parameter.name == "delivery" or parameter.kind is Parameter.VAR_KEYWORD
                for parameter in parameters
            )
        except (TypeError, ValueError):
            accepts_delivery = False
    try:
        receipt = await emit_output(
            controller,
            context,
            AgentError(visible, output=notification, delivery=live_delivery if accepts_delivery else None),
        )
        if (
            live_delivery is not None
            and live_delivery.delivered_id is None
            and receipt.delivered
            and not bool((context.platform_specific or {}).get("suppress_delivery"))
        ):
            live_delivery.delivered_id = str(receipt.message_id)
            live_delivery.send_returned = True
    finally:
        await settle_terminal_failure()
    return False