Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat(dify-agent): propagate run limit error types (#40116)

盐粒 Yanli12300f383308
diff --git a/api/clients/agent_backend/errors.py b/api/clients/agent_backend/errors.py
index b48c4d08da1..99ef2fecf5d 100644
--- a/api/clients/agent_backend/errors.py
+++ b/api/clients/agent_backend/errors.py
@@ -10,6 +10,8 @@ from __future__ import annotations
from typing import Any
+from dify_agent.protocol import RunFailureType
+
class AgentBackendError(Exception):
"""Base error for API-side Agent backend integration failures."""
@@ -54,6 +56,7 @@ class AgentBackendRunFailedError(AgentBackendError):
run_id: str
detail: Any
+ error_type: RunFailureType | None
reason: str | None
source_event_id: str | None
@@ -63,11 +66,13 @@ class AgentBackendRunFailedError(AgentBackendError):
detail: Any,
*,
message: str | None = None,
+ error_type: RunFailureType | None = None,
reason: str | None = None,
source_event_id: str | None = None,
) -> None:
self.run_id = run_id
self.detail = detail
+ self.error_type = error_type
self.reason = reason
self.source_event_id = source_event_id
display_message = message or f"Agent backend run failed: {run_id}"
diff --git a/api/clients/agent_backend/event_adapter.py b/api/clients/agent_backend/event_adapter.py
index 8fdc165ab3c..e72d2fbfb51 100644
--- a/api/clients/agent_backend/event_adapter.py
+++ b/api/clients/agent_backend/event_adapter.py
@@ -22,6 +22,7 @@ from dify_agent.protocol import (
RunCancelledEvent,
RunEvent,
RunFailedEvent,
+ RunFailureType,
RunStartedEvent,
RunSucceededEvent,
)
@@ -96,6 +97,7 @@ class AgentBackendRunFailedInternalEvent(AgentBackendInternalEventBase):
type: Literal[AgentBackendInternalEventType.RUN_FAILED] = AgentBackendInternalEventType.RUN_FAILED
error: str
+ error_type: RunFailureType | None = None
reason: str | None = None
@@ -180,6 +182,7 @@ class AgentBackendRunEventAdapter:
run_id=event.run_id,
source_event_id=event.id,
error=event.data.error,
+ error_type=event.data.error_type,
reason=event.data.reason,
)
]
diff --git a/api/core/app/apps/agent_app/app_runner.py b/api/core/app/apps/agent_app/app_runner.py
index 30c5d9a5395..da1008ddfe8 100644
--- a/api/core/app/apps/agent_app/app_runner.py
+++ b/api/core/app/apps/agent_app/app_runner.py
@@ -89,9 +89,10 @@ _AGENT_BACKEND_INVOKE_ERROR_BY_REASON: Mapping[str, type[InvokeError]] = {
def _agent_backend_failure_to_exception(event: AgentBackendRunFailedInternalEvent) -> Exception:
- err_cls = _AGENT_BACKEND_INVOKE_ERROR_BY_REASON.get(event.reason or "")
- if err_cls is not None:
- return err_cls(event.error)
+ if event.error_type is None:
+ err_cls = _AGENT_BACKEND_INVOKE_ERROR_BY_REASON.get(event.reason or "")
+ if err_cls is not None:
+ return err_cls(event.error)
message = event.error or "Agent backend run did not complete successfully."
return AgentBackendRunFailedError(
event.run_id,
@@ -101,6 +102,7 @@ def _agent_backend_failure_to_exception(event: AgentBackendRunFailedInternalEven
"source_event_id": event.source_event_id,
},
message=message,
+ error_type=event.error_type,
reason=event.reason,
source_event_id=event.source_event_id,
)
@@ -701,7 +703,7 @@ class AgentAppRunner:
if not isinstance(terminal, AgentBackendRunSucceededInternalEvent):
if isinstance(terminal, AgentBackendRunFailedInternalEvent):
reason = terminal.reason
- if reason == "binding_lost":
+ if terminal.error_type is None and reason == "binding_lost":
raise AgentBackendError("The retained agent working environment is no longer available.")
raise _agent_backend_failure_to_exception(terminal)
raise AgentBackendError("Agent backend run did not complete successfully.")
diff --git a/api/core/app/apps/base_app_generate_response_converter.py b/api/core/app/apps/base_app_generate_response_converter.py
index d30969d3714..aef54cc049e 100644
--- a/api/core/app/apps/base_app_generate_response_converter.py
+++ b/api/core/app/apps/base_app_generate_response_converter.py
@@ -3,9 +3,10 @@ from abc import ABC, abstractmethod
from collections.abc import Generator, Mapping
from typing import Any, Union, cast
+from dify_agent.protocol import RunFailureType
from pydantic import JsonValue
-from clients.agent_backend.errors import AgentBackendError
+from clients.agent_backend.errors import AgentBackendError, AgentBackendRunFailedError
from core.app.entities.app_invoke_entities import InvokeFrom
from core.app.entities.task_entities import AppBlockingResponse, AppStreamResponse
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
@@ -117,6 +118,13 @@ class AppGenerateResponseConverter[TBlockingResponse: AppBlockingResponse](ABC):
:param e: exception
:return:
"""
+ if isinstance(e, AgentBackendRunFailedError) and e.error_type == RunFailureType.AGENT_RUN_LIMIT_EXCEEDED:
+ return {
+ "code": RunFailureType.AGENT_RUN_LIMIT_EXCEEDED.value,
+ "status": 400,
+ "message": str(e),
+ }
+
error_responses: dict[type[Exception], dict[str, JsonValue]] = {
ValueError: {"code": "invalid_param", "status": 400},
ProviderTokenNotInitError: {"code": "provider_not_initialize", "status": 400},
diff --git a/api/core/workflow/nodes/agent_v2/output_adapter.py b/api/core/workflow/nodes/agent_v2/output_adapter.py
index b5ac93df6ae..f2d408a7999 100644
--- a/api/core/workflow/nodes/agent_v2/output_adapter.py
+++ b/api/core/workflow/nodes/agent_v2/output_adapter.py
@@ -98,7 +98,7 @@ class WorkflowAgentOutputAdapter:
match event:
case AgentBackendRunFailedInternalEvent():
error = event.error
- error_type = event.reason or "agent_backend_run_failed"
+ error_type = event.error_type or event.reason or "agent_backend_run_failed"
terminal_status = "failed"
case AgentBackendRunCancelledInternalEvent():
error = event.message or "Agent backend run was cancelled."
diff --git a/api/tests/unit_tests/clients/agent_backend/test_event_adapter.py b/api/tests/unit_tests/clients/agent_backend/test_event_adapter.py
index 14c51895df0..961e40fec7c 100644
--- a/api/tests/unit_tests/clients/agent_backend/test_event_adapter.py
+++ b/api/tests/unit_tests/clients/agent_backend/test_event_adapter.py
@@ -10,6 +10,7 @@ from dify_agent.protocol import (
RunCancelledEventData,
RunFailedEvent,
RunFailedEventData,
+ RunFailureType,
RunStartedEvent,
RunSucceededEvent,
RunSucceededEventData,
@@ -133,7 +134,11 @@ def test_event_adapter_maps_run_failed_to_failed_result():
RunFailedEvent(
id="4-0",
run_id="run-1",
- data=RunFailedEventData(error="boom", reason="runtime"),
+ data=RunFailedEventData(
+ error="boom",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ reason="runtime",
+ ),
)
)
@@ -142,6 +147,7 @@ def test_event_adapter_maps_run_failed_to_failed_result():
run_id="run-1",
source_event_id="4-0",
error="boom",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
reason="runtime",
)
]
diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py
index 63c198b2c43..26e44377e3a 100644
--- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py
+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_runner.py
@@ -19,6 +19,9 @@ from dify_agent.protocol import (
CancelRunResponse,
PydanticAIStreamRunEvent,
RunEvent,
+ RunFailedEvent,
+ RunFailedEventData,
+ RunFailureType,
RunStartedEvent,
RunSucceededEvent,
RunSucceededEventData,
@@ -117,6 +120,30 @@ class _RecordingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
return super().cancel_run(run_id, request=request)
+class _RunLimitBindingLostFakeAgentBackendRunClient(FakeAgentBackendRunClient):
+ @override
+ def stream_events(
+ self,
+ run_id: str,
+ *,
+ after: str | None = None,
+ should_stop: Callable[[], bool] | None = None,
+ ) -> Iterator[RunEvent]:
+ del after, should_stop
+ created_at = datetime(2026, 1, 1, tzinfo=UTC)
+ yield RunStartedEvent(id="1-0", run_id=run_id, created_at=created_at)
+ yield RunFailedEvent(
+ id="2-0",
+ run_id=run_id,
+ created_at=created_at,
+ data=RunFailedEventData(
+ error="run limit reached",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ reason="binding_lost",
+ ),
+ )
+
+
class _StreamingFakeAgentBackendRunClient(FakeAgentBackendRunClient):
@override
def stream_events(
@@ -1142,6 +1169,17 @@ def test_failed_run_raises_agent_backend_error() -> None:
assert store.saved == []
+def test_failed_run_prefers_run_failure_type_over_binding_lost_reason() -> None:
+ client = _RunLimitBindingLostFakeAgentBackendRunClient()
+ store = _FakeSessionStore()
+
+ with pytest.raises(AgentBackendRunFailedError) as raised:
+ _run(_runner(client, store), _FakeQueueManager())
+
+ assert raised.value.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+ assert raised.value.reason == "binding_lost"
+
+
def test_agent_backend_failure_to_exception_maps_rate_limit_reason() -> None:
err = app_runner_module._agent_backend_failure_to_exception(
AgentBackendRunFailedInternalEvent(
@@ -1177,6 +1215,26 @@ def test_agent_backend_failure_to_exception_preserves_unknown_reason_context() -
assert str(err) == "Knowledge retrieval failed (agent_run_id=run-1)"
+def test_agent_backend_failure_to_exception_prefers_run_failure_type_over_known_reason() -> None:
+ err = app_runner_module._agent_backend_failure_to_exception(
+ AgentBackendRunFailedInternalEvent(
+ run_id="run-1",
+ error="run limit reached",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ reason="InvokeRateLimitError",
+ )
+ )
+
+ assert isinstance(err, AgentBackendRunFailedError)
+ assert err.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+ assert err.reason == "InvokeRateLimitError"
+ assert err.detail == {
+ "error": "run limit reached",
+ "reason": "InvokeRateLimitError",
+ "source_event_id": None,
+ }
+
+
def test_stopped_task_cancels_agent_backend_run_and_skips_session_save() -> None:
client = _RecordingFakeAgentBackendRunClient()
store = _FakeSessionStore()
diff --git a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py
index 527d2c76cfa..18c2fdd97a2 100644
--- a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py
+++ b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py
@@ -3,6 +3,7 @@ from types import SimpleNamespace
from unittest.mock import Mock
import pytest
+from dify_agent.protocol import RunFailureType
from sqlalchemy.orm import Session
from clients.agent_backend.errors import AgentBackendRunFailedError
@@ -151,6 +152,22 @@ class TestBasedGenerateTaskPipeline:
"message": "Knowledge retrieval failed (agent_run_id=run-1)",
}
+ def test_stream_converter_maps_agent_run_limit_error(self):
+ data = AppGenerateResponseConverter._error_to_stream_response(
+ AgentBackendRunFailedError(
+ "run-1",
+ {},
+ message="run limit reached",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ )
+ )
+
+ assert data == {
+ "code": "agent_run_limit_exceeded",
+ "status": 400,
+ "message": "run limit reached (agent_run_id=run-1)",
+ }
+
def test_handle_output_moderation_when_flagged(self, pipeline):
handler = Mock()
handler.moderation_completion.return_value = ("filtered", True)
diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_adapter.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_adapter.py
index 760786c5b75..33f41a5ea3d 100644
--- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_adapter.py
+++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_output_adapter.py
@@ -3,6 +3,7 @@ from unittest.mock import patch
import pytest
from agenton.compositor import CompositorSessionSnapshot
+from dify_agent.protocol import RunFailureType
from clients.agent_backend import (
AgentBackendRunCancelledInternalEvent,
@@ -162,6 +163,38 @@ def test_failure_output_adapter_preserves_backend_failed_reason():
assert result.error_type == "validation"
+def test_failure_output_adapter_prefers_run_failure_type_over_reason():
+ result = WorkflowAgentOutputAdapter().build_failure_result(
+ event=AgentBackendRunFailedInternalEvent(
+ run_id="run-1",
+ error="run limit reached",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ reason="runtime",
+ ),
+ inputs={},
+ process_data={},
+ metadata={},
+ )
+
+ assert result.error_type == "agent_run_limit_exceeded"
+
+
+def test_failure_output_adapter_uses_default_error_type_without_backend_classification():
+ result = WorkflowAgentOutputAdapter().build_failure_result(
+ event=AgentBackendRunFailedInternalEvent(
+ run_id="run-1",
+ error="backend failed",
+ error_type=None,
+ reason=None,
+ ),
+ inputs={},
+ process_data={},
+ metadata={},
+ )
+
+ assert result.error_type == "agent_backend_run_failed"
+
+
def test_success_output_adapter_normalizes_string_and_scalar_outputs():
adapter = WorkflowAgentOutputAdapter()
string_result = adapter.build_success_result(
diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.md
index b79c0629628..2474d3d0d82 100644
--- a/dify-agent/docs/dify-agent/guide/index.md
+++ b/dify-agent/docs/dify-agent/guide/index.md
@@ -335,6 +335,8 @@ progress:
- `POST /runs` creates a running run and schedules it locally.
- `GET /runs/{run_id}` returns `running`, `succeeded`, `failed`, or `cancelled`.
+ Failed records can also expose a stable machine-readable `error_type` alongside
+ the diagnostic `error` text.
- `POST /runs/{run_id}/cancel` atomically accepts cancellation on any API process
and emits `run_cancelled`; it returns `409` only when a success/failure terminal
already won. Runner cleanup continues asynchronously on the owner process.
@@ -357,6 +359,17 @@ terminal `run_succeeded.data` object containing a `CompositorSessionSnapshot` fo
resumption. A successful run has exactly one active result branch: JSON-safe
`output` for final answers, or `deferred_tool_call` when a layer such as
`dify.ask_human` ends the current agent run with an external deferred tool call.
+Failed event payloads contain the diagnostic `error`, optional source-specific
+`reason`, and optional stable `error_type`. Pydantic AI request/step budget
+exhaustion enforced by Dify Agent is reported as
+`error_type: "agent_run_limit_exceeded"`; consumers should branch on that value
+rather than parsing the error text. This type does not classify wall-clock run
+timeouts, whose classification is not implemented in this release, or provider
+and connection timeouts. The matching failed run record and terminal event are
+committed atomically with the same error type. For independently deployed Agent
+backend and API services, deploy consumers that accept the optional field before
+producers begin emitting it because the public protocol models reject unknown
+fields.
## Examples
diff --git a/dify-agent/src/dify_agent/protocol/__init__.py b/dify-agent/src/dify_agent/protocol/__init__.py
index 11f413d428c..32ce269ee3d 100644
--- a/dify-agent/src/dify_agent/protocol/__init__.py
+++ b/dify-agent/src/dify_agent/protocol/__init__.py
@@ -28,6 +28,7 @@ from .schemas import (
RunEventsResponse,
RunFailedEvent,
RunFailedEventData,
+ RunFailureType,
RunLayerSpec,
RunStartedEvent,
RunStatus,
@@ -88,6 +89,7 @@ __all__ = [
"RunEventsResponse",
"RunFailedEvent",
"RunFailedEventData",
+ "RunFailureType",
"RunLayerSpec",
"RunStartedEvent",
"RunStatus",
diff --git a/dify-agent/src/dify_agent/protocol/schemas.py b/dify-agent/src/dify_agent/protocol/schemas.py
index 533c61956ce..ca23f8bfd59 100644
--- a/dify-agent/src/dify_agent/protocol/schemas.py
+++ b/dify-agent/src/dify_agent/protocol/schemas.py
@@ -36,6 +36,7 @@ from __future__ import annotations
from datetime import datetime, timezone
from decimal import Decimal
+from enum import StrEnum
from typing import Annotated, ClassVar, Final, Literal, TypeAlias
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, model_serializer, model_validator
@@ -59,6 +60,16 @@ RunEventType = Literal[
]
+class RunFailureType(StrEnum):
+ """Stable machine-readable categories for failed Dify Agent runs.
+
+ Run-limit failures cover execution budgets enforced by Dify Agent, not
+ provider, connection, or wall-clock timeouts.
+ """
+
+ AGENT_RUN_LIMIT_EXCEEDED = "agent_run_limit_exceeded"
+
+
def utc_now() -> datetime:
"""Return the timezone-aware timestamp format used by public schemas."""
return datetime.now(timezone.utc)
@@ -213,6 +224,7 @@ class RunStatusResponse(BaseModel):
created_at: datetime
updated_at: datetime
error: str | None = None
+ error_type: RunFailureType | None = None
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
@@ -315,6 +327,7 @@ class RunFailedEventData(BaseModel):
"""Terminal failure payload shown to polling and SSE consumers."""
error: str
+ error_type: RunFailureType | None = None
reason: str | None = None
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
@@ -416,6 +429,7 @@ __all__ = [
"RunEventsResponse",
"RunFailedEvent",
"RunFailedEventData",
+ "RunFailureType",
"RunStartedEvent",
"RunStatus",
"RunStatusResponse",
diff --git a/dify-agent/src/dify_agent/runtime/event_sink.py b/dify-agent/src/dify_agent/runtime/event_sink.py
index 0f60b4be2a2..121d58cecde 100644
--- a/dify-agent/src/dify_agent/runtime/event_sink.py
+++ b/dify-agent/src/dify_agent/runtime/event_sink.py
@@ -25,6 +25,7 @@ from dify_agent.protocol.schemas import (
RunCancelledEventData,
RunFailedEvent,
RunFailedEventData,
+ RunFailureType,
RunStartedEvent,
RunStatus,
RunSucceededEvent,
@@ -65,11 +66,13 @@ class InMemoryRunEventSink:
events: dict[str, list[RunEvent]]
statuses: dict[str, RunStatus]
errors: dict[str, str | None]
+ error_types: dict[str, RunFailureType | None]
def __init__(self) -> None:
self.events = defaultdict(list)
self.statuses = {}
self.errors = {}
+ self.error_types = {}
async def append_event(self, event: NonTerminalRunEvent) -> str:
"""Store a non-terminal event and assign a monotonic per-run cursor."""
@@ -84,23 +87,26 @@ class InMemoryRunEventSink:
if current_status != "running":
return RunFinalizationResult(applied=False, status=current_status)
- status, error = terminal_event_status_and_error(event)
+ status, error, error_type = terminal_event_status_fields(event)
event_id = str(len(self.events[event.run_id]) + 1)
self.events[event.run_id].append(event.model_copy(update={"id": event_id}))
self.statuses[event.run_id] = status
self.errors[event.run_id] = error
+ self.error_types[event.run_id] = error_type
return RunFinalizationResult(applied=True, status=status, event_id=event_id)
-def terminal_event_status_and_error(event: TerminalRunEvent) -> tuple[RunStatus, str | None]:
+def terminal_event_status_fields(
+ event: TerminalRunEvent,
+) -> tuple[RunStatus, str | None, RunFailureType | None]:
"""Derive the persisted terminal status fields from one typed event."""
match event:
case RunSucceededEvent():
- return "succeeded", None
+ return "succeeded", None, None
case RunFailedEvent():
- return "failed", event.data.error
+ return "failed", event.data.error, event.data.error_type
case RunCancelledEvent():
- return "cancelled", event.data.message or event.data.reason
+ return "cancelled", event.data.message or event.data.reason, None
async def emit_run_event(
@@ -180,11 +186,16 @@ async def emit_run_failed(
*,
run_id: str,
error: str,
+ error_type: RunFailureType | None = None,
reason: str | None = None,
) -> RunFinalizationResult:
"""Finalize a run with a failed terminal event."""
return await sink.finalize_run(
- RunFailedEvent(run_id=run_id, data=RunFailedEventData(error=error, reason=reason), created_at=utc_now()),
+ RunFailedEvent(
+ run_id=run_id,
+ data=RunFailedEventData(error=error, error_type=error_type, reason=reason),
+ created_at=utc_now(),
+ ),
)
@@ -217,5 +228,5 @@ __all__ = [
"emit_run_failed",
"emit_run_started",
"emit_run_succeeded",
- "terminal_event_status_and_error",
+ "terminal_event_status_fields",
]
diff --git a/dify-agent/src/dify_agent/runtime/runner.py b/dify-agent/src/dify_agent/runtime/runner.py
index 7f16f451ccd..814447871e5 100644
--- a/dify-agent/src/dify_agent/runtime/runner.py
+++ b/dify-agent/src/dify_agent/runtime/runner.py
@@ -36,7 +36,7 @@ from typing import Any, Literal, Protocol, cast, runtime_checkable
import httpx
from graphon.model_runtime.entities.llm_entities import LLMUsage
from pydantic import JsonValue, TypeAdapter
-from pydantic_ai.exceptions import ModelHTTPError
+from pydantic_ai.exceptions import ModelHTTPError, UsageLimitExceeded
from pydantic_ai.messages import AgentStreamEvent, PartDeltaEvent, PartStartEvent, TextPart, TextPartDelta
from pydantic_ai.output import OutputSpec
from pydantic_ai.tools import DeferredToolRequests, DeferredToolResults
@@ -55,6 +55,7 @@ from dify_agent.protocol.schemas import (
CreateRunRequest,
DIFY_AGENT_MODEL_LAYER_ID,
DeferredToolCallPayload,
+ RunFailureType,
normalize_composition,
)
from dify_agent.runtime.agent_factory import create_agent, normalize_user_input
@@ -113,13 +114,16 @@ class AgentRunValidationError(ValueError):
"""Raised when a run request is valid JSON but cannot execute."""
-def _run_failed_error_payload(exc: Exception) -> tuple[str, str | None]:
- """Return the public failed-run error text and structured reason."""
+def _run_failed_error_payload(exc: Exception) -> tuple[str, RunFailureType | None, str | None]:
+ """Return the public failed-run error text, type, and structured reason."""
message = str(exc) or type(exc).__name__
reason: str | None = None
+ if isinstance(exc, UsageLimitExceeded):
+ return message, RunFailureType.AGENT_RUN_LIMIT_EXCEEDED, None
+
if isinstance(exc, BindingLostError):
- return message, "binding_lost"
+ return message, None, "binding_lost"
if isinstance(exc, ModelHTTPError):
body = exc.body
@@ -138,7 +142,7 @@ def _run_failed_error_payload(exc: Exception) -> tuple[str, str | None]:
if isinstance(exc, DifyKnowledgeBaseClientError):
reason = exc.error_code or "DifyKnowledgeBaseClientError"
- return message, reason
+ return message, None, reason
def _has_model_layer(request: CreateRunRequest) -> bool:
@@ -208,8 +212,14 @@ class AgentRunRunner:
except Exception as exc:
if self.is_cancelled():
return
- message, reason = _run_failed_error_payload(exc)
- finalization = await emit_run_failed(self.sink, run_id=self.run_id, error=message, reason=reason)
+ message, error_type, reason = _run_failed_error_payload(exc)
+ finalization = await emit_run_failed(
+ self.sink,
+ run_id=self.run_id,
+ error=message,
+ error_type=error_type,
+ reason=reason,
+ )
if finalization.applied:
raise
return
diff --git a/dify-agent/src/dify_agent/server/routes/runs.py b/dify-agent/src/dify_agent/server/routes/runs.py
index 09e1fe3857b..4f08e76aa9f 100644
--- a/dify-agent/src/dify_agent/server/routes/runs.py
+++ b/dify-agent/src/dify_agent/server/routes/runs.py
@@ -68,6 +68,7 @@ def create_runs_router(
created_at=record.created_at,
updated_at=record.updated_at,
error=record.error,
+ error_type=record.error_type,
)
@router.post("/{run_id}/cancel", response_model=CancelRunResponse)
diff --git a/dify-agent/src/dify_agent/server/schemas.py b/dify-agent/src/dify_agent/server/schemas.py
index 21e8e624a0b..879d1b211e2 100644
--- a/dify-agent/src/dify_agent/server/schemas.py
+++ b/dify-agent/src/dify_agent/server/schemas.py
@@ -33,6 +33,7 @@ class RunRecord(BaseModel):
created_at: datetime = Field(default_factory=_protocol_schemas.utc_now)
updated_at: datetime = Field(default_factory=_protocol_schemas.utc_now)
error: str | None = None
+ error_type: _protocol_schemas.RunFailureType | None = None
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
diff --git a/dify-agent/src/dify_agent/storage/redis_run_store.py b/dify-agent/src/dify_agent/storage/redis_run_store.py
index 140ea5a7f3d..6bba2368b16 100644
--- a/dify-agent/src/dify_agent/storage/redis_run_store.py
+++ b/dify-agent/src/dify_agent/storage/redis_run_store.py
@@ -20,7 +20,7 @@ from dify_agent.runtime.event_sink import (
RunEventSink,
RunFinalizationResult,
TerminalRunEvent,
- terminal_event_status_and_error,
+ terminal_event_status_fields,
)
from dify_agent.server.schemas import RunRecord, new_run_id
from dify_agent.server.settings import DEFAULT_RUN_RETENTION_SECONDS
@@ -51,10 +51,15 @@ if ARGV[3] == "1" then
else
record.error = cjson.null
end
+if ARGV[5] == "1" then
+ record.error_type = ARGV[6]
+else
+ record.error_type = cjson.null
+end
-local ttl = tonumber(ARGV[6])
+local ttl = tonumber(ARGV[8])
local updated_record_json = cjson.encode(record)
-local event_id = redis.call("XADD", KEYS[2], "*", "payload", ARGV[5])
+local event_id = redis.call("XADD", KEYS[2], "*", "payload", ARGV[7])
redis.call("EXPIRE", KEYS[2], ttl)
redis.call("SET", KEYS[1], updated_record_json, "EX", ttl)
return {1, ARGV[1], event_id}
@@ -126,7 +131,7 @@ class RedisRunStore(RunEventSink):
async def finalize_run(self, event: TerminalRunEvent) -> RunFinalizationResult:
"""Atomically append the first terminal event and update its run record."""
- status, error = terminal_event_status_and_error(event)
+ status, error, error_type = terminal_event_status_fields(event)
payload = RUN_EVENT_ADAPTER.dump_json(event, exclude={"id"}).decode()
evaluation = cast(
Awaitable[object],
@@ -139,6 +144,8 @@ class RedisRunStore(RunEventSink):
event.created_at.isoformat(),
"1" if error is not None else "0",
error or "",
+ "1" if error_type is not None else "0",
+ error_type.value if error_type is not None else "",
payload,
str(self.run_retention_seconds),
),
diff --git a/dify-agent/tests/integration/dify_agent/storage/test_terminal_finalization.py b/dify-agent/tests/integration/dify_agent/storage/test_terminal_finalization.py
index 790ac76f780..396e7736f68 100644
--- a/dify-agent/tests/integration/dify_agent/storage/test_terminal_finalization.py
+++ b/dify-agent/tests/integration/dify_agent/storage/test_terminal_finalization.py
@@ -19,10 +19,13 @@ from dify_agent.protocol.schemas import (
RunCancelledEvent,
RunCancelledEventData,
RunComposition,
+ RunFailedEvent,
+ RunFailedEventData,
+ RunFailureType,
RunSucceededEvent,
RunSucceededEventData,
)
-from dify_agent.runtime.event_sink import TerminalRunEvent, terminal_event_status_and_error
+from dify_agent.runtime.event_sink import TerminalRunEvent, terminal_event_status_fields
from dify_agent.runtime.run_scheduler import RunScheduler
from dify_agent.storage.redis_keys import run_events_key, run_record_key
from dify_agent.storage.redis_run_store import RedisRunStore
@@ -115,12 +118,13 @@ def test_two_redis_clients_commit_exactly_one_matching_terminal(redis_url: str)
winner_index = next(index for index, result in enumerate(results) if result.applied)
winner_event = terminal_events[winner_index]
winner_result = results[winner_index]
- expected_status, expected_error = terminal_event_status_and_error(winner_event)
+ expected_status, expected_error, expected_error_type = terminal_event_status_fields(winner_event)
persisted = await first_store.get_run(record.run_id)
page = await second_store.get_events(record.run_id)
assert persisted.status == expected_status
assert persisted.error == expected_error
+ assert persisted.error_type == expected_error_type
assert persisted.updated_at == winner_event.created_at
assert len(page.events) == 1
assert page.events[0].type == winner_event.type
@@ -138,6 +142,36 @@ def test_two_redis_clients_commit_exactly_one_matching_terminal(redis_url: str)
asyncio.run(scenario())
+def test_classified_failure_persists_matching_record_and_event_error_type(redis_url: str) -> None:
+ async def scenario() -> None:
+ client = Redis.from_url(redis_url)
+ store = RedisRunStore(client, prefix=f"classified-failure-{uuid4().hex}", run_retention_seconds=60)
+ try:
+ record = await store.create_run()
+ event = RunFailedEvent(
+ run_id=record.run_id,
+ data=RunFailedEventData(
+ error="run limit reached",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ ),
+ )
+
+ result = await store.finalize_run(event)
+ persisted = await store.get_run(record.run_id)
+ page = await store.get_events(record.run_id)
+
+ assert result.applied is True
+ assert persisted.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+ assert len(page.events) == 1
+ persisted_event = page.events[0]
+ assert isinstance(persisted_event, RunFailedEvent)
+ assert persisted_event.data.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+ finally:
+ await client.aclose()
+
+ asyncio.run(scenario())
+
+
def test_non_owner_scheduler_cancellation_stops_owner_runner(redis_url: str) -> None:
class BlockingRunner:
def __init__(self, *, started: asyncio.Event, stopped: asyncio.Event) -> None:
diff --git a/dify-agent/tests/local/dify_agent/protocol/test_protocol_schemas.py b/dify-agent/tests/local/dify_agent/protocol/test_protocol_schemas.py
index 9aa8cf8af0f..5bc3915ffad 100644
--- a/dify-agent/tests/local/dify_agent/protocol/test_protocol_schemas.py
+++ b/dify-agent/tests/local/dify_agent/protocol/test_protocol_schemas.py
@@ -25,6 +25,7 @@ from dify_agent.protocol.schemas import (
RunComposition,
RunFailedEvent,
RunFailedEventData,
+ RunFailureType,
RunLayerSpec,
RunStartedEvent,
RunSucceededEvent,
@@ -68,7 +69,14 @@ def test_run_event_adapter_round_trips_typed_variants() -> None:
session_snapshot=CompositorSessionSnapshot(layers=[]),
),
),
- RunFailedEvent(run_id="run-1", data=RunFailedEventData(error="boom", reason="shutdown")),
+ RunFailedEvent(
+ run_id="run-1",
+ data=RunFailedEventData(
+ error="boom",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ reason="shutdown",
+ ),
+ ),
RunCancelledEvent(run_id="run-1", data=RunCancelledEventData(reason="user_cancelled")),
]
@@ -80,6 +88,31 @@ def test_run_event_adapter_round_trips_typed_variants() -> None:
assert decoded.run_id == event.run_id
+def test_run_failed_event_error_type_is_optional_and_round_trips() -> None:
+ legacy = RUN_EVENT_ADAPTER.validate_python(
+ {
+ "run_id": "legacy-run",
+ "type": "run_failed",
+ "data": {"error": "legacy failure", "reason": None},
+ }
+ )
+ classified = RunFailedEvent(
+ run_id="classified-run",
+ data=RunFailedEventData(
+ error="run limit reached",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ ),
+ )
+
+ decoded = RUN_EVENT_ADAPTER.validate_json(RUN_EVENT_ADAPTER.dump_json(classified))
+
+ assert isinstance(legacy, RunFailedEvent)
+ assert legacy.data.error_type is None
+ assert isinstance(decoded, RunFailedEvent)
+ assert decoded.data.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+ assert protocol_exports.RunFailureType is RunFailureType
+
+
def test_pydantic_ai_event_data_uses_agent_stream_event_model() -> None:
event = RUN_EVENT_ADAPTER.validate_python(
{
diff --git a/dify-agent/tests/local/dify_agent/runtime/test_run_scheduler.py b/dify-agent/tests/local/dify_agent/runtime/test_run_scheduler.py
index b38985b2732..a22ce101d2b 100644
--- a/dify-agent/tests/local/dify_agent/runtime/test_run_scheduler.py
+++ b/dify-agent/tests/local/dify_agent/runtime/test_run_scheduler.py
@@ -11,7 +11,7 @@ from agenton_collections.layers.plain import PromptLayerConfig
from dify_agent.layers.dify_plugin import DifyPluginLLMLayerConfig
from dify_agent.layers.execution_context import DIFY_EXECUTION_CONTEXT_LAYER_TYPE_ID, DifyExecutionContextLayerConfig
from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID, DifyOutputLayerConfig
-from dify_agent.protocol import DIFY_AGENT_MODEL_LAYER_ID, DIFY_AGENT_OUTPUT_LAYER_ID
+from dify_agent.protocol import DIFY_AGENT_MODEL_LAYER_ID, DIFY_AGENT_OUTPUT_LAYER_ID, RunFailureType
from dify_agent.protocol.schemas import (
CancelRunRequest,
CreateRunRequest,
@@ -26,7 +26,7 @@ from dify_agent.runtime.event_sink import (
TerminalRunEvent,
emit_run_failed,
emit_run_succeeded,
- terminal_event_status_and_error,
+ terminal_event_status_fields,
)
from dify_agent.runtime.run_scheduler import RunCancellationConflictError, RunScheduler, SchedulerStoppingError
from dify_agent.server.schemas import RunRecord
@@ -93,6 +93,7 @@ class FakeStore:
events: dict[str, list[RunEvent]]
statuses: dict[str, RunStatus]
errors: dict[str, str | None]
+ error_types: dict[str, RunFailureType | None]
terminal_changes: dict[str, asyncio.Event]
def __init__(self) -> None:
@@ -100,6 +101,7 @@ class FakeStore:
self.events = defaultdict(list)
self.statuses = {}
self.errors = {}
+ self.error_types = {}
self.terminal_changes = {}
async def create_run(self) -> RunRecord:
@@ -117,7 +119,11 @@ class FakeStore:
async def get_run(self, run_id: str) -> RunRecord:
return self.records[run_id].model_copy(
- update={"status": self.statuses[run_id], "error": self.errors.get(run_id)},
+ update={
+ "status": self.statuses[run_id],
+ "error": self.errors.get(run_id),
+ "error_type": self.error_types.get(run_id),
+ },
)
async def finalize_run(self, event: TerminalRunEvent) -> RunFinalizationResult:
@@ -125,11 +131,12 @@ class FakeStore:
if current_status != "running":
return RunFinalizationResult(applied=False, status=current_status)
- status, error = terminal_event_status_and_error(event)
+ status, error, error_type = terminal_event_status_fields(event)
event_id = str(len(self.events[event.run_id]) + 1)
self.events[event.run_id].append(event.model_copy(update={"id": event_id}))
self.statuses[event.run_id] = status
self.errors[event.run_id] = error
+ self.error_types[event.run_id] = error_type
self.terminal_changes[event.run_id].set()
return RunFinalizationResult(applied=True, status=status, event_id=event_id)
diff --git a/dify-agent/tests/local/dify_agent/runtime/test_runner.py b/dify-agent/tests/local/dify_agent/runtime/test_runner.py
index f9c580f3116..e2fd7a4099c 100644
--- a/dify-agent/tests/local/dify_agent/runtime/test_runner.py
+++ b/dify-agent/tests/local/dify_agent/runtime/test_runner.py
@@ -8,7 +8,7 @@ import pytest
from graphon.model_runtime.entities.llm_entities import LLMUsage
from pydantic import JsonValue
from pydantic_ai import Tool
-from pydantic_ai.exceptions import ModelHTTPError, UnexpectedModelBehavior
+from pydantic_ai.exceptions import ModelHTTPError, UnexpectedModelBehavior, UsageLimitExceeded
from pydantic_ai.messages import (
ToolReturnPart,
ModelMessage,
@@ -61,6 +61,8 @@ from dify_agent.protocol.schemas import (
LayerExitSignals,
PydanticAIStreamRunEvent,
RunComposition,
+ RunFailedEvent,
+ RunFailureType,
RunLayerSpec,
RunSucceededEvent,
)
@@ -179,18 +181,20 @@ def test_run_failed_error_payload_preserves_plugin_rate_limit_error() -> None:
{"error_type": "InvokeRateLimitError", "message": "quota exceeded"},
)
- message, reason = _run_failed_error_payload(exc)
+ message, error_type, reason = _run_failed_error_payload(exc)
assert message == "quota exceeded"
+ assert error_type is None
assert reason == "InvokeRateLimitError"
def test_run_failed_error_payload_infers_rate_limit_reason_from_status_code() -> None:
exc = ModelHTTPError(429, "gpt-4o-mini", {"message": "too many requests"})
- message, reason = _run_failed_error_payload(exc)
+ message, error_type, reason = _run_failed_error_payload(exc)
assert message == "too many requests"
+ assert error_type is None
assert reason == "InvokeRateLimitError"
@@ -202,12 +206,23 @@ def test_run_failed_error_payload_preserves_knowledge_error_code() -> None:
retryable=False,
)
- message, reason = _run_failed_error_payload(exc)
+ message, error_type, reason = _run_failed_error_payload(exc)
assert message == "Knowledge base search failed with HTTP 400 (dataset_not_found): Dataset not found"
+ assert error_type is None
assert reason == "dataset_not_found"
+def test_run_failed_error_payload_classifies_usage_limit() -> None:
+ exc = UsageLimitExceeded("The next request would exceed the request_limit of 100")
+
+ message, error_type, reason = _run_failed_error_payload(exc)
+
+ assert message == "The next request would exceed the request_limit of 100"
+ assert error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+ assert reason is None
+
+
def test_cancelled_runner_does_not_overwrite_cancelled_status_with_late_failure(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -1957,6 +1972,37 @@ def test_runner_failure_with_history_layer_emits_failed_terminal_event_without_s
assert _history_messages_from_snapshot(request.session_snapshot) == stored_history
+def test_runner_persists_usage_limit_failure_type_in_event_and_status(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ sink = InMemoryRunEventSink()
+
+ async def scenario() -> None:
+ async with httpx.AsyncClient() as client:
+ runner = AgentRunRunner(
+ sink=sink,
+ request=_request(),
+ run_id="run-limit",
+ plugin_daemon_http_client=client,
+ dify_api_http_client=client,
+ )
+
+ async def exceed_limit() -> RunSuccessOutcome:
+ raise UsageLimitExceeded("The next request would exceed the request_limit of 100")
+
+ monkeypatch.setattr(runner, "_run_agent", exceed_limit)
+ with pytest.raises(UsageLimitExceeded):
+ await runner.run()
+
+ asyncio.run(scenario())
+
+ terminal = sink.events["run-limit"][-1]
+ assert isinstance(terminal, RunFailedEvent)
+ assert terminal.data.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+ assert sink.statuses["run-limit"] == "failed"
+ assert sink.error_types["run-limit"] is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
+
+
def test_runner_applies_on_exit_overrides_to_success_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:
def fake_get_model(_self: DifyPluginLLMLayer, *, http_client: httpx.AsyncClient):
assert http_client.is_closed is False
diff --git a/dify-agent/tests/local/dify_agent/server/test_runs_routes.py b/dify-agent/tests/local/dify_agent/server/test_runs_routes.py
index 7a8da51eb29..ce2c6247a7c 100644
--- a/dify-agent/tests/local/dify_agent/server/test_runs_routes.py
+++ b/dify-agent/tests/local/dify_agent/server/test_runs_routes.py
@@ -1,6 +1,6 @@
from fastapi.testclient import TestClient
-from dify_agent.protocol import CancelRunResponse, DIFY_AGENT_MODEL_LAYER_ID
+from dify_agent.protocol import CancelRunResponse, DIFY_AGENT_MODEL_LAYER_ID, RunFailureType
from dify_agent.runtime.run_scheduler import RunCancellationConflictError, SchedulerStoppingError
from dify_agent.server.routes.runs import create_runs_router
from dify_agent.server.schemas import RunRecord
@@ -21,6 +21,29 @@ class FakeStore:
pass
+def test_get_run_status_returns_failure_type() -> None:
+ from fastapi import FastAPI
+
+ class FailedRunStore:
+ async def get_run(self, run_id: str) -> RunRecord:
+ return RunRecord(
+ run_id=run_id,
+ status="failed",
+ error="run limit reached",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ )
+
+ app = FastAPI()
+ app.include_router(
+ create_runs_router(lambda: FailedRunStore(), lambda: FakeScheduler()) # pyright: ignore[reportArgumentType]
+ )
+ response = TestClient(app).get("/runs/run-1")
+
+ assert response.status_code == 200
+ assert response.json()["error"] == "run limit reached"
+ assert response.json()["error_type"] == "agent_run_limit_exceeded"
+
+
def test_create_run_accepts_effectively_blank_user_prompt_list() -> None:
from fastapi import FastAPI
diff --git a/dify-agent/tests/local/dify_agent/storage/test_redis_run_store.py b/dify-agent/tests/local/dify_agent/storage/test_redis_run_store.py
index ce839f80f33..e33e1ec8188 100644
--- a/dify-agent/tests/local/dify_agent/storage/test_redis_run_store.py
+++ b/dify-agent/tests/local/dify_agent/storage/test_redis_run_store.py
@@ -13,6 +13,7 @@ from dify_agent.protocol.schemas import (
RunCancelledEventData,
RunFailedEvent,
RunFailedEventData,
+ RunFailureType,
RunStartedEvent,
RunStatus,
RunSucceededEvent,
@@ -115,7 +116,9 @@ class FakeRedis:
updated_at = str(keys_and_args[3])
has_error = str(keys_and_args[4]) == "1"
error = str(keys_and_args[5]) if has_error else None
- payload = str(keys_and_args[6])
+ has_error_type = str(keys_and_args[6]) == "1"
+ error_type = str(keys_and_args[7]) if has_error_type else None
+ payload = str(keys_and_args[8])
record_json = self.values.get(record_key)
if record_json is None:
return [-1, "", ""]
@@ -125,7 +128,7 @@ class FakeRedis:
if record["status"] != "running":
return [0, record["status"], ""]
- record.update({"status": status, "updated_at": updated_at, "error": error})
+ record.update({"status": status, "updated_at": updated_at, "error": error, "error_type": error_type})
event_id = self._append_stream_entry(events_key, {"payload": payload})
self.values[record_key] = json.dumps(record, separators=(",", ":"))
return [1, status, event_id]
@@ -207,6 +210,20 @@ def test_create_run_writes_running_record_without_job_queue_and_with_retention()
assert "request" not in str(redis.commands[0][2])
+def test_get_run_accepts_legacy_record_without_error_type() -> None:
+ redis = FakeRedis()
+ store = RedisRunStore(redis, prefix="test") # pyright: ignore[reportArgumentType]
+ record = asyncio.run(store.create_run())
+ record_key = f"test:runs:{record.run_id}:record"
+ payload = json.loads(cast(str, redis.values[record_key]))
+ del payload["error_type"]
+ redis.values[record_key] = json.dumps(payload)
+
+ loaded = asyncio.run(store.get_run(record.run_id))
+
+ assert loaded.error_type is None
+
+
def test_finalize_run_atomically_writes_terminal_event_and_status() -> None:
redis = FakeRedis()
store = RedisRunStore(redis, prefix="test", run_retention_seconds=60) # pyright: ignore[reportArgumentType]
@@ -225,6 +242,7 @@ def test_finalize_run_atomically_writes_terminal_event_and_status() -> None:
assert result.event_id == "1-0"
assert updated.status == "cancelled"
assert updated.error == "workflow stopped"
+ assert updated.error_type is None
assert updated.updated_at == event.created_at
stream_entry_id, stream_fields = redis.streams[f"test:runs:{record.run_id}:events"][0]
assert stream_entry_id == result.event_id
@@ -275,7 +293,11 @@ def test_finalize_failed_run_derives_error_and_timestamp_from_event() -> None:
record = asyncio.run(store.create_run())
event = RunFailedEvent(
run_id=record.run_id,
- data=RunFailedEventData(error="model failed", reason="model_error"),
+ data=RunFailedEventData(
+ error="model failed",
+ error_type=RunFailureType.AGENT_RUN_LIMIT_EXCEEDED,
+ reason="model_error",
+ ),
)
result = asyncio.run(store.finalize_run(event))
@@ -285,7 +307,11 @@ def test_finalize_failed_run_derives_error_and_timestamp_from_event() -> None:
assert result.status == "failed"
assert updated.status == "failed"
assert updated.error == "model failed"
+ assert updated.error_type is RunFailureType.AGENT_RUN_LIMIT_EXCEEDED
assert updated.updated_at == event.created_at
+ stream_entry = redis.streams[f"test:runs:{record.run_id}:events"][0]
+ payload = json.loads(cast(str, stream_entry[1]["payload"]))
+ assert payload["data"]["error_type"] == "agent_run_limit_exceeded"
def test_two_store_instances_choose_exactly_one_terminal_winner() -> None: