Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat: tool fallback observability (#42596)

杨俊波e6f1d77ed586
diff --git a/api/core/workflow/generator/runner.py b/api/core/workflow/generator/runner.py
index 42095fbf85c..64f8b76e417 100644
--- a/api/core/workflow/generator/runner.py
+++ b/api/core/workflow/generator/runner.py
@@ -57,12 +57,13 @@ from core.workflow.generator.prompts.planner_prompts import (
from core.workflow.generator.prompts.tool_router_prompts import TOOL_ROUTER_SYSTEM_PROMPT, TOOL_ROUTER_USER_PROMPT
from core.workflow.generator.tool_catalogue import (
DIRECT_TOOL_INJECTION_LIMIT,
+ LegacyToolFallbackSelection,
ToolCapabilityQuery,
ToolCatalogueEntry,
find_tool_entry,
format_tool_builder_context,
format_tool_catalogue,
- select_legacy_fallback_tools,
+ select_legacy_fallback_selection,
select_tool_candidates,
text_mentions_tool_identifier,
)
@@ -120,6 +121,7 @@ _DEFAULT_FILE_UPLOAD_METHODS = ("local_file", "remote_url")
# keep the caller's budget so complex node configs are not truncated.
_PLANNER_DEFAULT_MAX_TOKENS = 4096
_TOOL_ROUTER_MAX_TOKENS = 256
+_MAX_OMITTED_TOOL_PREVIEW = 20
# Per-node calls trade a larger request count for a shorter critical path.
@@ -341,6 +343,41 @@ def _find_planned_tool_entry(node: dict[str, Any], entries: list[ToolCatalogueEn
return None
+def _format_omitted_tool_preview(entries: list[ToolCatalogueEntry]) -> str:
+ """Build a bounded identity preview for fallback diagnostics."""
+ identifiers = [f"{entry['provider_name']}/{entry['tool_name']}" for entry in entries]
+ preview = ",".join(identifiers[:_MAX_OMITTED_TOOL_PREVIEW])
+ if len(identifiers) > _MAX_OMITTED_TOOL_PREVIEW:
+ preview = f"{preview},...(+{len(identifiers) - _MAX_OMITTED_TOOL_PREVIEW})"
+ return preview
+
+
+def _log_tool_catalogue_fallback(
+ *,
+ reason: str,
+ total: int,
+ query_count: int,
+ selection: LegacyToolFallbackSelection,
+ started_at: float,
+) -> None:
+ """Record selected and omitted identities for legacy catalogue fallback."""
+ logger.warning(
+ "Workflow generator: tool catalogue selection mode=fallback reason=%s total=%s "
+ "query_count=%s candidates=%s omitted=%s pinned=%s limit=%s overflow=%s "
+ "omitted_tools=%s elapsed_ms=%.1f",
+ reason,
+ total,
+ query_count,
+ len(selection.entries),
+ len(selection.omitted_entries),
+ selection.pinned_count,
+ selection.limit,
+ selection.overflow_count,
+ _format_omitted_tool_preview(selection.omitted_entries),
+ (time.monotonic() - started_at) * 1000,
+ )
+
+
def _stage_error_to_envelope_code(exc: Exception) -> str:
"""Map a stage-typed exception to the result envelope's error code."""
if isinstance(exc, _StageJSONError):
@@ -730,21 +767,19 @@ class WorkflowGenerator:
)
candidate_count = len(selection.entries)
if selection.unmatched_queries:
- fallback_entries = select_legacy_fallback_tools(
+ fallback_selection = select_legacy_fallback_selection(
tool_catalogue_entries,
explicit_text=explicit_text,
current_graph=current_graph,
)
- logger.warning(
- "Workflow generator: tool catalogue selection mode=fallback reason=unmatched_query "
- "total=%s query_count=%s candidates=%s pinned=%s elapsed_ms=%.1f",
- tool_count,
- query_count,
- len(fallback_entries),
- selection.pinned_count,
- (time.monotonic() - started_at) * 1000,
+ _log_tool_catalogue_fallback(
+ reason="unmatched_query",
+ total=tool_count,
+ query_count=query_count,
+ selection=fallback_selection,
+ started_at=started_at,
)
- return format_tool_catalogue(fallback_entries, max_tools=None)
+ return format_tool_catalogue(fallback_selection.entries, max_tools=None)
logger.info(
"Workflow generator: tool catalogue selection mode=routed total=%s needs_tools=%s "
@@ -758,21 +793,19 @@ class WorkflowGenerator:
)
return format_tool_catalogue(selection.entries, max_tools=None)
except Exception as e:
- fallback_entries = select_legacy_fallback_tools(
+ fallback_selection = select_legacy_fallback_selection(
tool_catalogue_entries,
explicit_text=explicit_text,
current_graph=current_graph,
)
- logger.warning(
- "Workflow generator: tool catalogue selection mode=fallback reason=%s total=%s "
- "query_count=%s candidates=%s elapsed_ms=%.1f",
- type(e).__name__,
- tool_count,
- query_count,
- len(fallback_entries),
- (time.monotonic() - started_at) * 1000,
+ _log_tool_catalogue_fallback(
+ reason=type(e).__name__,
+ total=tool_count,
+ query_count=query_count,
+ selection=fallback_selection,
+ started_at=started_at,
)
- return format_tool_catalogue(fallback_entries, max_tools=None)
+ return format_tool_catalogue(fallback_selection.entries, max_tools=None)
@classmethod
def _run_tool_router(
diff --git a/api/core/workflow/generator/tool_catalogue.py b/api/core/workflow/generator/tool_catalogue.py
index 83eb902abc7..55f5a0ef0a3 100644
--- a/api/core/workflow/generator/tool_catalogue.py
+++ b/api/core/workflow/generator/tool_catalogue.py
@@ -92,6 +92,19 @@ class ToolCandidateSelection:
pinned_count: int
+@dataclass(frozen=True)
+class LegacyToolFallbackSelection:
+ entries: list[ToolCatalogueEntry]
+ omitted_entries: list[ToolCatalogueEntry]
+ pinned_count: int
+ limit: int
+
+ @property
+ def overflow_count(self) -> int:
+ """Count intentional pinned overflow beyond the legacy prompt cap."""
+ return max(0, len(self.entries) - self.limit)
+
+
def build_tool_catalogue(tenant_id: str) -> list[ToolCatalogueEntry]:
"""
Enumerate installed tools for the given tenant.
@@ -292,13 +305,13 @@ def select_tool_candidates(
)
-def select_legacy_fallback_tools(
+def select_legacy_fallback_selection(
entries: list[ToolCatalogueEntry],
*,
explicit_text: str = "",
current_graph: dict[str, Any] | None = None,
-) -> list[ToolCatalogueEntry]:
- """Keep pinned tools while reproducing the legacy bounded prompt fallback."""
+) -> LegacyToolFallbackSelection:
+ """Keep pinned tools and expose the tools omitted by the legacy fallback."""
pinned = select_tool_candidates(
entries,
[],
@@ -307,10 +320,27 @@ def select_legacy_fallback_tools(
).entries
pinned_keys = {(entry["provider_name"], entry["tool_name"]) for entry in pinned}
regular_limit = max(0, _MAX_PROMPT_TOOLS - len(pinned))
- regular = [entry for entry in entries if (entry["provider_name"], entry["tool_name"]) not in pinned_keys][
- :regular_limit
- ]
- return [*pinned, *regular]
+ regular = [entry for entry in entries if (entry["provider_name"], entry["tool_name"]) not in pinned_keys]
+ return LegacyToolFallbackSelection(
+ entries=[*pinned, *regular[:regular_limit]],
+ omitted_entries=regular[regular_limit:],
+ pinned_count=len(pinned),
+ limit=_MAX_PROMPT_TOOLS,
+ )
+
+
+def select_legacy_fallback_tools(
+ entries: list[ToolCatalogueEntry],
+ *,
+ explicit_text: str = "",
+ current_graph: dict[str, Any] | None = None,
+) -> list[ToolCatalogueEntry]:
+ """Keep pinned tools while reproducing the legacy bounded prompt fallback."""
+ return select_legacy_fallback_selection(
+ entries,
+ explicit_text=explicit_text,
+ current_graph=current_graph,
+ ).entries
def _find_explicit_tool_keys(
diff --git a/api/tests/unit_tests/core/workflow/generator/test_runner.py b/api/tests/unit_tests/core/workflow/generator/test_runner.py
index 1a067ca2089..89f192fa286 100644
--- a/api/tests/unit_tests/core/workflow/generator/test_runner.py
+++ b/api/tests/unit_tests/core/workflow/generator/test_runner.py
@@ -339,6 +339,58 @@ class TestDynamicToolRouting:
assert "reason=unmatched_query" in caplog.text
assert "candidates=80" in caplog.text
+ def test_unmatched_router_query_logs_capped_omitted_tool_preview(self, caplog: pytest.LogCaptureFixture):
+ model = MagicMock()
+ model.invoke_llm.return_value = _llm_result(
+ json.dumps(
+ {
+ "needs_tools": True,
+ "queries": [{"capability": "quantum melody", "keywords": ["qubits", "music"]}],
+ }
+ )
+ )
+ entries = [_tool_entry("provider", f"tool_{index:03d}", description="Manage records.") for index in range(101)]
+
+ text = WorkflowGenerator._resolve_prompt_tool_catalogue(
+ model_instance=model,
+ model_parameters={},
+ instruction="Create something external",
+ ideal_output="",
+ tool_catalogue_text="",
+ tool_catalogue_entries=entries,
+ current_graph=None,
+ )
+
+ assert len(text.splitlines()) == 80
+ assert "reason=unmatched_query" in caplog.text
+ assert "candidates=80" in caplog.text
+ assert "omitted=21" in caplog.text
+ assert "overflow=0" in caplog.text
+ assert "omitted_tools=provider/tool_080" in caplog.text
+ assert "provider/tool_099,...(+1)" in caplog.text
+
+ def test_router_exception_logs_pinned_fallback_tool(self, caplog: pytest.LogCaptureFixture):
+ model = MagicMock()
+ model.invoke_llm.side_effect = RuntimeError("router unavailable")
+ entries = [_tool_entry("provider", f"tool_{index:03d}") for index in range(100)]
+
+ text = WorkflowGenerator._resolve_prompt_tool_catalogue(
+ model_instance=model,
+ model_parameters={},
+ instruction="Use provider/tool_099 exactly.",
+ ideal_output="",
+ tool_catalogue_text="",
+ tool_catalogue_entries=entries,
+ current_graph=None,
+ )
+
+ assert len(text.splitlines()) == 80
+ assert "reason=RuntimeError" in caplog.text
+ assert "candidates=80" in caplog.text
+ assert "omitted=20" in caplog.text
+ assert "pinned=1" in caplog.text
+ assert "overflow=0" in caplog.text
+
def test_router_failure_does_not_surface_as_model_error(self):
planner = json.dumps(
{
diff --git a/api/tests/unit_tests/core/workflow/generator/test_tool_catalogue.py b/api/tests/unit_tests/core/workflow/generator/test_tool_catalogue.py
index 7f54f314266..74043747f5c 100644
--- a/api/tests/unit_tests/core/workflow/generator/test_tool_catalogue.py
+++ b/api/tests/unit_tests/core/workflow/generator/test_tool_catalogue.py
@@ -17,6 +17,7 @@ from core.workflow.generator.tool_catalogue import (
format_tool_builder_context,
format_tool_catalogue,
installed_tool_keys,
+ select_legacy_fallback_selection,
select_legacy_fallback_tools,
select_tool_candidates,
)
@@ -268,6 +269,45 @@ class TestSelectToolCandidates:
assert all(entry["tool_name"] != "tool_079" for entry in selected)
+class TestLegacyFallbackSelection:
+ def test_returns_selected_and_omitted_tools(self):
+ entries = [_entry("provider", f"tool_{index:03d}") for index in range(100)]
+
+ selection = select_legacy_fallback_selection(entries)
+
+ assert selection.entries == entries[:80]
+ assert selection.omitted_entries == entries[80:]
+ assert selection.pinned_count == 0
+ assert selection.limit == 80
+ assert selection.overflow_count == 0
+
+ def test_keeps_legacy_fallback_tools_compatible(self):
+ entries = [_entry("provider", f"tool_{index:03d}") for index in range(100)]
+
+ assert select_legacy_fallback_tools(entries) == select_legacy_fallback_selection(entries).entries
+
+ def test_reports_pinned_overflow_without_fabricated_omissions(self):
+ entries = [_entry("provider", f"tool_{index:03d}") for index in range(85)]
+ explicit_text = "Use " + ", ".join(f"provider/{entry['tool_name']}" for entry in entries) + " exactly."
+
+ selection = select_legacy_fallback_selection(entries, explicit_text=explicit_text)
+
+ assert selection.entries == entries
+ assert selection.omitted_entries == []
+ assert selection.pinned_count == 85
+ assert selection.limit == 80
+ assert selection.overflow_count == 5
+
+ def test_empty_catalogue_has_no_omissions_or_overflow(self):
+ selection = select_legacy_fallback_selection([])
+
+ assert selection.entries == []
+ assert selection.omitted_entries == []
+ assert selection.pinned_count == 0
+ assert selection.limit == 80
+ assert selection.overflow_count == 0
+
+
class TestToolBuilderContext:
def test_finds_exact_provider_and_tool_pair(self):
entries = [_entry("google", "search"), _entry("google", "maps")]