Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): bound suggested question generation latency (#41705)

zyssyz123688107c3e5a2
diff --git a/api/core/llm_generator/llm_generator.py b/api/core/llm_generator/llm_generator.py
index 03fc06f0d3f..2ea9bf0af39 100644
--- a/api/core/llm_generator/llm_generator.py
+++ b/api/core/llm_generator/llm_generator.py
@@ -24,10 +24,11 @@ from core.llm_generator.prompts import (
WORKFLOW_RULE_CONFIG_PROMPT_GENERATE_TEMPLATE,
)
from core.model_context import with_credit_usage_created_by
-from core.model_manager import ModelManager
+from core.model_manager import ModelInstance, ModelManager
from core.ops.entities.trace_entity import TraceTaskName
from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
from core.ops.utils import measure_time
+from core.plugin.impl.base import use_plugin_daemon_request_timeout
from core.prompt.utils.prompt_template_parser import PromptTemplateParser
from core.telemetry import PromptGenerationEvent, TelemetryContext
from core.telemetry import emit as telemetry_emit
@@ -36,13 +37,17 @@ from extensions.ext_storage import storage
from graphon.enums import WorkflowNodeExecutionMetadataKey
from graphon.model_runtime.entities.llm_entities import LLMResult
from graphon.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage
-from graphon.model_runtime.entities.model_entities import ModelType
-from graphon.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
+from graphon.model_runtime.entities.model_entities import ModelType, ParameterType
+from graphon.model_runtime.errors.invoke import InvokeError
from models import App, Message, WorkflowNodeExecutionModel
from models.workflow import Workflow
logger = logging.getLogger(__name__)
+_SUGGESTED_QUESTIONS_MAX_TOKENS = 256
+_SUGGESTED_QUESTIONS_TIMEOUT_SECONDS = 30.0
+_LOW_REASONING_EFFORTS = ("none", "minimal", "low")
+
class SuggestedQuestionsModelConfig(TypedDict):
provider: str
@@ -74,6 +79,39 @@ def _normalize_completion_params(completion_params: dict[str, object]) -> tuple[
return normalized_parameters, stop
+def _default_suggested_questions_model_parameters(model_instance: ModelInstance) -> dict[str, object]:
+ """Build a low-latency parameter set for the workspace default model."""
+ parameters: dict[str, object] = {
+ "max_tokens": _SUGGESTED_QUESTIONS_MAX_TOKENS,
+ "temperature": 0.0,
+ }
+
+ try:
+ model_schema = model_instance.get_model_schema()
+ except Exception:
+ logger.warning("Failed to inspect the default model schema for suggested questions", exc_info=True)
+ return parameters
+
+ parameter_rules = {rule.name: rule for rule in model_schema.parameter_rules}
+ thinking_rule = parameter_rules.get("thinking")
+ if thinking_rule is not None:
+ if thinking_rule.type == ParameterType.BOOLEAN:
+ parameters["thinking"] = False
+ return parameters
+ if "disabled" in thinking_rule.options:
+ parameters["thinking"] = "disabled"
+ return parameters
+
+ reasoning_effort_rule = parameter_rules.get("reasoning_effort")
+ if reasoning_effort_rule is not None:
+ for effort in _LOW_REASONING_EFFORTS:
+ if effort in reasoning_effort_rule.options:
+ parameters["reasoning_effort"] = effort
+ break
+
+ return parameters
+
+
# ── Workflow instruction-suggestion tuning ────────────────────────────────
# Suggestions are a soft, pre-model-pick enhancement: short, buildable example
# instructions proposed from the tenant's DEFAULT model. Every failure path
@@ -306,7 +344,8 @@ class LLMGenerator:
tenant_id=tenant_id,
model_type=ModelType.LLM,
)
- except InvokeAuthorizationError:
+ except Exception:
+ logger.exception("Failed to resolve the suggested-questions model")
return []
prompt_messages: list[PromptMessage] = [UserPromptMessage(content=prompt)]
@@ -324,18 +363,16 @@ class LLMGenerator:
stop = []
else:
# Default-model generation keeps the built-in suggested-questions tuning.
- model_parameters = {
- "max_tokens": 2560,
- "temperature": 0.0,
- }
+ model_parameters = _default_suggested_questions_model_parameters(model_instance)
stop = []
- response: LLMResult = model_instance.invoke_llm(
- prompt_messages=list(prompt_messages),
- model_parameters=model_parameters,
- stop=stop,
- stream=False,
- )
+ with use_plugin_daemon_request_timeout(_SUGGESTED_QUESTIONS_TIMEOUT_SECONDS):
+ response: LLMResult = model_instance.invoke_llm(
+ prompt_messages=list(prompt_messages),
+ model_parameters=model_parameters,
+ stop=stop,
+ stream=False,
+ )
text_content = response.message.get_text_content()
questions = output_parser.parse(text_content) if text_content else []
diff --git a/api/core/plugin/impl/base.py b/api/core/plugin/impl/base.py
index 385ab67e34f..e26a04e71ae 100644
--- a/api/core/plugin/impl/base.py
+++ b/api/core/plugin/impl/base.py
@@ -2,6 +2,8 @@ import inspect
import json
import logging
from collections.abc import Callable, Generator, Mapping
+from contextlib import contextmanager
+from contextvars import ContextVar
from typing import Any, cast
from urllib.parse import unquote
@@ -60,6 +62,11 @@ match _plugin_daemon_timeout_config:
case _:
plugin_daemon_request_timeout = httpx.Timeout(_plugin_daemon_timeout_config)
+_plugin_daemon_request_timeout_override: ContextVar[httpx.Timeout | None] = ContextVar(
+ "plugin_daemon_request_timeout_override",
+ default=None,
+)
+
logger = logging.getLogger(__name__)
PLUGIN_DAEMON_MAX_PATH_LENGTH = 4096
@@ -71,6 +78,23 @@ _httpx_client: httpx.Client = get_pooled_http_client(
)
+@contextmanager
+def use_plugin_daemon_request_timeout(timeout_seconds: float) -> Generator[None, None, None]:
+ """Temporarily shorten plugin-daemon requests made in the current context."""
+ if timeout_seconds <= 0:
+ raise ValueError("timeout_seconds must be greater than zero")
+
+ token = _plugin_daemon_request_timeout_override.set(httpx.Timeout(timeout_seconds))
+ try:
+ yield
+ finally:
+ _plugin_daemon_request_timeout_override.reset(token)
+
+
+def _get_plugin_daemon_request_timeout() -> httpx.Timeout | None:
+ return _plugin_daemon_request_timeout_override.get() or plugin_daemon_request_timeout
+
+
def _normalize_plugin_daemon_response_for_type(json_response: Any, type_: type[object]) -> Any:
if type_ is not PluginListResponse:
return json_response
@@ -114,7 +138,7 @@ class BasePluginClient:
"headers": headers,
"params": params,
"files": files,
- "timeout": plugin_daemon_request_timeout,
+ "timeout": _get_plugin_daemon_request_timeout(),
}
if isinstance(prepared_data, dict):
request_kwargs["data"] = prepared_data
@@ -215,7 +239,7 @@ class BasePluginClient:
"headers": headers,
"params": params,
"files": files,
- "timeout": plugin_daemon_request_timeout,
+ "timeout": _get_plugin_daemon_request_timeout(),
}
if isinstance(prepared_data, dict):
stream_kwargs["data"] = prepared_data
diff --git a/api/services/message_service.py b/api/services/message_service.py
index cb5ba0306f2..9b20854ab17 100644
--- a/api/services/message_service.py
+++ b/api/services/message_service.py
@@ -411,10 +411,14 @@ class MessageService:
if suggested_questions_after_answer_config.get("enabled", False) is False:
raise SuggestedQuestionsAfterAnswerDisabledError()
- model_instance = model_manager.get_default_model_instance(
- tenant_id=app_model.tenant_id,
- model_type=ModelType.LLM,
- )
+ try:
+ model_instance = model_manager.get_default_model_instance(
+ tenant_id=app_model.tenant_id,
+ model_type=ModelType.LLM,
+ )
+ except Exception:
+ logger.exception("Failed to resolve the history model for suggested questions")
+ return []
# get memory of conversation (read-only)
memory = TokenBufferMemory(conversation=conversation, model_instance=model_instance)
diff --git a/api/tests/unit_tests/core/llm_generator/test_llm_generator.py b/api/tests/unit_tests/core/llm_generator/test_llm_generator.py
index 003514450dc..511102eeba9 100644
--- a/api/tests/unit_tests/core/llm_generator/test_llm_generator.py
+++ b/api/tests/unit_tests/core/llm_generator/test_llm_generator.py
@@ -4,6 +4,7 @@ import json
from collections.abc import Iterator
from datetime import datetime
from decimal import Decimal
+from types import SimpleNamespace
from unittest.mock import MagicMock, Mock, patch
from uuid import uuid4
@@ -18,7 +19,7 @@ from core.model_manager import ModelInstance, ModelManager
from graphon.enums import WorkflowNodeExecutionStatus
from graphon.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMUsage
from graphon.model_runtime.entities.message_entities import AssistantPromptMessage
-from graphon.model_runtime.entities.model_entities import ModelType
+from graphon.model_runtime.entities.model_entities import ModelType, ParameterType
from graphon.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
from models.enums import ConversationFromSource, CreatorUserRole
from models.model import App, AppMode, Message
@@ -258,16 +259,88 @@ class TestLLMGenerator:
assert len(questions) == 2
assert questions[0] == "Question 1?"
assert mock_model_instance.invoke_llm.call_args.kwargs["model_parameters"] == {
- "max_tokens": 2560,
+ "max_tokens": 256,
"temperature": 0.0,
}
+ def test_generate_suggested_questions_after_answer_uses_lowest_reasoning_effort(self, mock_model_instance):
+ mock_response = MagicMock()
+ mock_response.message.get_text_content.return_value = '["Question 1?"]'
+ mock_model_instance.invoke_llm.return_value = mock_response
+ mock_model_instance.get_model_schema.return_value.parameter_rules = [
+ SimpleNamespace(
+ name="reasoning_effort",
+ type=ParameterType.STRING,
+ options=["minimal", "low", "medium", "high"],
+ )
+ ]
+
+ questions = LLMGenerator.generate_suggested_questions_after_answer("tenant_id", "histories")
+
+ assert questions == ["Question 1?"]
+ assert mock_model_instance.invoke_llm.call_args.kwargs["model_parameters"] == {
+ "max_tokens": 256,
+ "temperature": 0.0,
+ "reasoning_effort": "minimal",
+ }
+
+ def test_generate_suggested_questions_after_answer_uses_defaults_when_schema_lookup_fails(
+ self, mock_model_instance
+ ):
+ mock_response = MagicMock()
+ mock_response.message.get_text_content.return_value = '["Question 1?"]'
+ mock_model_instance.invoke_llm.return_value = mock_response
+ mock_model_instance.get_model_schema.side_effect = ValueError("schema unavailable")
+
+ questions = LLMGenerator.generate_suggested_questions_after_answer("tenant_id", "histories")
+
+ assert questions == ["Question 1?"]
+ assert mock_model_instance.invoke_llm.call_args.kwargs["model_parameters"] == {
+ "max_tokens": 256,
+ "temperature": 0.0,
+ }
+
+ @pytest.mark.parametrize(
+ ("parameter_type", "options", "expected_value"),
+ [
+ (ParameterType.BOOLEAN, [], False),
+ (ParameterType.STRING, ["enabled", "disabled"], "disabled"),
+ ],
+ )
+ def test_generate_suggested_questions_after_answer_disables_thinking(
+ self, mock_model_instance, parameter_type, options, expected_value
+ ):
+ mock_response = MagicMock()
+ mock_response.message.get_text_content.return_value = '["Question 1?"]'
+ mock_model_instance.invoke_llm.return_value = mock_response
+ mock_model_instance.get_model_schema.return_value.parameter_rules = [
+ SimpleNamespace(name="thinking", type=parameter_type, options=options),
+ SimpleNamespace(name="reasoning_effort", type=ParameterType.STRING, options=["low", "high"]),
+ ]
+
+ questions = LLMGenerator.generate_suggested_questions_after_answer("tenant_id", "histories")
+
+ assert questions == ["Question 1?"]
+ assert mock_model_instance.invoke_llm.call_args.kwargs["model_parameters"] == {
+ "max_tokens": 256,
+ "temperature": 0.0,
+ "thinking": expected_value,
+ }
+
def test_generate_suggested_questions_after_answer_auth_error(self, mock_model_instance):
with patch("core.llm_generator.llm_generator.ModelManager.for_tenant") as mock_manager:
mock_manager.return_value.get_default_model_instance.side_effect = InvokeAuthorizationError("Auth failed")
questions = LLMGenerator.generate_suggested_questions_after_answer("tenant_id", "histories")
assert questions == []
+ def test_generate_suggested_questions_after_answer_model_resolution_error(self, mock_model_instance):
+ with patch("core.llm_generator.llm_generator.ModelManager.for_tenant") as mock_manager:
+ mock_manager.return_value.get_default_model_instance.side_effect = ValueError("unsupported default model")
+
+ questions = LLMGenerator.generate_suggested_questions_after_answer("tenant_id", "histories")
+
+ assert questions == []
+
def test_generate_suggested_questions_after_answer_invoke_error(self, mock_model_instance):
mock_model_instance.invoke_llm.side_effect = InvokeError("Invoke failed")
questions = LLMGenerator.generate_suggested_questions_after_answer("tenant_id", "histories")
@@ -311,6 +384,27 @@ class TestLLMGenerator:
assert invoke_kwargs["stop"] == []
assert "custom prompt" in invoke_kwargs["prompt_messages"][0].content
+ @patch("core.llm_generator.llm_generator.ModelManager.for_tenant")
+ def test_generate_suggested_questions_after_answer_with_custom_model_without_completion_params(
+ self, mock_for_tenant
+ ):
+ custom_model_instance = MagicMock()
+ custom_response = MagicMock()
+ custom_response.message.get_text_content.return_value = '["Question 1?"]'
+ custom_model_instance.invoke_llm.return_value = custom_response
+ mock_for_tenant.return_value.get_model_instance.return_value = custom_model_instance
+
+ questions = LLMGenerator.generate_suggested_questions_after_answer(
+ "tenant_id",
+ "histories",
+ model_config={"provider": "openai", "name": "gpt-4o"},
+ )
+
+ assert questions == ["Question 1?"]
+ invoke_kwargs = custom_model_instance.invoke_llm.call_args.kwargs
+ assert invoke_kwargs["model_parameters"] == {}
+ assert invoke_kwargs["stop"] == []
+
@patch("core.llm_generator.llm_generator.ModelManager.for_tenant")
def test_generate_suggested_questions_after_answer_fallback_to_default_model(self, mock_for_tenant):
default_model_instance = MagicMock()
@@ -337,7 +431,7 @@ class TestLLMGenerator:
model_type=ModelType.LLM,
)
assert default_model_instance.invoke_llm.call_args.kwargs["model_parameters"] == {
- "max_tokens": 2560,
+ "max_tokens": 256,
"temperature": 0.0,
}
assert default_model_instance.invoke_llm.call_args.kwargs["stop"] == []
diff --git a/api/tests/unit_tests/core/plugin/test_plugin_runtime.py b/api/tests/unit_tests/core/plugin/test_plugin_runtime.py
index e3dcf2996b4..d7d0c0fd388 100644
--- a/api/tests/unit_tests/core/plugin/test_plugin_runtime.py
+++ b/api/tests/unit_tests/core/plugin/test_plugin_runtime.py
@@ -23,7 +23,7 @@ from core.plugin.entities.plugin_daemon import (
CredentialType,
PluginDaemonInnerError,
)
-from core.plugin.impl.base import BasePluginClient
+from core.plugin.impl.base import BasePluginClient, use_plugin_daemon_request_timeout
from core.plugin.impl.exc import (
PluginDaemonBadRequestError,
PluginDaemonClientSideError,
@@ -154,6 +154,30 @@ class TestPluginRuntimeExecution:
call_kwargs = mock_request.call_args[1]
assert "timeout" in call_kwargs
+ def test_request_timeout_can_be_scoped_to_current_context(self, plugin_client, mock_config):
+ mock_response = MagicMock(status_code=200)
+
+ with (
+ patch("core.plugin.impl.base.plugin_daemon_request_timeout", httpx.Timeout(600.0)),
+ patch("httpx.request", return_value=mock_response, autospec=True) as mock_request,
+ ):
+ with use_plugin_daemon_request_timeout(30.0):
+ plugin_client._request("GET", "plugin/test-tenant/test")
+ plugin_client._request("GET", "plugin/test-tenant/test")
+
+ scoped_timeout = mock_request.call_args_list[0].kwargs["timeout"]
+ default_timeout = mock_request.call_args_list[1].kwargs["timeout"]
+ assert isinstance(scoped_timeout, httpx.Timeout)
+ assert isinstance(default_timeout, httpx.Timeout)
+ assert scoped_timeout.read == 30.0
+ assert default_timeout.read == 600.0
+
+ @pytest.mark.parametrize("timeout_seconds", [0.0, -1.0])
+ def test_request_timeout_override_rejects_non_positive_values(self, timeout_seconds):
+ with pytest.raises(ValueError, match="greater than zero"):
+ with use_plugin_daemon_request_timeout(timeout_seconds):
+ pass
+
def test_request_connection_error(self, plugin_client, mock_config):
"""Test handling of connection errors during request."""
# Arrange
diff --git a/api/tests/unit_tests/services/test_message_service.py b/api/tests/unit_tests/services/test_message_service.py
index e6e5c76a9ca..21b7093a6f7 100644
--- a/api/tests/unit_tests/services/test_message_service.py
+++ b/api/tests/unit_tests/services/test_message_service.py
@@ -950,6 +950,32 @@ class TestMessageServiceSuggestedQuestions:
model_config=None,
)
+ def test_agent_unavailable_default_model_returns_empty_questions(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ factory: MessageServiceTestDataFactory,
+ sqlite_session: Session,
+ ) -> None:
+ conversation = factory.create_conversation()
+ conversation.mode = AppMode.AGENT
+ model_manager, memory, llm_generator = self._chat_boundaries(monkeypatch, conversation)
+ model_manager.get_default_model_instance.side_effect = ValueError("unsupported default model")
+ roster_service = MagicMock()
+ roster_service.return_value.get_published_agent_soul_for_app.return_value = self._agent_soul()
+ monkeypatch.setattr("services.agent.roster_service.AgentRosterService", roster_service)
+
+ result = MessageService.get_suggested_questions_after_answer(
+ app_model=factory.create_app(mode=AppMode.AGENT),
+ user=factory.create_end_user(),
+ message_id="msg-123",
+ invoke_from=InvokeFrom.SERVICE_API,
+ session=sqlite_session,
+ )
+
+ assert result == []
+ memory.assert_not_called()
+ llm_generator.generate_suggested_questions_after_answer.assert_not_called()
+
def test_historical_agent_without_soul_uses_current_app_model_config(
self,
monkeypatch: pytest.MonkeyPatch,