Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(agent): send uploaded images as native multimodal input (#41685)
kenwoodjwcf34e3ee1a6f
diff --git a/api/clients/agent_backend/request_builder.py b/api/clients/agent_backend/request_builder.pyindex 2f4d09ae9d5..d7a3b8ddc38 100644--- a/api/clients/agent_backend/request_builder.py+++ b/api/clients/agent_backend/request_builder.py@@ -36,6 +36,11 @@ from dify_agent.layers.knowledge import DIFY_KNOWLEDGE_BASE_LAYER_TYPE_ID, DifyKfrom dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID, DifyOutputLayerConfigfrom dify_agent.layers.runtime import DIFY_RUNTIME_LAYER_TYPE_ID, DifyRuntimeLayerConfigfrom dify_agent.layers.shell import DIFY_SHELL_LAYER_TYPE_ID, DifyShellLayerConfig+from dify_agent.layers.user_prompt import (+ DIFY_USER_PROMPT_LAYER_TYPE_ID,+ DifyUserPromptFileConfig,+ DifyUserPromptLayerConfig,+)from dify_agent.protocol import (DIFY_AGENT_HISTORY_LAYER_ID,DIFY_AGENT_MODEL_LAYER_ID,@@ -246,6 +251,7 @@ class AgentBackendAgentAppRunInput(BaseModel):execution_context: DifyExecutionContextLayerConfigbackend_binding_ref: str = Field(min_length=1)user_prompt: str+ user_files: list[DifyUserPromptFileConfig] = Field(default_factory=list)agent_soul_prompt: str | None = Noneagent_config_version_kind: AgentConfigVersionKind = "snapshot"idempotency_key: str | None = None@@ -310,9 +316,9 @@ class AgentBackendRunRequestBuilder:[RunLayerSpec(name=AGENT_APP_USER_PROMPT_LAYER_ID,- type=PLAIN_PROMPT_LAYER_TYPE_ID,+ type=DIFY_USER_PROMPT_LAYER_TYPE_ID,metadata={**run_input.metadata, "origin": "agent_app_user_prompt"},- config=PromptLayerConfig(user=run_input.user_prompt),+ config=DifyUserPromptLayerConfig(text=run_input.user_prompt, files=run_input.user_files),),RunLayerSpec(name=DIFY_EXECUTION_CONTEXT_LAYER_ID,@@ -632,7 +638,7 @@ class AgentBackendRunRequestBuilder:)-_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key")+_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key", "base64_data")def redact_for_agent_backend_log(value: object) -> object:@@ -641,9 +647,10 @@ def redact_for_agent_backend_log(value: object) -> object:return redact_for_agent_backend_log(value.model_dump(mode="json", warnings=False))if isinstance(value, dict):redacted: dict[object, object] = {}+ is_multimodal_file = value.get("type") == "image" and "filename" in value and "mime_type" in valuefor key, item in value.items():key_text = str(key).lower()- if any(part in key_text for part in _SENSITIVE_KEY_PARTS):+ if any(part in key_text for part in _SENSITIVE_KEY_PARTS) or (is_multimodal_file and key_text == "url"):redacted[key] = "[REDACTED]"else:redacted[key] = redact_for_agent_backend_log(item)diff --git a/api/core/app/apps/agent_app/app_generator.py b/api/core/app/apps/agent_app/app_generator.pyindex 5381cd4b1c3..2d58ca1b4e4 100644--- a/api/core/app/apps/agent_app/app_generator.py+++ b/api/core/app/apps/agent_app/app_generator.py@@ -11,15 +11,13 @@ changes the runtime exit policy carried to the backend.from __future__ import annotationsimport contextvars-import jsonimport loggingimport threadingimport uuid-from collections.abc import Generator, Mapping, Sequence+from collections.abc import Generator, Mappingfrom typing import Any, Literalfrom flask import Flask, current_app-from pydantic import JsonValuefrom sqlalchemy import and_, or_, selectfrom sqlalchemy.orm import Session@@ -29,6 +27,7 @@ from configs import dify_configfrom constants import UUID_NILfrom core.agent.publish_visibility import agent_has_workflow_callable_active_snapshotfrom core.app.app_config.easy_ui_based_app.model_config.converter import ModelConfigConverter+from core.app.app_config.features.file_upload.manager import FileUploadConfigManagerfrom core.app.apps.agent_app.app_config_manager import AgentAppConfigManagerfrom core.app.apps.agent_app.app_runner import AgentAppRunnerfrom core.app.apps.agent_app.errors import (@@ -52,8 +51,8 @@ from core.app.entities.app_invoke_entities import (from core.credit_usage import CreditUsageAppTypefrom core.db.session_factory import session_factoryfrom core.ops.ops_trace_manager import TraceQueueManager-from core.workflow.file_reference import build_file_reference, is_canonical_file_referencefrom extensions.ext_database import db+from factories import file_factoryfrom models import Account, App, AppModelConfig, Conversation, EndUser, Message, MessageAnnotationfrom models.agent import (APP_BACKED_AGENT_SOURCES,@@ -75,69 +74,6 @@ from services.conversation_service import ConversationServicelogger = logging.getLogger(__name__)-_REFERENCE_FILE_TRANSFER_METHODS = {"local_file", "tool_file", "datasource_file"}---def _append_prompt_file_mappings(query: str, prompt_file_mappings: Sequence[JsonValue]) -> str:- """Append labeled, prompt-safe file locators to the backend user prompt."""- prompt_files = _prompt_file_locators(prompt_file_mappings)- if not prompt_files:- return query- payload = json.dumps(prompt_files, ensure_ascii=False, separators=(",", ":"))- return (- f"{query}\n"- "User provided files: use dify-agent file download with the listed transfer_method and reference/url "- "to get the files and investigate them\n"- f"{payload}"- )---def _prompt_file_locators(prompt_file_mappings: Sequence[JsonValue]) -> list[dict[str, str]]:- locators: list[dict[str, str]] = []- for file_mapping in prompt_file_mappings:- if not isinstance(file_mapping, Mapping):- continue- locator = _prompt_file_locator(file_mapping)- if locator is not None:- locators.append(locator)- return locators---def _prompt_file_locator(file_mapping: Mapping[str, object]) -> dict[str, str] | None:- transfer_method = _string_value(file_mapping, "transfer_method")- if transfer_method == "remote_url":- url = _string_value(file_mapping, "url") or _string_value(file_mapping, "remote_url")- if url is None:- return None- return {"transfer_method": "remote_url", "url": url}- elif transfer_method in _REFERENCE_FILE_TRANSFER_METHODS:- if transfer_method is None:- return None- reference = _canonical_file_reference(- _string_value(file_mapping, "reference")- or _string_value(file_mapping, "upload_file_id")- or _string_value(file_mapping, "file_id")- or _string_value(file_mapping, "id")- )- if reference is None:- return None- return {"transfer_method": transfer_method, "reference": reference}- else:- return None---def _canonical_file_reference(reference: str | None) -> str | None:- if reference is None:- return None- if reference.startswith("dify-file-ref:"):- return reference if is_canonical_file_reference(reference) else None- return build_file_reference(record_id=reference)---def _string_value(mapping: Mapping[str, object], key: str) -> str | None:- value = mapping.get(key)- return value if isinstance(value, str) and value else None-class AgentAppGenerator(MessageBasedAppGenerator):def generate(@@ -155,7 +91,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):query = self._require_query(args)inputs = args["inputs"]- prompt_file_mappings = args.get("files") or []+ raw_files = args.get("files") or []conversation = Noneconversation_id = args.get("conversation_id")@@ -193,19 +129,33 @@ class AgentAppGenerator(MessageBasedAppGenerator):conversation=conversation,)model_conf = ModelConfigConverter.convert(app_config)+ with self._bind_file_access_scope(tenant_id=app_model.tenant_id, user=user, invoke_from=invoke_from):+ file_upload_config = (+ FileUploadConfigManager.convert(app_config.app_model_config_dict, is_vision=True) if raw_files else None+ )+ file_objs = (+ file_factory.build_from_mappings(+ mappings=raw_files,+ tenant_id=app_model.tenant_id,+ config=file_upload_config,+ access_controller=self._file_access_controller,+ )+ if raw_files and file_upload_config is not None+ else []+ )trace_manager = TraceQueueManager(app_model.id, user.id if isinstance(user, Account) else user.session_id)application_generate_entity = AgentAppGenerateEntity(task_id=str(uuid.uuid4()),app_config=app_config,model_conf=model_conf,+ file_upload_config=file_upload_config,conversation_id=conversation.id if conversation else None,inputs=self._prepare_user_inputs(user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id),query=query,- files=[],- prompt_file_mappings=prompt_file_mappings,+ files=list(file_objs),parent_message_id=(args.get("parent_message_id")if invoke_from not in {InvokeFrom.SERVICE_API, InvokeFrom.OPENAPI}@@ -492,10 +442,6 @@ class AgentAppGenerator(MessageBasedAppGenerator):)if handled:return- query = _append_prompt_file_mappings(- query=query,- prompt_file_mappings=application_generate_entity.prompt_file_mappings,- )dify_context = DifyRunContext(tenant_id=app_config.tenant_id,@@ -514,6 +460,14 @@ class AgentAppGenerator(MessageBasedAppGenerator):)runner = self._build_runner()+ image_detail_config = (+ application_generate_entity.file_upload_config.image_config.detail+ if (+ application_generate_entity.file_upload_config+ and application_generate_entity.file_upload_config.image_config+ )+ else None+ )runner.run(dify_context=dify_context,agent_id=application_generate_entity.agent_id,@@ -526,6 +480,8 @@ class AgentAppGenerator(MessageBasedAppGenerator):message_id=message.id,model_name=application_generate_entity.model_conf.model,queue_manager=queue_manager,+ files=list(application_generate_entity.files),+ image_detail_config=image_detail_config,session_scope_snapshot_id=application_generate_entity.agent_session_scope_config_version_id,build_draft_id=(application_generate_entity.agent_config_snapshot_iddiff --git a/api/core/app/apps/agent_app/app_runner.py b/api/core/app/apps/agent_app/app_runner.pyindex 665af35a035..f155c07a333 100644--- a/api/core/app/apps/agent_app/app_runner.py+++ b/api/core/app/apps/agent_app/app_runner.py@@ -55,8 +55,14 @@ from core.repositories.human_input_repository import HumanInputFormRepository, Hfrom core.workflow.nodes.agent_v2.ask_human_hitl import AskHumanFormBuildError, create_ask_human_formfrom core.workflow.nodes.agent_v2.ask_human_resume import build_deferred_tool_results, resolve_ask_human_formfrom extensions.ext_database import db+from graphon.file import Filefrom graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage-from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage+from graphon.model_runtime.entities.message_entities import (+ AssistantPromptMessage,+ ImagePromptMessageContent,+ PromptMessage,+ UserPromptMessage,+)from graphon.model_runtime.errors.invoke import (InvokeAuthorizationError,InvokeBadRequestError,@@ -643,6 +649,8 @@ class AgentAppRunner:message_id: str,model_name: str,queue_manager: AppQueueManager,+ files: list[File] | None = None,+ image_detail_config: ImagePromptMessageContent.DETAIL | None = None,session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId = _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID,build_draft_id: str | None = None,) -> None:@@ -669,6 +677,8 @@ class AgentAppRunner:backend_binding_ref=stored.backend_binding_ref,conversation_id=conversation_id,query=query,+ files=tuple(files or ()),+ image_detail_config=image_detail_config,idempotency_key=message_id,stored=stored,message_id=message_id,@@ -796,6 +806,8 @@ class AgentAppRunner:backend_binding_ref: str,conversation_id: str,query: str,+ files: tuple[File, ...],+ image_detail_config: ImagePromptMessageContent.DETAIL | None,idempotency_key: str,stored: StoredAgentAppSession,message_id: str | None,@@ -815,6 +827,8 @@ class AgentAppRunner:agent_soul=agent_soul,conversation_id=conversation_id,user_query=query,+ files=files,+ image_detail_config=image_detail_config,idempotency_key=idempotency_key,binding_id=binding_id,backend_binding_ref=backend_binding_ref,diff --git a/api/core/app/apps/agent_app/runtime_request_builder.py b/api/core/app/apps/agent_app/runtime_request_builder.pyindex 70c813d0fd5..5f05b0c99d6 100644--- a/api/core/app/apps/agent_app/runtime_request_builder.py+++ b/api/core/app/apps/agent_app/runtime_request_builder.py@@ -19,6 +19,12 @@ from dify_agent.layers.execution_context import (DifyExecutionContextLayerConfig,DifyExecutionContextUserFrom,)+from dify_agent.layers.user_prompt import (+ DifyUserPromptDownloadConfig,+ DifyUserPromptFileConfig,+ DifyUserPromptFileType,+ DifyUserPromptImageConfig,+)from dify_agent.protocol import CreateRunRequest, DeferredToolResultsPayloadfrom clients.agent_backend import (@@ -29,8 +35,9 @@ from clients.agent_backend import ()from configs import dify_configfrom core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom-from core.app.llm.model_access import resolve_model_context_window+from core.app.llm.model_access import resolve_model_context_window, resolve_model_supports_visionfrom core.plugin.provider_identity import normalize_plugin_daemon_provider_identity+from core.workflow.file_reference import build_file_reference, is_canonical_file_referencefrom core.workflow.nodes.agent_v2.dify_tools_builder import (WorkflowAgentDifyToolLayersBuilder,WorkflowAgentDifyToolsBuilder,@@ -46,6 +53,8 @@ from core.workflow.nodes.agent_v2.runtime_request_builder import (build_shell_layer_config,load_runtime_agent_skill_configs,)+from graphon.file import File, FileTransferMethod, FileType, file_manager+from graphon.model_runtime.entities.message_entities import ImagePromptMessageContentfrom models.agent_config_entities import AgentSoulConfig, AgentSoulToolsConfigfrom models.provider_ids import ModelProviderIDfrom services.agent.prompt_mentions import expand_prompt_mentions@@ -61,6 +70,9 @@ class AgentAppRuntimeRequestBuildError(ValueError):super().__init__(message)+type _ReferenceFileTransferMethod = Literal["local_file", "tool_file", "datasource_file"]++@dataclass(frozen=True, slots=True)class AgentAppRuntimeBuildContext:dify_context: DifyRunContext@@ -72,6 +84,8 @@ class AgentAppRuntimeBuildContext:idempotency_key: strbinding_id: strbackend_binding_ref: str+ files: tuple[File, ...] = ()+ image_detail_config: ImagePromptMessageContent.DETAIL | None = Noneagent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"session_snapshot: CompositorSessionSnapshot | None = None# ENG-638: set when resuming a chat turn after a submitted ask_human form.@@ -150,6 +164,13 @@ class AgentAppRuntimeRequestBuilder:ModelProviderID(agent_soul.model.model_provider),agent_soul.model.plugin_id,)+ user_files = self._build_user_files(+ files=context.files,+ run_context=context.dify_context,+ provider_name=agent_soul.model.model_provider,+ model_name=agent_soul.model.model,+ image_detail_config=context.image_detail_config,+ )request = self._request_builder.build_for_agent_app(AgentBackendAgentAppRunInput(@@ -180,6 +201,7 @@ class AgentAppRuntimeRequestBuilder:or None,agent_config_version_kind=context.agent_config_version_kind,user_prompt=context.user_query,+ user_files=user_files,tools=tool_layers.plugin_tools,core_tools=tool_layers.core_tools,knowledge=knowledge_config,@@ -202,6 +224,27 @@ class AgentAppRuntimeRequestBuilder:binding_id=context.binding_id,)+ @staticmethod+ def _build_user_files(+ *,+ files: tuple[File, ...],+ run_context: DifyRunContext,+ provider_name: str,+ model_name: str,+ image_detail_config: ImagePromptMessageContent.DETAIL | None,+ ) -> list[DifyUserPromptFileConfig]:+ supports_vision = any(file.type == FileType.IMAGE for file in files) and resolve_model_supports_vision(+ run_context=run_context,+ provider_name=provider_name,+ model_name=model_name,+ )+ return [+ _build_user_image(file, image_detail_config=image_detail_config)+ if supports_vision and file.type == FileType.IMAGE+ else _build_user_download(file)+ for file in files+ ]+@staticmethoddef _validate_session_snapshot_layers(request: CreateRunRequest) -> None:"""Reject stale snapshots before they reach the Agent backend.@@ -251,6 +294,57 @@ class AgentAppRuntimeRequestBuilder:}+def _build_user_image(+ file: File,+ *,+ image_detail_config: ImagePromptMessageContent.DETAIL | None,+) -> DifyUserPromptImageConfig:+ content = file_manager.to_prompt_message_content(file, image_detail_config=image_detail_config)+ if not isinstance(content, ImagePromptMessageContent):+ raise AgentAppRuntimeRequestBuildError(+ "agent_user_file_unsupported",+ f"Agent App cannot send file '{file.filename or 'image'}' as vision content.",+ )+ detail = content.detail.value+ return DifyUserPromptImageConfig(+ filename=content.filename or file.filename or f"image.{content.format}",+ mime_type=content.mime_type,+ format=content.format,+ url=content.url or None,+ base64_data=content.base64_data or None,+ detail=detail if detail in {"low", "high"} else None,+ )+++def _build_user_download(file: File) -> DifyUserPromptDownloadConfig:+ file_type = cast(DifyUserPromptFileType, file.type.value)+ if file.transfer_method == FileTransferMethod.REMOTE_URL:+ if file.remote_url is None:+ raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "Remote user file is missing its URL.")+ return DifyUserPromptDownloadConfig(type=file_type, transfer_method="remote_url", url=file.remote_url)+ if file.reference is None:+ raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "User file is missing its reference.")+ reference = file.reference+ if not reference.startswith("dify-file-ref:"):+ reference = build_file_reference(record_id=reference)+ elif not is_canonical_file_reference(reference):+ raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "User file reference is invalid.")+ transfer_method: _ReferenceFileTransferMethod+ match file.transfer_method:+ case FileTransferMethod.LOCAL_FILE:+ transfer_method = "local_file"+ case FileTransferMethod.TOOL_FILE:+ transfer_method = "tool_file"+ case FileTransferMethod.DATASOURCE_FILE:+ transfer_method = "datasource_file"+ case _:+ raise AgentAppRuntimeRequestBuildError(+ "agent_user_file_invalid",+ f"User file transfer method '{file.transfer_method.value}' is unsupported.",+ )+ return DifyUserPromptDownloadConfig(type=file_type, transfer_method=transfer_method, reference=reference)++__all__ = ["AgentAppRuntimeBuildContext","AgentAppRuntimeRequest",diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.pyindex 7f983135994..cb3f52d12a0 100644--- a/api/core/app/entities/app_invoke_entities.py+++ b/api/core/app/entities/app_invoke_entities.py@@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequencefrom enum import StrEnumfrom typing import TYPE_CHECKING, Any, Literal-from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationInfo, field_validator+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validatorfrom constants import UUID_NILfrom core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAppConfig@@ -286,16 +286,15 @@ class AgentAppGenerateEntity(ChatAppGenerateEntity):``agent_session_scope_config_version_id`` identifies the draft or immutableconfig version whose Workspace Binding should be reused for this session.- ``prompt_file_mappings`` preserves the raw request ``files`` array for the- Agent backend prompt. These references are appended to the backend prompt- text while the stored chat message keeps the user's original query.+ Uploaded files use the inherited ``files`` field. The Agent App runtime+ sends supported images directly to vision models and preserves the sandbox+ locator fallback for all other files."""agent_id: stragent_config_snapshot_id: stragent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"agent_session_scope_config_version_id: str | None = None- prompt_file_mappings: Sequence[JsonValue] = Field(default_factory=list)agent_llm_gateway_enabled: bool = Falsediff --git a/api/core/app/llm/model_access.py b/api/core/app/llm/model_access.pyindex 52fb9add70b..e793200ffb7 100644--- a/api/core/app/llm/model_access.py+++ b/api/core/app/llm/model_access.py@@ -9,7 +9,7 @@ from core.errors.error import ProviderTokenNotInitErrorfrom core.model_manager import ModelInstance, ModelManagerfrom core.plugin.impl.model_runtime_factory import create_plugin_provider_managerfrom core.provider_manager import ProviderManager-from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType+from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey, ModelTypefrom graphon.nodes.llm.entities import ModelConfigfrom graphon.nodes.llm.exc import LLMModeRequiredError, ModelNotExistErrorfrom graphon.nodes.llm.protocols import CredentialsProvider@@ -149,6 +149,18 @@ def resolve_model_context_window(return context_window+def resolve_model_supports_vision(+ *,+ run_context: DifyRunContext,+ provider_name: str,+ model_name: str,+) -> bool:+ """Return whether the credential-bound model advertises vision support."""++ model_instance = DifyModelFactory(run_context=run_context).init_model_instance(provider_name, model_name)+ return ModelFeature.VISION in (model_instance.get_model_schema().features or [])++def _normalize_completion_params(completion_params: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:"""Split node-level completion params into provider parameters and stop sequences.diff --git a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py b/api/tests/unit_tests/clients/agent_backend/test_request_builder.pyindex 978b522d509..29771083de6 100644--- a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py+++ b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py@@ -20,6 +20,11 @@ from dify_agent.layers.execution_context import DIFY_EXECUTION_CONTEXT_LAYER_TYPfrom dify_agent.layers.knowledge import DIFY_KNOWLEDGE_BASE_LAYER_TYPE_ID, DifyKnowledgeBaseLayerConfigfrom dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_IDfrom dify_agent.layers.shell import DIFY_SHELL_LAYER_TYPE_ID, DifyShellEnvVarConfig, DifyShellLayerConfig+from dify_agent.layers.user_prompt import (+ DIFY_USER_PROMPT_LAYER_TYPE_ID,+ DifyUserPromptDownloadConfig,+ DifyUserPromptImageConfig,+)from dify_agent.protocol import (DIFY_AGENT_HISTORY_LAYER_ID,DIFY_AGENT_MODEL_LAYER_ID,@@ -367,6 +372,64 @@ def test_agent_app_request_builder_omits_shell_layer_by_default():assert DIFY_SHELL_LAYER_ID not in {layer.name for layer in request.composition.layers}+def test_agent_app_request_builder_emits_multimodal_user_prompt_layer():+ run_input = _agent_app_input()+ run_input.user_files = [+ DifyUserPromptDownloadConfig(+ type="document", transfer_method="remote_url", url="https://example.com/brief.pdf"+ ),+ DifyUserPromptImageConfig(+ filename="earth.png",+ mime_type="image/png",+ format="png",+ url="https://files.example.com/earth.png?sign=secret",+ detail="high",+ ),+ ]++ request = AgentBackendRunRequestBuilder().build_for_agent_app(run_input)+ layer = next(layer for layer in request.composition.layers if layer.name == "agent_app_user_prompt")++ assert layer.type == DIFY_USER_PROMPT_LAYER_TYPE_ID+ assert layer.config.text == "List files."+ assert layer.config.files == run_input.user_files+ restored_request = CreateRunRequest.model_validate_json(request.model_dump_json())+ restored_layer = next(+ layer for layer in restored_request.composition.layers if layer.name == "agent_app_user_prompt"+ )+ assert restored_layer.config == layer.config.model_dump(mode="json")+ assert "locators" not in restored_layer.config+++def test_agent_backend_log_redacts_multimodal_file_transport():+ run_input = _agent_app_input()+ run_input.metadata = {"source_url": "https://example.com/docs"}+ run_input.user_files = [+ DifyUserPromptImageConfig(+ filename="earth.png",+ mime_type="image/png",+ format="png",+ url="https://files.example.com/earth.png?sign=secret",+ ),+ DifyUserPromptImageConfig(+ filename="inline.png",+ mime_type="image/png",+ format="png",+ base64_data="aW1hZ2UtYnl0ZXM=",+ ),+ ]++ redacted = cast(+ dict[str, Any],+ redact_for_agent_backend_log(AgentBackendRunRequestBuilder().build_for_agent_app(run_input)),+ )+ layer = next(item for item in redacted["composition"]["layers"] if item["name"] == "agent_app_user_prompt")++ assert layer["config"]["files"][0]["url"] == "[REDACTED]"+ assert layer["config"]["files"][1]["base64_data"] == "[REDACTED]"+ assert redacted["metadata"]["source_url"] == "https://example.com/docs"++def test_agent_app_request_builder_keeps_build_draft_prompt_when_agent_soul_prompt_is_blank():run_input = _agent_app_input().model_copy(update={"agent_soul_prompt": " ", "agent_config_version_kind": "build_draft"}diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.pyindex 410efadba33..4a84bcc4689 100644--- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py@@ -11,7 +11,6 @@ from __future__ import annotationsimport contextlibimport inspect-import jsonfrom decimal import Decimalfrom types import SimpleNamespace@@ -29,7 +28,6 @@ from core.app.apps.agent_app.errors import AgentSessionSnapshotIncompatibleErrorfrom core.app.apps.exc import GenerateTaskStoppedErrorfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFromfrom core.app.entities.queue_entities import QueueAnnotationReplyEvent-from core.workflow.file_reference import build_file_referencefrom models import Account, AppModelConfigfrom models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatusfrom models.agent_config_entities import AgentSoulConfig@@ -209,6 +207,10 @@ class TestGenerateSuccess:return_value=mocker.MagicMock(variables=[], tenant_id="tenant", app_id="app1"),)mocker.patch(f"{MODULE}.ModelConfigConverter.convert", return_value=mocker.MagicMock(model="gpt-4o-mini"))+ file_upload_config = mocker.MagicMock()+ mocker.patch(f"{MODULE}.FileUploadConfigManager.convert", return_value=file_upload_config)+ parsed_file = mocker.MagicMock()+ build_files = mocker.patch(f"{MODULE}.file_factory.build_from_mappings", return_value=[parsed_file])mocker.patch(f"{MODULE}.TraceQueueManager", return_value=mocker.MagicMock())generate_entity = mocker.patch(f"{MODULE}.AgentAppGenerateEntity", return_value=mocker.MagicMock(task_id="t", user_id="user")@@ -248,7 +250,10 @@ class TestGenerateSuccess:conversation=None,)assert session.get(AppModelConfig, "config-1") is config- assert generate_entity.call_args.kwargs["prompt_file_mappings"] == file_mappings+ build_files.assert_called_once()+ assert build_files.call_args.kwargs["mappings"] == file_mappings+ assert generate_entity.call_args.kwargs["files"] == [parsed_file]+ assert generate_entity.call_args.kwargs["file_upload_config"] is file_upload_configassert "agent_runtime_exit_intent" not in generate_entity.call_args.kwargsdef test_generate_loads_existing_conversation(self, generator: AgentAppGenerator, mocker: MockerFixture):@@ -379,7 +384,8 @@ class TestGenerateWorker:is_resume=False,query="query",session_scope_config_version_id="s",- prompt_file_mappings=(),+ files=(),+ file_upload_config=None,):generator._generate_worker(flask_app=mocker.MagicMock(),@@ -391,7 +397,8 @@ class TestGenerateWorker:agent_session_scope_config_version_id=session_scope_config_version_id,model_conf=mocker.MagicMock(model="m"),query=query,- prompt_file_mappings=prompt_file_mappings,+ files=files,+ file_upload_config=file_upload_config,),queue_manager=queue_manager,conversation_id="conv",@@ -420,48 +427,22 @@ class TestGenerateWorker:assert runner.run.call_args.kwargs["agent_config_snapshot_id"] == "s"assert runner.run.call_args.kwargs["session_scope_snapshot_id"] is None- def test_worker_appends_prompt_files_to_backend_query(self, generator, mocker: MockerFixture):+ def test_worker_passes_files_to_backend_runner_without_rewriting_query(self, generator, mocker: MockerFixture):runner, _ = self._wire(generator, mocker, guard_query="你看得见这张图片吗")queue_manager = mocker.MagicMock()- file_mappings = [- {- "type": "image",- "transfer_method": "local_file",- "url": "",- "upload_file_id": "upload-file-1",- },- {- "type": "document",- "transfer_method": "remote_url",- "url": "https://example.com/source.pdf",- "upload_file_id": "ignored",- },- ]- expected_file_mappings = [- {- "transfer_method": "local_file",- "reference": build_file_reference(record_id="upload-file-1"),- },- {- "transfer_method": "remote_url",- "url": "https://example.com/source.pdf",- },- ]+ files = [mocker.MagicMock(), mocker.MagicMock()]self._call(generator,mocker,queue_manager,query="你看得见这张图片吗",- prompt_file_mappings=file_mappings,+ files=files,)- assert runner.run.call_args.kwargs["query"] == (- "你看得见这张图片吗\nUser provided files: "- "use dify-agent file download with the listed transfer_method and reference/url "- "to get the files and investigate them\n"- f"{json.dumps(expected_file_mappings, ensure_ascii=False, separators=(',', ':'))}"- )+ assert runner.run.call_args.kwargs["query"] == "你看得见这张图片吗"+ assert runner.run.call_args.kwargs["files"] == files+ assert runner.run.call_args.kwargs["image_detail_config"] is Nonedef test_input_guard_short_circuit_skips_backend(self, generator, mocker: MockerFixture):runner, _ = self._wire(generator, mocker, handled=True)diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.pyindex db33a1ad03b..b93f0e6ff90 100644--- a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py@@ -12,6 +12,7 @@ from dify_agent.layers.config import DifyConfigSkillConfigfrom dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolsLayerConfigfrom dify_agent.layers.dify_plugin import DifyPluginToolConfig, DifyPluginToolsLayerConfigfrom dify_agent.layers.execution_context import DifyExecutionContextLayerConfig+from dify_agent.layers.user_prompt import DifyUserPromptLayerConfigfrom clients.agent_backend import (DIFY_CONFIG_LAYER_ID,@@ -29,6 +30,9 @@ from core.app.apps.agent_app.runtime_request_builder import (AgentAppRuntimeRequestBuildError,)from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom+from core.workflow.file_reference import build_file_reference+from graphon.file import File, FileTransferMethod, FileType+from graphon.model_runtime.entities.message_entities import ImagePromptMessageContentfrom models.agent_config_entities import AgentSoulConfigfrom tests.unit_tests.config_override import apply_config_overrides@@ -168,6 +172,8 @@ def _ctx(query: str = "hello",agent_config_version_kind: str = "snapshot",session_snapshot: CompositorSessionSnapshot | None = None,+ files: tuple[File, ...] = (),+ image_detail_config: ImagePromptMessageContent.DETAIL | None = None,) -> AgentAppRuntimeBuildContext:dify_context = SimpleNamespace(tenant_id="tenant-1",@@ -188,6 +194,8 @@ def _ctx(backend_binding_ref="binding-ref-1",agent_config_version_kind=agent_config_version_kind, # type: ignore[arg-type]session_snapshot=session_snapshot,+ files=files,+ image_detail_config=image_detail_config,)@@ -204,6 +212,32 @@ def _soul_with_model() -> AgentSoulConfig:)+def _image_file() -> File:+ return File(+ file_id="file-1",+ file_type=FileType.IMAGE,+ transfer_method=FileTransferMethod.LOCAL_FILE,+ reference="upload-file-1",+ filename="earth.png",+ extension=".png",+ mime_type="image/png",+ size=12,+ )+++def _document_file() -> File:+ return File(+ file_id="file-2",+ file_type=FileType.DOCUMENT,+ transfer_method=FileTransferMethod.LOCAL_FILE,+ reference="upload-document-1",+ filename="brief.pdf",+ extension=".pdf",+ mime_type="application/pdf",+ size=24,+ )++def _snapshot_for_layer_names(layer_names: list[str]) -> CompositorSessionSnapshot:return CompositorSessionSnapshot(layers=[@@ -251,6 +285,135 @@ class TestAgentAppRuntimeRequestBuilder:assert "credentials" not in result.redacted_request["composition"]["layers"][-1]["config"]assert result.metadata["conversation_id"] == "conv-1"+ def test_build_sends_images_directly_to_vision_model(self, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",+ lambda **_kwargs: True,+ )+ prompt_content_calls: list[tuple[File, ImagePromptMessageContent.DETAIL | None]] = []++ def to_prompt_message_content(+ file: File,+ *,+ image_detail_config: ImagePromptMessageContent.DETAIL | None,+ ) -> ImagePromptMessageContent:+ prompt_content_calls.append((file, image_detail_config))+ return ImagePromptMessageContent(+ format="png",+ url="https://files.example.com/earth.png?sign=secret",+ mime_type="image/png",+ filename="earth.png",+ detail=image_detail_config or ImagePromptMessageContent.DETAIL.LOW,+ )++ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content",+ to_prompt_message_content,+ )+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]++ result = builder.build(+ _ctx(+ _soul_with_model(),+ query="Describe this image.",+ files=(_image_file(),),+ image_detail_config=ImagePromptMessageContent.DETAIL.HIGH,+ )+ )+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")+ config = DifyUserPromptLayerConfig.model_validate(layer.config)++ assert config.text == "Describe this image."+ assert len(config.files) == 1+ image = config.files[0]+ assert image.delivery == "multimodal"+ assert image.type == "image"+ assert image.url == "https://files.example.com/earth.png?sign=secret"+ assert image.detail == "high"+ assert prompt_content_calls == [(_image_file(), ImagePromptMessageContent.DETAIL.HIGH)]++ def test_build_keeps_image_locator_for_non_vision_model(self, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",+ lambda **_kwargs: False,+ )+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]++ result = builder.build(_ctx(_soul_with_model(), query="Inspect the attachment.", files=(_image_file(),)))+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")+ config = DifyUserPromptLayerConfig.model_validate(layer.config)++ assert config.text == "Inspect the attachment."+ assert [file.model_dump(exclude_none=True) for file in config.files] == [+ {+ "delivery": "download",+ "type": "image",+ "transfer_method": "local_file",+ "reference": build_file_reference(record_id="upload-file-1"),+ }+ ]++ def test_build_preserves_inline_base64_transport_for_vision_model(self, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",+ lambda **_kwargs: True,+ )+ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content",+ lambda *_args, **_kwargs: ImagePromptMessageContent(+ format="png",+ base64_data="aW1hZ2UtYnl0ZXM=",+ mime_type="image/png",+ filename="earth.png",+ detail="low",+ ),+ )+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]++ result = builder.build(_ctx(_soul_with_model(), query="Describe this image.", files=(_image_file(),)))+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")+ config = DifyUserPromptLayerConfig.model_validate(layer.config)++ image = config.files[0]+ assert image.delivery == "multimodal"+ assert image.url is None+ assert image.base64_data == "aW1hZ2UtYnl0ZXM="++ def test_build_keeps_non_image_locator_when_vision_image_is_direct(self, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",+ lambda **_kwargs: True,+ )+ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content",+ lambda *_args, **_kwargs: ImagePromptMessageContent(+ format="png",+ url="https://files.example.com/earth.png",+ mime_type="image/png",+ filename="earth.png",+ detail="low",+ ),+ )+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]++ result = builder.build(+ _ctx(_soul_with_model(), files=(_image_file(), _document_file()), query="Compare the attachments.")+ )+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")+ config = DifyUserPromptLayerConfig.model_validate(layer.config)++ assert config.text == "Compare the attachments."+ assert len(config.files) == 2+ image, download = config.files+ assert image.delivery == "multimodal"+ assert image.filename == "earth.png"+ assert download.model_dump(exclude_none=True) == {+ "delivery": "download",+ "type": "document",+ "transfer_method": "local_file",+ "reference": build_file_reference(record_id="upload-document-1"),+ }+@pytest.mark.parametrize(("previous_prompt", "current_prompt"),[("", "You are Iris."), ("You are Iris.", "")],diff --git a/api/tests/unit_tests/core/app/llm/test_model_access.py b/api/tests/unit_tests/core/app/llm/test_model_access.pyindex 8c22d56223e..9bb0ef37712 100644--- a/api/tests/unit_tests/core/app/llm/test_model_access.py+++ b/api/tests/unit_tests/core/app/llm/test_model_access.py@@ -5,10 +5,15 @@ import pytestfrom core.app.entities.app_invoke_entities import DifyRunContextfrom core.app.llm import model_access-from graphon.model_runtime.entities.model_entities import ModelPropertyKey+from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey-def _stub_model_factory(monkeypatch: pytest.MonkeyPatch, context_window: object) -> dict[str, object]:+def _stub_model_factory(+ monkeypatch: pytest.MonkeyPatch,+ context_window: object,+ *,+ features: list[ModelFeature] | None = None,+) -> dict[str, object]:calls: dict[str, object] = {}class FakeModelFactory:@@ -18,7 +23,10 @@ def _stub_model_factory(monkeypatch: pytest.MonkeyPatch, context_window: object)def init_model_instance(self, provider_name: str, model_name: str) -> object:calls["provider_name"] = provider_namecalls["model_name"] = model_name- schema = SimpleNamespace(model_properties={ModelPropertyKey.CONTEXT_SIZE: context_window})+ schema = SimpleNamespace(+ model_properties={ModelPropertyKey.CONTEXT_SIZE: context_window},+ features=features,+ )return SimpleNamespace(get_model_schema=lambda: schema)monkeypatch.setattr(model_access, "DifyModelFactory", FakeModelFactory)@@ -58,3 +66,24 @@ def test_resolve_model_context_window_ignores_invalid_schema_values()is None)+++ ("features", "expected"),+ [([ModelFeature.VISION], True), ([], False), (None, False)],+)+def test_resolve_model_supports_vision_reads_selected_model_schema(+ monkeypatch: pytest.MonkeyPatch,+ features: list[ModelFeature] | None,+ expected: bool,+) -> None:+ _stub_model_factory(monkeypatch, 128_000, features=features)++ assert (+ model_access.resolve_model_supports_vision(+ run_context=cast(DifyRunContext, object()),+ provider_name="openai",+ model_name="gpt-4o",+ )+ is expected+ )diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.mdindex 99ff7b0ac28..bc6eacee02f 100644--- a/dify-agent/docs/dify-agent/guide/index.md+++ b/dify-agent/docs/dify-agent/guide/index.md@@ -359,6 +359,49 @@ whose Agenton layers provide user input. With the MVP provider set, useeffective prompts are rejected during create-run validation before the run ispersisted or scheduled.+Agent App callers can use the `dify.user_prompt` layer to send text and images+in the same model turn. Each image must provide exactly one transport: an+HTTP(S) `url` or unprefixed Base64 data in `base64_data`. The image is passed as+structured multimodal content; it is not interpolated into `config.text`.++```json+{+ "name": "agent_app_user_prompt",+ "type": "dify.user_prompt",+ "config": {+ "text": "Describe this image.",+ "files": [+ {+ "delivery": "multimodal",+ "type": "image",+ "filename": "earth.png",+ "mime_type": "image/png",+ "format": "png",+ "url": "https://files.example.com/earth.png",+ "base64_data": null,+ "detail": "high"+ }+ ]+ }+}+```++All attachments use `config.files`. The `delivery` field selects how an attachment+is presented, independently of its file `type` (`image`, `document`, `audio`,+`video`, or `custom`):++- `delivery: "multimodal"` currently supports `type: "image"` only. Dify API+ chooses URL or Base64 transport according to `MULTIMODAL_SEND_FORMAT`.+- `delivery: "download"` preserves the original file `type`, including `image`+ when the selected model has no Vision feature. It contains either+ `transfer_method: "remote_url"` with `url`, or a `local_file`, `tool_file`, or+ `datasource_file` transfer method with a canonical `reference`.++The layer appends sandbox file-download instructions for download attachments+and adds multimodal attachments as structured model content. Callers keep+`config.text` as the original user text. Omitting `config.files` is equivalent+to an empty list.+The optional Pydantic AI history layer uses the reserved name `history` andpersists captured messages in session snapshots for later resume. Resume from aterminal event's `session_snapshot` using the same layer composition, names, anddiff --git a/dify-agent/src/dify_agent/layers/user_prompt/__init__.py b/dify-agent/src/dify_agent/layers/user_prompt/__init__.pynew file mode 100644index 00000000000..f675b1c0140--- /dev/null+++ b/dify-agent/src/dify_agent/layers/user_prompt/__init__.py@@ -0,0 +1,19 @@+"""Client-safe exports for the Agent App multimodal user prompt layer."""++from dify_agent.layers.user_prompt.configs import (+ DIFY_USER_PROMPT_LAYER_TYPE_ID,+ DifyUserPromptDownloadConfig,+ DifyUserPromptFileConfig,+ DifyUserPromptFileType,+ DifyUserPromptImageConfig,+ DifyUserPromptLayerConfig,+)++__all__ = [+ "DIFY_USER_PROMPT_LAYER_TYPE_ID",+ "DifyUserPromptDownloadConfig",+ "DifyUserPromptFileConfig",+ "DifyUserPromptFileType",+ "DifyUserPromptImageConfig",+ "DifyUserPromptLayerConfig",+]diff --git a/dify-agent/src/dify_agent/layers/user_prompt/configs.py b/dify-agent/src/dify_agent/layers/user_prompt/configs.pynew file mode 100644index 00000000000..fc9fa273e7d--- /dev/null+++ b/dify-agent/src/dify_agent/layers/user_prompt/configs.py@@ -0,0 +1,90 @@+"""Serializable DTOs for one Agent App multimodal user prompt."""++from __future__ import annotations++import base64+import binascii+from typing import Annotated, ClassVar, Final, Literal, Self++from pydantic import ConfigDict, Field, model_validator++from agenton.layers import LayerConfig+++DIFY_USER_PROMPT_LAYER_TYPE_ID: Final[str] = "dify.user_prompt"+type DifyUserPromptFileType = Literal["image", "document", "audio", "video", "custom"]+++class DifyUserPromptImageConfig(LayerConfig):+ """One image delivered directly to the configured vision model.++ Callers provide either an HTTP(S) URL or base64 bytes. Opaque Dify file+ references belong to the sandbox download flow and are not accepted here.+ """++ delivery: Literal["multimodal"] = "multimodal"+ type: Literal["image"] = "image"+ filename: str = Field(min_length=1)+ mime_type: str = Field(pattern=r"^image/[A-Za-z0-9.+-]+$")+ format: str = Field(min_length=1)+ url: str | None = None+ base64_data: str | None = None+ detail: Literal["low", "high"] | None = None++ model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")++ @model_validator(mode="after")+ def _validate_transport(self) -> Self:+ if (self.url is None) == (self.base64_data is None):+ raise ValueError("exactly one of url or base64_data must be provided")+ if self.url is not None and not self.url.startswith(("http://", "https://")):+ raise ValueError("url must use http or https")+ if self.base64_data is not None:+ try:+ base64.b64decode(self.base64_data, validate=True)+ except (ValueError, binascii.Error) as exc:+ raise ValueError("base64_data must be valid base64") from exc+ return self+++class DifyUserPromptDownloadConfig(LayerConfig):+ delivery: Literal["download"] = "download"+ type: DifyUserPromptFileType+ transfer_method: Literal["remote_url", "local_file", "tool_file", "datasource_file"]+ url: str | None = Field(default=None, min_length=1)+ reference: str | None = Field(default=None, min_length=1)++ model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")++ @model_validator(mode="after")+ def _validate_locator(self) -> Self:+ if self.transfer_method == "remote_url":+ if self.url is None or self.reference is not None:+ raise ValueError("remote_url requires url and must not include reference")+ elif self.reference is None or self.url is not None:+ raise ValueError("reference is required and url must not be provided")+ return self+++type DifyUserPromptFileConfig = Annotated[+ DifyUserPromptImageConfig | DifyUserPromptDownloadConfig, Field(discriminator="delivery")+]+++class DifyUserPromptLayerConfig(LayerConfig):+ """User text plus images that should be sent in the same model turn."""++ text: str = Field(min_length=1)+ files: list[DifyUserPromptFileConfig] = Field(default_factory=list)++ model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")+++__all__ = [+ "DIFY_USER_PROMPT_LAYER_TYPE_ID",+ "DifyUserPromptDownloadConfig",+ "DifyUserPromptFileConfig",+ "DifyUserPromptFileType",+ "DifyUserPromptImageConfig",+ "DifyUserPromptLayerConfig",+]diff --git a/dify-agent/src/dify_agent/layers/user_prompt/layer.py b/dify-agent/src/dify_agent/layers/user_prompt/layer.pynew file mode 100644index 00000000000..edcdf0a45bc--- /dev/null+++ b/dify-agent/src/dify_agent/layers/user_prompt/layer.py@@ -0,0 +1,91 @@+"""Turn serialized Agent App images into pydantic-ai user content."""++from __future__ import annotations++import base64+import json+from dataclasses import dataclass+from typing import ClassVar, assert_never++from pydantic_ai.messages import BinaryContent, ImageUrl, UserContent+from typing_extensions import Self, override++from agenton.layers import EmptyRuntimeState, NoLayerDeps, PydanticAILayer+from dify_agent.layers.user_prompt.configs import (+ DIFY_USER_PROMPT_LAYER_TYPE_ID,+ DifyUserPromptDownloadConfig,+ DifyUserPromptImageConfig,+ DifyUserPromptLayerConfig,+)+++@dataclass(slots=True)+class DifyUserPromptLayer(PydanticAILayer[NoLayerDeps, object, DifyUserPromptLayerConfig, EmptyRuntimeState]):+ """State-free layer for a text prompt and directly attached images."""++ type_id: ClassVar[str | None] = DIFY_USER_PROMPT_LAYER_TYPE_ID+ config: DifyUserPromptLayerConfig++ @classmethod+ @override+ def from_config(cls, config: DifyUserPromptLayerConfig) -> Self:+ return cls(config=DifyUserPromptLayerConfig.model_validate(config))++ @property+ @override+ def user_prompts(self) -> list[UserContent]:+ images: list[UserContent] = []+ downloads: list[DifyUserPromptDownloadConfig] = []+ for file in self.config.files:+ match file.delivery:+ case "multimodal":+ images.append(_to_image_content(file))+ case "download":+ downloads.append(file)+ case _:+ assert_never(file)+ return [_append_file_downloads(self.config.text, downloads), *images]+++def _append_file_downloads(text: str, files: list[DifyUserPromptDownloadConfig]) -> str:+ if not files:+ return text+ locators = [file.model_dump(mode="json", exclude={"delivery", "type"}, exclude_none=True) for file in files]+ payload = json.dumps(locators, ensure_ascii=False, separators=(",", ":"))+ return (+ f"{text}\n"+ "User provided files: use dify-agent file download with the listed transfer_method and reference/url "+ "to get the files and investigate them\n"+ f"{payload}"+ )+++def _to_image_content(file: DifyUserPromptImageConfig) -> ImageUrl | BinaryContent:+ vendor_metadata: dict[str, str] = {"filename": file.filename}+ if file.detail is not None:+ vendor_metadata["detail"] = file.detail+ identifier = _identifier_from_filename(file.filename, file.format)+ if file.url is not None:+ return ImageUrl(+ url=file.url,+ media_type=file.mime_type,+ identifier=identifier,+ vendor_metadata=vendor_metadata,+ )+ assert file.base64_data is not None+ return BinaryContent(+ data=base64.b64decode(file.base64_data, validate=True),+ media_type=file.mime_type,+ identifier=identifier,+ vendor_metadata=vendor_metadata,+ )+++def _identifier_from_filename(filename: str, file_format: str) -> str:+ suffix = f".{file_format}"+ if filename.lower().endswith(suffix.lower()):+ return filename[: -len(suffix)] or "image"+ return filename+++__all__ = ["DifyUserPromptLayer"]diff --git a/dify-agent/src/dify_agent/runtime/compositor_factory.py b/dify-agent/src/dify_agent/runtime/compositor_factory.pyindex f56b37e3c17..a22df749ea6 100644--- a/dify-agent/src/dify_agent/runtime/compositor_factory.py+++ b/dify-agent/src/dify_agent/runtime/compositor_factory.py@@ -10,6 +10,7 @@ layer, and the Dify plugin/knowledge business-layer family:- ``dify.execution_context`` for shared tenant/user/run daemon context,- ``dify.runtime`` for operation-scoped RuntimeLease acquisition,- ``dify.shell`` for command/file capabilities from the active RuntimeLease,+- ``dify.user_prompt`` for structured text-plus-image user turns,- ``dify.plugin.llm`` for plugin-backed model selection,- ``dify.plugin.tools`` for prepared plugin tool exposure, and- ``dify.core.tools`` for API-routed Dify tool exposure, and@@ -55,6 +56,7 @@ from dify_agent.layers.runtime.configs import DifyRuntimeLayerConfigfrom dify_agent.layers.runtime.layer import DifyRuntimeLayerfrom dify_agent.layers.shell.configs import DifyShellLayerConfigfrom dify_agent.layers.shell.layer import DifyShellLayer+from dify_agent.layers.user_prompt.layer import DifyUserPromptLayerfrom dify_agent.runtime_backend import RuntimeBackendProfiletype DifyAgentLayerProvider = LayerProvider[Any]@@ -74,6 +76,7 @@ def create_default_layer_providers("""Return the server provider set of safe config-constructible layers."""providers: list[DifyAgentLayerProvider] = [LayerProvider.from_layer_type(PromptLayer),+ LayerProvider.from_layer_type(DifyUserPromptLayer),LayerProvider.from_layer_type(PydanticAIHistoryLayer),LayerProvider.from_layer_type(DifyOutputLayer),LayerProvider.from_layer_type(DifyAskHumanLayer),diff --git a/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py b/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.pynew file mode 100644index 00000000000..3c8927bd732--- /dev/null+++ b/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py@@ -0,0 +1,198 @@+from __future__ import annotations++import base64++import pytest+from pydantic import ValidationError+from pydantic_ai.messages import BinaryContent, ImageUrl++from dify_agent.layers.user_prompt import DifyUserPromptLayerConfig+from dify_agent.layers.user_prompt.layer import DifyUserPromptLayer+from dify_agent.runtime.compositor_factory import create_default_layer_providers++[email protected]("include_image", [False, True])[email protected]("download_type", ["image", "document"])+def test_user_prompt_layer_injects_file_locators(include_image: bool, download_type: str) -> None:+ config = DifyUserPromptLayerConfig.model_validate(+ {+ "text": "Inspect these files.",+ "files": [+ {+ "delivery": "download",+ "type": download_type,+ "transfer_method": "local_file",+ "reference": "dify-file-ref:upload-file-1",+ },+ {+ "delivery": "download",+ "type": "document",+ "transfer_method": "remote_url",+ "url": "https://example.com/说明.pdf",+ },+ {+ "delivery": "download",+ "type": "document",+ "transfer_method": "tool_file",+ "reference": "dify-file-ref:tool-file-1",+ },+ {+ "delivery": "download",+ "type": "document",+ "transfer_method": "datasource_file",+ "reference": "dify-file-ref:datasource-file-1",+ },+ *(+ [+ {+ "delivery": "multimodal",+ "type": "image",+ "filename": "earth.png",+ "mime_type": "image/png",+ "format": "png",+ "url": "https://files.example.com/earth.png",+ }+ ]+ if include_image+ else []+ ),+ ],+ }+ )+ layer = DifyUserPromptLayer.from_config(DifyUserPromptLayerConfig.model_validate_json(config.model_dump_json()))++ prompts = layer.user_prompts++ assert prompts[0] == (+ "Inspect these files.\n"+ "User provided files: use dify-agent file download with the listed transfer_method and reference/url "+ "to get the files and investigate them\n"+ '[{"transfer_method":"local_file","reference":"dify-file-ref:upload-file-1"},'+ '{"transfer_method":"remote_url","url":"https://example.com/说明.pdf"},'+ '{"transfer_method":"tool_file","reference":"dify-file-ref:tool-file-1"},'+ '{"transfer_method":"datasource_file","reference":"dify-file-ref:datasource-file-1"}]'+ )+ assert layer.config.text == "Inspect these files."+ assert "locators" not in layer.config.model_dump()+ assert layer.user_prompts == prompts+ assert len(prompts) == (2 if include_image else 1)+ if include_image:+ assert isinstance(prompts[1], ImageUrl)+ assert prompts[1].url == "https://files.example.com/earth.png"+++ "locator",+ [+ {"transfer_method": "remote_url", "reference": "file-1"},+ {"transfer_method": "local_file", "url": "https://example.com/file.pdf"},+ {"transfer_method": "unsupported", "reference": "file-1"},+ {"transfer_method": "local_file"},+ {"transfer_method": "remote_url"},+ {"transfer_method": "local_file", "reference": ""},+ {"transfer_method": "remote_url", "url": ""},+ {"transfer_method": "remote_url", "url": "https://example.com/file.pdf", "reference": "file-1"},+ {"transfer_method": "tool_file", "url": "https://example.com/file.pdf", "reference": "file-1"},+ ],+)+def test_user_prompt_layer_rejects_invalid_locators(locator: dict[str, str]) -> None:+ with pytest.raises(ValidationError):+ DifyUserPromptLayerConfig.model_validate(+ {"text": "Inspect it.", "files": [{"delivery": "download", "type": "document", **locator}]}+ )+++def test_user_prompt_layer_preserves_text_without_locators() -> None:+ layer = DifyUserPromptLayer.from_config(DifyUserPromptLayerConfig(text=" Original prompt.\n"))++ assert layer.user_prompts == [" Original prompt.\n"]+++def test_user_prompt_layer_restores_image_url_content() -> None:+ layer = DifyUserPromptLayer.from_config(+ DifyUserPromptLayerConfig.model_validate(+ {+ "text": "What is in this image?",+ "files": [+ {+ "delivery": "multimodal",+ "type": "image",+ "filename": "earth.png",+ "mime_type": "image/png",+ "format": "png",+ "url": "https://files.example.com/earth.png?sign=secret",+ "detail": "high",+ }+ ],+ }+ )+ )++ prompts = layer.user_prompts++ assert prompts[0] == "What is in this image?"+ assert isinstance(prompts[1], ImageUrl)+ assert prompts[1].url == "https://files.example.com/earth.png?sign=secret"+ assert prompts[1].media_type == "image/png"+ assert prompts[1].vendor_metadata == {"filename": "earth.png", "detail": "high"}+++def test_user_prompt_layer_restores_inline_binary_content() -> None:+ payload = base64.b64encode(b"image-bytes").decode()+ layer = DifyUserPromptLayer.from_config(+ DifyUserPromptLayerConfig.model_validate(+ {+ "text": "Describe it.",+ "files": [+ {+ "delivery": "multimodal",+ "type": "image",+ "filename": "inline.png",+ "mime_type": "image/png",+ "format": "png",+ "base64_data": payload,+ }+ ],+ }+ )+ )++ content = layer.user_prompts[1]++ assert isinstance(content, BinaryContent)+ assert content.data == b"image-bytes"+ assert content.media_type == "image/png"+++ "file_payload",+ [+ {+ "delivery": "multimodal",+ "type": "image",+ "filename": "missing.png",+ "mime_type": "image/png",+ "format": "png",+ },+ {+ "delivery": "multimodal",+ "type": "image",+ "filename": "ambiguous.png",+ "mime_type": "image/png",+ "format": "png",+ "url": "https://files.example.com/ambiguous.png",+ "base64_data": "aW1hZ2U=",+ },+ ],+)+def test_user_prompt_file_requires_exactly_one_transport(file_payload: dict[str, str]) -> None:+ with pytest.raises(ValidationError, match="exactly one"):+ DifyUserPromptLayerConfig.model_validate({"text": "Describe it.", "files": [file_payload]})+++def test_default_compositor_registers_user_prompt_layer() -> None:+ provider = next(provider for provider in create_default_layer_providers() if provider.type_id == "dify.user_prompt")++ layer = provider.create_layer({"text": "Describe it."})++ assert isinstance(layer, DifyUserPromptLayer)diff --git a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py b/dify-agent/tests/local/dify_agent/test_client_safe_exports.pyindex ba1493701fb..ec4ca696882 100644--- a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py+++ b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py@@ -72,6 +72,7 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Patplugin_module = importlib.import_module("dify_agent.layers.dify_plugin")ask_human_module = importlib.import_module("dify_agent.layers.ask_human")output_module = importlib.import_module("dify_agent.layers.output")+ user_prompt_module = importlib.import_module("dify_agent.layers.user_prompt")assert agenton_layers.ExitIntent is not Noneassert agenton_layers.LayerConfig is not None@@ -93,6 +94,17 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Patassert plugin_module.DifyPluginLLMLayerConfig is not Noneassert ask_human_module.DifyAskHumanLayerConfig is not Noneassert output_module.DifyOutputLayerConfig is not None+ assert user_prompt_module.DifyUserPromptLayerConfig is not None+ assert user_prompt_module.DifyUserPromptFileConfig is not None+ download = user_prompt_module.DifyUserPromptDownloadConfig(+ type="image", transfer_method="local_file", reference="file-1"+ )+ image = user_prompt_module.DifyUserPromptImageConfig(+ filename="image.png", mime_type="image/png", format="png", url="https://example.com/image.png"+ )+ config = user_prompt_module.DifyUserPromptLayerConfig(text="Inspect it.", files=[download, image])+ assert user_prompt_module.DifyUserPromptLayerConfig.model_validate_json(config.model_dump_json()) == config+ assert [file.delivery for file in config.files] == ["download", "multimodal"]unexpectedly_installed = []for dependency_name in sorted(server_only_dependency_names):