Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(agent): send uploaded images as native multimodal input (#41685)

kenwoodjwcf34e3ee1a6f
diff --git a/api/clients/agent_backend/request_builder.py b/api/clients/agent_backend/request_builder.py
index 2f4d09ae9d5..d7a3b8ddc38 100644
--- a/api/clients/agent_backend/request_builder.py
+++ b/api/clients/agent_backend/request_builder.py
@@ -36,6 +36,11 @@ from dify_agent.layers.knowledge import DIFY_KNOWLEDGE_BASE_LAYER_TYPE_ID, DifyK
from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID, DifyOutputLayerConfig
from dify_agent.layers.runtime import DIFY_RUNTIME_LAYER_TYPE_ID, DifyRuntimeLayerConfig
from dify_agent.layers.shell import DIFY_SHELL_LAYER_TYPE_ID, DifyShellLayerConfig
+from dify_agent.layers.user_prompt import (
+ DIFY_USER_PROMPT_LAYER_TYPE_ID,
+ DifyUserPromptFileConfig,
+ DifyUserPromptLayerConfig,
+)
from dify_agent.protocol import (
DIFY_AGENT_HISTORY_LAYER_ID,
DIFY_AGENT_MODEL_LAYER_ID,
@@ -246,6 +251,7 @@ class AgentBackendAgentAppRunInput(BaseModel):
execution_context: DifyExecutionContextLayerConfig
backend_binding_ref: str = Field(min_length=1)
user_prompt: str
+ user_files: list[DifyUserPromptFileConfig] = Field(default_factory=list)
agent_soul_prompt: str | None = None
agent_config_version_kind: AgentConfigVersionKind = "snapshot"
idempotency_key: str | None = None
@@ -310,9 +316,9 @@ class AgentBackendRunRequestBuilder:
[
RunLayerSpec(
name=AGENT_APP_USER_PROMPT_LAYER_ID,
- type=PLAIN_PROMPT_LAYER_TYPE_ID,
+ type=DIFY_USER_PROMPT_LAYER_TYPE_ID,
metadata={**run_input.metadata, "origin": "agent_app_user_prompt"},
- config=PromptLayerConfig(user=run_input.user_prompt),
+ config=DifyUserPromptLayerConfig(text=run_input.user_prompt, files=run_input.user_files),
),
RunLayerSpec(
name=DIFY_EXECUTION_CONTEXT_LAYER_ID,
@@ -632,7 +638,7 @@ class AgentBackendRunRequestBuilder:
)
-_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key")
+_SENSITIVE_KEY_PARTS = ("secret", "credential", "token", "password", "api_key", "base64_data")
def redact_for_agent_backend_log(value: object) -> object:
@@ -641,9 +647,10 @@ def redact_for_agent_backend_log(value: object) -> object:
return redact_for_agent_backend_log(value.model_dump(mode="json", warnings=False))
if isinstance(value, dict):
redacted: dict[object, object] = {}
+ is_multimodal_file = value.get("type") == "image" and "filename" in value and "mime_type" in value
for key, item in value.items():
key_text = str(key).lower()
- if any(part in key_text for part in _SENSITIVE_KEY_PARTS):
+ if any(part in key_text for part in _SENSITIVE_KEY_PARTS) or (is_multimodal_file and key_text == "url"):
redacted[key] = "[REDACTED]"
else:
redacted[key] = redact_for_agent_backend_log(item)
diff --git a/api/core/app/apps/agent_app/app_generator.py b/api/core/app/apps/agent_app/app_generator.py
index 5381cd4b1c3..2d58ca1b4e4 100644
--- a/api/core/app/apps/agent_app/app_generator.py
+++ b/api/core/app/apps/agent_app/app_generator.py
@@ -11,15 +11,13 @@ changes the runtime exit policy carried to the backend.
from __future__ import annotations
import contextvars
-import json
import logging
import threading
import uuid
-from collections.abc import Generator, Mapping, Sequence
+from collections.abc import Generator, Mapping
from typing import Any, Literal
from flask import Flask, current_app
-from pydantic import JsonValue
from sqlalchemy import and_, or_, select
from sqlalchemy.orm import Session
@@ -29,6 +27,7 @@ from configs import dify_config
from constants import UUID_NIL
from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot
from core.app.app_config.easy_ui_based_app.model_config.converter import ModelConfigConverter
+from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
from core.app.apps.agent_app.app_config_manager import AgentAppConfigManager
from core.app.apps.agent_app.app_runner import AgentAppRunner
from core.app.apps.agent_app.errors import (
@@ -52,8 +51,8 @@ from core.app.entities.app_invoke_entities import (
from core.credit_usage import CreditUsageAppType
from core.db.session_factory import session_factory
from core.ops.ops_trace_manager import TraceQueueManager
-from core.workflow.file_reference import build_file_reference, is_canonical_file_reference
from extensions.ext_database import db
+from factories import file_factory
from models import Account, App, AppModelConfig, Conversation, EndUser, Message, MessageAnnotation
from models.agent import (
APP_BACKED_AGENT_SOURCES,
@@ -75,69 +74,6 @@ from services.conversation_service import ConversationService
logger = logging.getLogger(__name__)
-_REFERENCE_FILE_TRANSFER_METHODS = {"local_file", "tool_file", "datasource_file"}
-
-
-def _append_prompt_file_mappings(query: str, prompt_file_mappings: Sequence[JsonValue]) -> str:
- """Append labeled, prompt-safe file locators to the backend user prompt."""
- prompt_files = _prompt_file_locators(prompt_file_mappings)
- if not prompt_files:
- return query
- payload = json.dumps(prompt_files, ensure_ascii=False, separators=(",", ":"))
- return (
- f"{query}\n"
- "User provided files: use dify-agent file download with the listed transfer_method and reference/url "
- "to get the files and investigate them\n"
- f"{payload}"
- )
-
-
-def _prompt_file_locators(prompt_file_mappings: Sequence[JsonValue]) -> list[dict[str, str]]:
- locators: list[dict[str, str]] = []
- for file_mapping in prompt_file_mappings:
- if not isinstance(file_mapping, Mapping):
- continue
- locator = _prompt_file_locator(file_mapping)
- if locator is not None:
- locators.append(locator)
- return locators
-
-
-def _prompt_file_locator(file_mapping: Mapping[str, object]) -> dict[str, str] | None:
- transfer_method = _string_value(file_mapping, "transfer_method")
- if transfer_method == "remote_url":
- url = _string_value(file_mapping, "url") or _string_value(file_mapping, "remote_url")
- if url is None:
- return None
- return {"transfer_method": "remote_url", "url": url}
- elif transfer_method in _REFERENCE_FILE_TRANSFER_METHODS:
- if transfer_method is None:
- return None
- reference = _canonical_file_reference(
- _string_value(file_mapping, "reference")
- or _string_value(file_mapping, "upload_file_id")
- or _string_value(file_mapping, "file_id")
- or _string_value(file_mapping, "id")
- )
- if reference is None:
- return None
- return {"transfer_method": transfer_method, "reference": reference}
- else:
- return None
-
-
-def _canonical_file_reference(reference: str | None) -> str | None:
- if reference is None:
- return None
- if reference.startswith("dify-file-ref:"):
- return reference if is_canonical_file_reference(reference) else None
- return build_file_reference(record_id=reference)
-
-
-def _string_value(mapping: Mapping[str, object], key: str) -> str | None:
- value = mapping.get(key)
- return value if isinstance(value, str) and value else None
-
class AgentAppGenerator(MessageBasedAppGenerator):
def generate(
@@ -155,7 +91,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):
query = self._require_query(args)
inputs = args["inputs"]
- prompt_file_mappings = args.get("files") or []
+ raw_files = args.get("files") or []
conversation = None
conversation_id = args.get("conversation_id")
@@ -193,19 +129,33 @@ class AgentAppGenerator(MessageBasedAppGenerator):
conversation=conversation,
)
model_conf = ModelConfigConverter.convert(app_config)
+ with self._bind_file_access_scope(tenant_id=app_model.tenant_id, user=user, invoke_from=invoke_from):
+ file_upload_config = (
+ FileUploadConfigManager.convert(app_config.app_model_config_dict, is_vision=True) if raw_files else None
+ )
+ file_objs = (
+ file_factory.build_from_mappings(
+ mappings=raw_files,
+ tenant_id=app_model.tenant_id,
+ config=file_upload_config,
+ access_controller=self._file_access_controller,
+ )
+ if raw_files and file_upload_config is not None
+ else []
+ )
trace_manager = TraceQueueManager(app_model.id, user.id if isinstance(user, Account) else user.session_id)
application_generate_entity = AgentAppGenerateEntity(
task_id=str(uuid.uuid4()),
app_config=app_config,
model_conf=model_conf,
+ file_upload_config=file_upload_config,
conversation_id=conversation.id if conversation else None,
inputs=self._prepare_user_inputs(
user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
),
query=query,
- files=[],
- prompt_file_mappings=prompt_file_mappings,
+ files=list(file_objs),
parent_message_id=(
args.get("parent_message_id")
if invoke_from not in {InvokeFrom.SERVICE_API, InvokeFrom.OPENAPI}
@@ -492,10 +442,6 @@ class AgentAppGenerator(MessageBasedAppGenerator):
)
if handled:
return
- query = _append_prompt_file_mappings(
- query=query,
- prompt_file_mappings=application_generate_entity.prompt_file_mappings,
- )
dify_context = DifyRunContext(
tenant_id=app_config.tenant_id,
@@ -514,6 +460,14 @@ class AgentAppGenerator(MessageBasedAppGenerator):
)
runner = self._build_runner()
+ image_detail_config = (
+ application_generate_entity.file_upload_config.image_config.detail
+ if (
+ application_generate_entity.file_upload_config
+ and application_generate_entity.file_upload_config.image_config
+ )
+ else None
+ )
runner.run(
dify_context=dify_context,
agent_id=application_generate_entity.agent_id,
@@ -526,6 +480,8 @@ class AgentAppGenerator(MessageBasedAppGenerator):
message_id=message.id,
model_name=application_generate_entity.model_conf.model,
queue_manager=queue_manager,
+ files=list(application_generate_entity.files),
+ image_detail_config=image_detail_config,
session_scope_snapshot_id=application_generate_entity.agent_session_scope_config_version_id,
build_draft_id=(
application_generate_entity.agent_config_snapshot_id
diff --git a/api/core/app/apps/agent_app/app_runner.py b/api/core/app/apps/agent_app/app_runner.py
index 665af35a035..f155c07a333 100644
--- a/api/core/app/apps/agent_app/app_runner.py
+++ b/api/core/app/apps/agent_app/app_runner.py
@@ -55,8 +55,14 @@ from core.repositories.human_input_repository import HumanInputFormRepository, H
from core.workflow.nodes.agent_v2.ask_human_hitl import AskHumanFormBuildError, create_ask_human_form
from core.workflow.nodes.agent_v2.ask_human_resume import build_deferred_tool_results, resolve_ask_human_form
from extensions.ext_database import db
+from graphon.file import File
from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
+from graphon.model_runtime.entities.message_entities import (
+ AssistantPromptMessage,
+ ImagePromptMessageContent,
+ PromptMessage,
+ UserPromptMessage,
+)
from graphon.model_runtime.errors.invoke import (
InvokeAuthorizationError,
InvokeBadRequestError,
@@ -643,6 +649,8 @@ class AgentAppRunner:
message_id: str,
model_name: str,
queue_manager: AppQueueManager,
+ files: list[File] | None = None,
+ image_detail_config: ImagePromptMessageContent.DETAIL | None = None,
session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId = _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID,
build_draft_id: str | None = None,
) -> None:
@@ -669,6 +677,8 @@ class AgentAppRunner:
backend_binding_ref=stored.backend_binding_ref,
conversation_id=conversation_id,
query=query,
+ files=tuple(files or ()),
+ image_detail_config=image_detail_config,
idempotency_key=message_id,
stored=stored,
message_id=message_id,
@@ -796,6 +806,8 @@ class AgentAppRunner:
backend_binding_ref: str,
conversation_id: str,
query: str,
+ files: tuple[File, ...],
+ image_detail_config: ImagePromptMessageContent.DETAIL | None,
idempotency_key: str,
stored: StoredAgentAppSession,
message_id: str | None,
@@ -815,6 +827,8 @@ class AgentAppRunner:
agent_soul=agent_soul,
conversation_id=conversation_id,
user_query=query,
+ files=files,
+ image_detail_config=image_detail_config,
idempotency_key=idempotency_key,
binding_id=binding_id,
backend_binding_ref=backend_binding_ref,
diff --git a/api/core/app/apps/agent_app/runtime_request_builder.py b/api/core/app/apps/agent_app/runtime_request_builder.py
index 70c813d0fd5..5f05b0c99d6 100644
--- a/api/core/app/apps/agent_app/runtime_request_builder.py
+++ b/api/core/app/apps/agent_app/runtime_request_builder.py
@@ -19,6 +19,12 @@ from dify_agent.layers.execution_context import (
DifyExecutionContextLayerConfig,
DifyExecutionContextUserFrom,
)
+from dify_agent.layers.user_prompt import (
+ DifyUserPromptDownloadConfig,
+ DifyUserPromptFileConfig,
+ DifyUserPromptFileType,
+ DifyUserPromptImageConfig,
+)
from dify_agent.protocol import CreateRunRequest, DeferredToolResultsPayload
from clients.agent_backend import (
@@ -29,8 +35,9 @@ from clients.agent_backend import (
)
from configs import dify_config
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
-from core.app.llm.model_access import resolve_model_context_window
+from core.app.llm.model_access import resolve_model_context_window, resolve_model_supports_vision
from core.plugin.provider_identity import normalize_plugin_daemon_provider_identity
+from core.workflow.file_reference import build_file_reference, is_canonical_file_reference
from core.workflow.nodes.agent_v2.dify_tools_builder import (
WorkflowAgentDifyToolLayersBuilder,
WorkflowAgentDifyToolsBuilder,
@@ -46,6 +53,8 @@ from core.workflow.nodes.agent_v2.runtime_request_builder import (
build_shell_layer_config,
load_runtime_agent_skill_configs,
)
+from graphon.file import File, FileTransferMethod, FileType, file_manager
+from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent
from models.agent_config_entities import AgentSoulConfig, AgentSoulToolsConfig
from models.provider_ids import ModelProviderID
from services.agent.prompt_mentions import expand_prompt_mentions
@@ -61,6 +70,9 @@ class AgentAppRuntimeRequestBuildError(ValueError):
super().__init__(message)
+type _ReferenceFileTransferMethod = Literal["local_file", "tool_file", "datasource_file"]
+
+
@dataclass(frozen=True, slots=True)
class AgentAppRuntimeBuildContext:
dify_context: DifyRunContext
@@ -72,6 +84,8 @@ class AgentAppRuntimeBuildContext:
idempotency_key: str
binding_id: str
backend_binding_ref: str
+ files: tuple[File, ...] = ()
+ image_detail_config: ImagePromptMessageContent.DETAIL | None = None
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"
session_snapshot: CompositorSessionSnapshot | None = None
# ENG-638: set when resuming a chat turn after a submitted ask_human form.
@@ -150,6 +164,13 @@ class AgentAppRuntimeRequestBuilder:
ModelProviderID(agent_soul.model.model_provider),
agent_soul.model.plugin_id,
)
+ user_files = self._build_user_files(
+ files=context.files,
+ run_context=context.dify_context,
+ provider_name=agent_soul.model.model_provider,
+ model_name=agent_soul.model.model,
+ image_detail_config=context.image_detail_config,
+ )
request = self._request_builder.build_for_agent_app(
AgentBackendAgentAppRunInput(
@@ -180,6 +201,7 @@ class AgentAppRuntimeRequestBuilder:
or None,
agent_config_version_kind=context.agent_config_version_kind,
user_prompt=context.user_query,
+ user_files=user_files,
tools=tool_layers.plugin_tools,
core_tools=tool_layers.core_tools,
knowledge=knowledge_config,
@@ -202,6 +224,27 @@ class AgentAppRuntimeRequestBuilder:
binding_id=context.binding_id,
)
+ @staticmethod
+ def _build_user_files(
+ *,
+ files: tuple[File, ...],
+ run_context: DifyRunContext,
+ provider_name: str,
+ model_name: str,
+ image_detail_config: ImagePromptMessageContent.DETAIL | None,
+ ) -> list[DifyUserPromptFileConfig]:
+ supports_vision = any(file.type == FileType.IMAGE for file in files) and resolve_model_supports_vision(
+ run_context=run_context,
+ provider_name=provider_name,
+ model_name=model_name,
+ )
+ return [
+ _build_user_image(file, image_detail_config=image_detail_config)
+ if supports_vision and file.type == FileType.IMAGE
+ else _build_user_download(file)
+ for file in files
+ ]
+
@staticmethod
def _validate_session_snapshot_layers(request: CreateRunRequest) -> None:
"""Reject stale snapshots before they reach the Agent backend.
@@ -251,6 +294,57 @@ class AgentAppRuntimeRequestBuilder:
}
+def _build_user_image(
+ file: File,
+ *,
+ image_detail_config: ImagePromptMessageContent.DETAIL | None,
+) -> DifyUserPromptImageConfig:
+ content = file_manager.to_prompt_message_content(file, image_detail_config=image_detail_config)
+ if not isinstance(content, ImagePromptMessageContent):
+ raise AgentAppRuntimeRequestBuildError(
+ "agent_user_file_unsupported",
+ f"Agent App cannot send file '{file.filename or 'image'}' as vision content.",
+ )
+ detail = content.detail.value
+ return DifyUserPromptImageConfig(
+ filename=content.filename or file.filename or f"image.{content.format}",
+ mime_type=content.mime_type,
+ format=content.format,
+ url=content.url or None,
+ base64_data=content.base64_data or None,
+ detail=detail if detail in {"low", "high"} else None,
+ )
+
+
+def _build_user_download(file: File) -> DifyUserPromptDownloadConfig:
+ file_type = cast(DifyUserPromptFileType, file.type.value)
+ if file.transfer_method == FileTransferMethod.REMOTE_URL:
+ if file.remote_url is None:
+ raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "Remote user file is missing its URL.")
+ return DifyUserPromptDownloadConfig(type=file_type, transfer_method="remote_url", url=file.remote_url)
+ if file.reference is None:
+ raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "User file is missing its reference.")
+ reference = file.reference
+ if not reference.startswith("dify-file-ref:"):
+ reference = build_file_reference(record_id=reference)
+ elif not is_canonical_file_reference(reference):
+ raise AgentAppRuntimeRequestBuildError("agent_user_file_invalid", "User file reference is invalid.")
+ transfer_method: _ReferenceFileTransferMethod
+ match file.transfer_method:
+ case FileTransferMethod.LOCAL_FILE:
+ transfer_method = "local_file"
+ case FileTransferMethod.TOOL_FILE:
+ transfer_method = "tool_file"
+ case FileTransferMethod.DATASOURCE_FILE:
+ transfer_method = "datasource_file"
+ case _:
+ raise AgentAppRuntimeRequestBuildError(
+ "agent_user_file_invalid",
+ f"User file transfer method '{file.transfer_method.value}' is unsupported.",
+ )
+ return DifyUserPromptDownloadConfig(type=file_type, transfer_method=transfer_method, reference=reference)
+
+
__all__ = [
"AgentAppRuntimeBuildContext",
"AgentAppRuntimeRequest",
diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.py
index 7f983135994..cb3f52d12a0 100644
--- a/api/core/app/entities/app_invoke_entities.py
+++ b/api/core/app/entities/app_invoke_entities.py
@@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence
from enum import StrEnum
from typing import TYPE_CHECKING, Any, Literal
-from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationInfo, field_validator
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
from constants import UUID_NIL
from core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAppConfig
@@ -286,16 +286,15 @@ class AgentAppGenerateEntity(ChatAppGenerateEntity):
``agent_session_scope_config_version_id`` identifies the draft or immutable
config version whose Workspace Binding should be reused for this session.
- ``prompt_file_mappings`` preserves the raw request ``files`` array for the
- Agent backend prompt. These references are appended to the backend prompt
- text while the stored chat message keeps the user's original query.
+ Uploaded files use the inherited ``files`` field. The Agent App runtime
+ sends supported images directly to vision models and preserves the sandbox
+ locator fallback for all other files.
"""
agent_id: str
agent_config_snapshot_id: str
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"
agent_session_scope_config_version_id: str | None = None
- prompt_file_mappings: Sequence[JsonValue] = Field(default_factory=list)
agent_llm_gateway_enabled: bool = False
diff --git a/api/core/app/llm/model_access.py b/api/core/app/llm/model_access.py
index 52fb9add70b..e793200ffb7 100644
--- a/api/core/app/llm/model_access.py
+++ b/api/core/app/llm/model_access.py
@@ -9,7 +9,7 @@ from core.errors.error import ProviderTokenNotInitError
from core.model_manager import ModelInstance, ModelManager
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
from core.provider_manager import ProviderManager
-from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
+from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey, ModelType
from graphon.nodes.llm.entities import ModelConfig
from graphon.nodes.llm.exc import LLMModeRequiredError, ModelNotExistError
from graphon.nodes.llm.protocols import CredentialsProvider
@@ -149,6 +149,18 @@ def resolve_model_context_window(
return context_window
+def resolve_model_supports_vision(
+ *,
+ run_context: DifyRunContext,
+ provider_name: str,
+ model_name: str,
+) -> bool:
+ """Return whether the credential-bound model advertises vision support."""
+
+ model_instance = DifyModelFactory(run_context=run_context).init_model_instance(provider_name, model_name)
+ return ModelFeature.VISION in (model_instance.get_model_schema().features or [])
+
+
def _normalize_completion_params(completion_params: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
"""
Split node-level completion params into provider parameters and stop sequences.
diff --git a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py
index 978b522d509..29771083de6 100644
--- a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py
+++ b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py
@@ -20,6 +20,11 @@ from dify_agent.layers.execution_context import DIFY_EXECUTION_CONTEXT_LAYER_TYP
from dify_agent.layers.knowledge import DIFY_KNOWLEDGE_BASE_LAYER_TYPE_ID, DifyKnowledgeBaseLayerConfig
from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID
from dify_agent.layers.shell import DIFY_SHELL_LAYER_TYPE_ID, DifyShellEnvVarConfig, DifyShellLayerConfig
+from dify_agent.layers.user_prompt import (
+ DIFY_USER_PROMPT_LAYER_TYPE_ID,
+ DifyUserPromptDownloadConfig,
+ DifyUserPromptImageConfig,
+)
from dify_agent.protocol import (
DIFY_AGENT_HISTORY_LAYER_ID,
DIFY_AGENT_MODEL_LAYER_ID,
@@ -367,6 +372,64 @@ def test_agent_app_request_builder_omits_shell_layer_by_default():
assert DIFY_SHELL_LAYER_ID not in {layer.name for layer in request.composition.layers}
+def test_agent_app_request_builder_emits_multimodal_user_prompt_layer():
+ run_input = _agent_app_input()
+ run_input.user_files = [
+ DifyUserPromptDownloadConfig(
+ type="document", transfer_method="remote_url", url="https://example.com/brief.pdf"
+ ),
+ DifyUserPromptImageConfig(
+ filename="earth.png",
+ mime_type="image/png",
+ format="png",
+ url="https://files.example.com/earth.png?sign=secret",
+ detail="high",
+ ),
+ ]
+
+ request = AgentBackendRunRequestBuilder().build_for_agent_app(run_input)
+ layer = next(layer for layer in request.composition.layers if layer.name == "agent_app_user_prompt")
+
+ assert layer.type == DIFY_USER_PROMPT_LAYER_TYPE_ID
+ assert layer.config.text == "List files."
+ assert layer.config.files == run_input.user_files
+ restored_request = CreateRunRequest.model_validate_json(request.model_dump_json())
+ restored_layer = next(
+ layer for layer in restored_request.composition.layers if layer.name == "agent_app_user_prompt"
+ )
+ assert restored_layer.config == layer.config.model_dump(mode="json")
+ assert "locators" not in restored_layer.config
+
+
+def test_agent_backend_log_redacts_multimodal_file_transport():
+ run_input = _agent_app_input()
+ run_input.metadata = {"source_url": "https://example.com/docs"}
+ run_input.user_files = [
+ DifyUserPromptImageConfig(
+ filename="earth.png",
+ mime_type="image/png",
+ format="png",
+ url="https://files.example.com/earth.png?sign=secret",
+ ),
+ DifyUserPromptImageConfig(
+ filename="inline.png",
+ mime_type="image/png",
+ format="png",
+ base64_data="aW1hZ2UtYnl0ZXM=",
+ ),
+ ]
+
+ redacted = cast(
+ dict[str, Any],
+ redact_for_agent_backend_log(AgentBackendRunRequestBuilder().build_for_agent_app(run_input)),
+ )
+ layer = next(item for item in redacted["composition"]["layers"] if item["name"] == "agent_app_user_prompt")
+
+ assert layer["config"]["files"][0]["url"] == "[REDACTED]"
+ assert layer["config"]["files"][1]["base64_data"] == "[REDACTED]"
+ assert redacted["metadata"]["source_url"] == "https://example.com/docs"
+
+
def test_agent_app_request_builder_keeps_build_draft_prompt_when_agent_soul_prompt_is_blank():
run_input = _agent_app_input().model_copy(
update={"agent_soul_prompt": " ", "agent_config_version_kind": "build_draft"}
diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py
index 410efadba33..4a84bcc4689 100644
--- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py
+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py
@@ -11,7 +11,6 @@ from __future__ import annotations
import contextlib
import inspect
-import json
from decimal import Decimal
from types import SimpleNamespace
@@ -29,7 +28,6 @@ from core.app.apps.agent_app.errors import AgentSessionSnapshotIncompatibleError
from core.app.apps.exc import GenerateTaskStoppedError
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
from core.app.entities.queue_entities import QueueAnnotationReplyEvent
-from core.workflow.file_reference import build_file_reference
from models import Account, AppModelConfig
from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus
from models.agent_config_entities import AgentSoulConfig
@@ -209,6 +207,10 @@ class TestGenerateSuccess:
return_value=mocker.MagicMock(variables=[], tenant_id="tenant", app_id="app1"),
)
mocker.patch(f"{MODULE}.ModelConfigConverter.convert", return_value=mocker.MagicMock(model="gpt-4o-mini"))
+ file_upload_config = mocker.MagicMock()
+ mocker.patch(f"{MODULE}.FileUploadConfigManager.convert", return_value=file_upload_config)
+ parsed_file = mocker.MagicMock()
+ build_files = mocker.patch(f"{MODULE}.file_factory.build_from_mappings", return_value=[parsed_file])
mocker.patch(f"{MODULE}.TraceQueueManager", return_value=mocker.MagicMock())
generate_entity = mocker.patch(
f"{MODULE}.AgentAppGenerateEntity", return_value=mocker.MagicMock(task_id="t", user_id="user")
@@ -248,7 +250,10 @@ class TestGenerateSuccess:
conversation=None,
)
assert session.get(AppModelConfig, "config-1") is config
- assert generate_entity.call_args.kwargs["prompt_file_mappings"] == file_mappings
+ build_files.assert_called_once()
+ assert build_files.call_args.kwargs["mappings"] == file_mappings
+ assert generate_entity.call_args.kwargs["files"] == [parsed_file]
+ assert generate_entity.call_args.kwargs["file_upload_config"] is file_upload_config
assert "agent_runtime_exit_intent" not in generate_entity.call_args.kwargs
def test_generate_loads_existing_conversation(self, generator: AgentAppGenerator, mocker: MockerFixture):
@@ -379,7 +384,8 @@ class TestGenerateWorker:
is_resume=False,
query="query",
session_scope_config_version_id="s",
- prompt_file_mappings=(),
+ files=(),
+ file_upload_config=None,
):
generator._generate_worker(
flask_app=mocker.MagicMock(),
@@ -391,7 +397,8 @@ class TestGenerateWorker:
agent_session_scope_config_version_id=session_scope_config_version_id,
model_conf=mocker.MagicMock(model="m"),
query=query,
- prompt_file_mappings=prompt_file_mappings,
+ files=files,
+ file_upload_config=file_upload_config,
),
queue_manager=queue_manager,
conversation_id="conv",
@@ -420,48 +427,22 @@ class TestGenerateWorker:
assert runner.run.call_args.kwargs["agent_config_snapshot_id"] == "s"
assert runner.run.call_args.kwargs["session_scope_snapshot_id"] is None
- def test_worker_appends_prompt_files_to_backend_query(self, generator, mocker: MockerFixture):
+ def test_worker_passes_files_to_backend_runner_without_rewriting_query(self, generator, mocker: MockerFixture):
runner, _ = self._wire(generator, mocker, guard_query="你看得见这张图片吗")
queue_manager = mocker.MagicMock()
- file_mappings = [
- {
- "type": "image",
- "transfer_method": "local_file",
- "url": "",
- "upload_file_id": "upload-file-1",
- },
- {
- "type": "document",
- "transfer_method": "remote_url",
- "url": "https://example.com/source.pdf",
- "upload_file_id": "ignored",
- },
- ]
- expected_file_mappings = [
- {
- "transfer_method": "local_file",
- "reference": build_file_reference(record_id="upload-file-1"),
- },
- {
- "transfer_method": "remote_url",
- "url": "https://example.com/source.pdf",
- },
- ]
+ files = [mocker.MagicMock(), mocker.MagicMock()]
self._call(
generator,
mocker,
queue_manager,
query="你看得见这张图片吗",
- prompt_file_mappings=file_mappings,
+ files=files,
)
- assert runner.run.call_args.kwargs["query"] == (
- "你看得见这张图片吗\nUser provided files: "
- "use dify-agent file download with the listed transfer_method and reference/url "
- "to get the files and investigate them\n"
- f"{json.dumps(expected_file_mappings, ensure_ascii=False, separators=(',', ':'))}"
- )
+ assert runner.run.call_args.kwargs["query"] == "你看得见这张图片吗"
+ assert runner.run.call_args.kwargs["files"] == files
+ assert runner.run.call_args.kwargs["image_detail_config"] is None
def test_input_guard_short_circuit_skips_backend(self, generator, mocker: MockerFixture):
runner, _ = self._wire(generator, mocker, handled=True)
diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
index db33a1ad03b..b93f0e6ff90 100644
--- a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
@@ -12,6 +12,7 @@ from dify_agent.layers.config import DifyConfigSkillConfig
from dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolsLayerConfig
from dify_agent.layers.dify_plugin import DifyPluginToolConfig, DifyPluginToolsLayerConfig
from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
+from dify_agent.layers.user_prompt import DifyUserPromptLayerConfig
from clients.agent_backend import (
DIFY_CONFIG_LAYER_ID,
@@ -29,6 +30,9 @@ from core.app.apps.agent_app.runtime_request_builder import (
AgentAppRuntimeRequestBuildError,
)
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
+from core.workflow.file_reference import build_file_reference
+from graphon.file import File, FileTransferMethod, FileType
+from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent
from models.agent_config_entities import AgentSoulConfig
from tests.unit_tests.config_override import apply_config_overrides
@@ -168,6 +172,8 @@ def _ctx(
query: str = "hello",
agent_config_version_kind: str = "snapshot",
session_snapshot: CompositorSessionSnapshot | None = None,
+ files: tuple[File, ...] = (),
+ image_detail_config: ImagePromptMessageContent.DETAIL | None = None,
) -> AgentAppRuntimeBuildContext:
dify_context = SimpleNamespace(
tenant_id="tenant-1",
@@ -188,6 +194,8 @@ def _ctx(
backend_binding_ref="binding-ref-1",
agent_config_version_kind=agent_config_version_kind, # type: ignore[arg-type]
session_snapshot=session_snapshot,
+ files=files,
+ image_detail_config=image_detail_config,
)
@@ -204,6 +212,32 @@ def _soul_with_model() -> AgentSoulConfig:
)
+def _image_file() -> File:
+ return File(
+ file_id="file-1",
+ file_type=FileType.IMAGE,
+ transfer_method=FileTransferMethod.LOCAL_FILE,
+ reference="upload-file-1",
+ filename="earth.png",
+ extension=".png",
+ mime_type="image/png",
+ size=12,
+ )
+
+
+def _document_file() -> File:
+ return File(
+ file_id="file-2",
+ file_type=FileType.DOCUMENT,
+ transfer_method=FileTransferMethod.LOCAL_FILE,
+ reference="upload-document-1",
+ filename="brief.pdf",
+ extension=".pdf",
+ mime_type="application/pdf",
+ size=24,
+ )
+
+
def _snapshot_for_layer_names(layer_names: list[str]) -> CompositorSessionSnapshot:
return CompositorSessionSnapshot(
layers=[
@@ -251,6 +285,135 @@ class TestAgentAppRuntimeRequestBuilder:
assert "credentials" not in result.redacted_request["composition"]["layers"][-1]["config"]
assert result.metadata["conversation_id"] == "conv-1"
+ def test_build_sends_images_directly_to_vision_model(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",
+ lambda **_kwargs: True,
+ )
+ prompt_content_calls: list[tuple[File, ImagePromptMessageContent.DETAIL | None]] = []
+
+ def to_prompt_message_content(
+ file: File,
+ *,
+ image_detail_config: ImagePromptMessageContent.DETAIL | None,
+ ) -> ImagePromptMessageContent:
+ prompt_content_calls.append((file, image_detail_config))
+ return ImagePromptMessageContent(
+ format="png",
+ url="https://files.example.com/earth.png?sign=secret",
+ mime_type="image/png",
+ filename="earth.png",
+ detail=image_detail_config or ImagePromptMessageContent.DETAIL.LOW,
+ )
+
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content",
+ to_prompt_message_content,
+ )
+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]
+
+ result = builder.build(
+ _ctx(
+ _soul_with_model(),
+ query="Describe this image.",
+ files=(_image_file(),),
+ image_detail_config=ImagePromptMessageContent.DETAIL.HIGH,
+ )
+ )
+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")
+ config = DifyUserPromptLayerConfig.model_validate(layer.config)
+
+ assert config.text == "Describe this image."
+ assert len(config.files) == 1
+ image = config.files[0]
+ assert image.delivery == "multimodal"
+ assert image.type == "image"
+ assert image.url == "https://files.example.com/earth.png?sign=secret"
+ assert image.detail == "high"
+ assert prompt_content_calls == [(_image_file(), ImagePromptMessageContent.DETAIL.HIGH)]
+
+ def test_build_keeps_image_locator_for_non_vision_model(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",
+ lambda **_kwargs: False,
+ )
+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]
+
+ result = builder.build(_ctx(_soul_with_model(), query="Inspect the attachment.", files=(_image_file(),)))
+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")
+ config = DifyUserPromptLayerConfig.model_validate(layer.config)
+
+ assert config.text == "Inspect the attachment."
+ assert [file.model_dump(exclude_none=True) for file in config.files] == [
+ {
+ "delivery": "download",
+ "type": "image",
+ "transfer_method": "local_file",
+ "reference": build_file_reference(record_id="upload-file-1"),
+ }
+ ]
+
+ def test_build_preserves_inline_base64_transport_for_vision_model(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",
+ lambda **_kwargs: True,
+ )
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content",
+ lambda *_args, **_kwargs: ImagePromptMessageContent(
+ format="png",
+ base64_data="aW1hZ2UtYnl0ZXM=",
+ mime_type="image/png",
+ filename="earth.png",
+ detail="low",
+ ),
+ )
+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]
+
+ result = builder.build(_ctx(_soul_with_model(), query="Describe this image.", files=(_image_file(),)))
+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")
+ config = DifyUserPromptLayerConfig.model_validate(layer.config)
+
+ image = config.files[0]
+ assert image.delivery == "multimodal"
+ assert image.url is None
+ assert image.base64_data == "aW1hZ2UtYnl0ZXM="
+
+ def test_build_keeps_non_image_locator_when_vision_image_is_direct(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_supports_vision",
+ lambda **_kwargs: True,
+ )
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.file_manager.to_prompt_message_content",
+ lambda *_args, **_kwargs: ImagePromptMessageContent(
+ format="png",
+ url="https://files.example.com/earth.png",
+ mime_type="image/png",
+ filename="earth.png",
+ detail="low",
+ ),
+ )
+ builder = AgentAppRuntimeRequestBuilder(dify_tools_builder=_NoToolsBuilder()) # type: ignore[arg-type]
+
+ result = builder.build(
+ _ctx(_soul_with_model(), files=(_image_file(), _document_file()), query="Compare the attachments.")
+ )
+ layer = next(item for item in result.request.composition.layers if item.name == "agent_app_user_prompt")
+ config = DifyUserPromptLayerConfig.model_validate(layer.config)
+
+ assert config.text == "Compare the attachments."
+ assert len(config.files) == 2
+ image, download = config.files
+ assert image.delivery == "multimodal"
+ assert image.filename == "earth.png"
+ assert download.model_dump(exclude_none=True) == {
+ "delivery": "download",
+ "type": "document",
+ "transfer_method": "local_file",
+ "reference": build_file_reference(record_id="upload-document-1"),
+ }
+
@pytest.mark.parametrize(
("previous_prompt", "current_prompt"),
[("", "You are Iris."), ("You are Iris.", "")],
diff --git a/api/tests/unit_tests/core/app/llm/test_model_access.py b/api/tests/unit_tests/core/app/llm/test_model_access.py
index 8c22d56223e..9bb0ef37712 100644
--- a/api/tests/unit_tests/core/app/llm/test_model_access.py
+++ b/api/tests/unit_tests/core/app/llm/test_model_access.py
@@ -5,10 +5,15 @@ import pytest
from core.app.entities.app_invoke_entities import DifyRunContext
from core.app.llm import model_access
-from graphon.model_runtime.entities.model_entities import ModelPropertyKey
+from graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey
-def _stub_model_factory(monkeypatch: pytest.MonkeyPatch, context_window: object) -> dict[str, object]:
+def _stub_model_factory(
+ monkeypatch: pytest.MonkeyPatch,
+ context_window: object,
+ *,
+ features: list[ModelFeature] | None = None,
+) -> dict[str, object]:
calls: dict[str, object] = {}
class FakeModelFactory:
@@ -18,7 +23,10 @@ def _stub_model_factory(monkeypatch: pytest.MonkeyPatch, context_window: object)
def init_model_instance(self, provider_name: str, model_name: str) -> object:
calls["provider_name"] = provider_name
calls["model_name"] = model_name
- schema = SimpleNamespace(model_properties={ModelPropertyKey.CONTEXT_SIZE: context_window})
+ schema = SimpleNamespace(
+ model_properties={ModelPropertyKey.CONTEXT_SIZE: context_window},
+ features=features,
+ )
return SimpleNamespace(get_model_schema=lambda: schema)
monkeypatch.setattr(model_access, "DifyModelFactory", FakeModelFactory)
@@ -58,3 +66,24 @@ def test_resolve_model_context_window_ignores_invalid_schema_values(
)
is None
)
+
+
+ ("features", "expected"),
+ [([ModelFeature.VISION], True), ([], False), (None, False)],
+)
+def test_resolve_model_supports_vision_reads_selected_model_schema(
+ monkeypatch: pytest.MonkeyPatch,
+ features: list[ModelFeature] | None,
+ expected: bool,
+) -> None:
+ _stub_model_factory(monkeypatch, 128_000, features=features)
+
+ assert (
+ model_access.resolve_model_supports_vision(
+ run_context=cast(DifyRunContext, object()),
+ provider_name="openai",
+ model_name="gpt-4o",
+ )
+ is expected
+ )
diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.md
index 99ff7b0ac28..bc6eacee02f 100644
--- a/dify-agent/docs/dify-agent/guide/index.md
+++ b/dify-agent/docs/dify-agent/guide/index.md
@@ -359,6 +359,49 @@ whose Agenton layers provide user input. With the MVP provider set, use
effective prompts are rejected during create-run validation before the run is
persisted or scheduled.
+Agent App callers can use the `dify.user_prompt` layer to send text and images
+in the same model turn. Each image must provide exactly one transport: an
+HTTP(S) `url` or unprefixed Base64 data in `base64_data`. The image is passed as
+structured multimodal content; it is not interpolated into `config.text`.
+
+```json
+{
+ "name": "agent_app_user_prompt",
+ "type": "dify.user_prompt",
+ "config": {
+ "text": "Describe this image.",
+ "files": [
+ {
+ "delivery": "multimodal",
+ "type": "image",
+ "filename": "earth.png",
+ "mime_type": "image/png",
+ "format": "png",
+ "url": "https://files.example.com/earth.png",
+ "base64_data": null,
+ "detail": "high"
+ }
+ ]
+ }
+}
+```
+
+All attachments use `config.files`. The `delivery` field selects how an attachment
+is presented, independently of its file `type` (`image`, `document`, `audio`,
+`video`, or `custom`):
+
+- `delivery: "multimodal"` currently supports `type: "image"` only. Dify API
+ chooses URL or Base64 transport according to `MULTIMODAL_SEND_FORMAT`.
+- `delivery: "download"` preserves the original file `type`, including `image`
+ when the selected model has no Vision feature. It contains either
+ `transfer_method: "remote_url"` with `url`, or a `local_file`, `tool_file`, or
+ `datasource_file` transfer method with a canonical `reference`.
+
+The layer appends sandbox file-download instructions for download attachments
+and adds multimodal attachments as structured model content. Callers keep
+`config.text` as the original user text. Omitting `config.files` is equivalent
+to an empty list.
+
The optional Pydantic AI history layer uses the reserved name `history` and
persists captured messages in session snapshots for later resume. Resume from a
terminal event's `session_snapshot` using the same layer composition, names, and
diff --git a/dify-agent/src/dify_agent/layers/user_prompt/__init__.py b/dify-agent/src/dify_agent/layers/user_prompt/__init__.py
new file mode 100644
index 00000000000..f675b1c0140
--- /dev/null
+++ b/dify-agent/src/dify_agent/layers/user_prompt/__init__.py
@@ -0,0 +1,19 @@
+"""Client-safe exports for the Agent App multimodal user prompt layer."""
+
+from dify_agent.layers.user_prompt.configs import (
+ DIFY_USER_PROMPT_LAYER_TYPE_ID,
+ DifyUserPromptDownloadConfig,
+ DifyUserPromptFileConfig,
+ DifyUserPromptFileType,
+ DifyUserPromptImageConfig,
+ DifyUserPromptLayerConfig,
+)
+
+__all__ = [
+ "DIFY_USER_PROMPT_LAYER_TYPE_ID",
+ "DifyUserPromptDownloadConfig",
+ "DifyUserPromptFileConfig",
+ "DifyUserPromptFileType",
+ "DifyUserPromptImageConfig",
+ "DifyUserPromptLayerConfig",
+]
diff --git a/dify-agent/src/dify_agent/layers/user_prompt/configs.py b/dify-agent/src/dify_agent/layers/user_prompt/configs.py
new file mode 100644
index 00000000000..fc9fa273e7d
--- /dev/null
+++ b/dify-agent/src/dify_agent/layers/user_prompt/configs.py
@@ -0,0 +1,90 @@
+"""Serializable DTOs for one Agent App multimodal user prompt."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from typing import Annotated, ClassVar, Final, Literal, Self
+
+from pydantic import ConfigDict, Field, model_validator
+
+from agenton.layers import LayerConfig
+
+
+DIFY_USER_PROMPT_LAYER_TYPE_ID: Final[str] = "dify.user_prompt"
+type DifyUserPromptFileType = Literal["image", "document", "audio", "video", "custom"]
+
+
+class DifyUserPromptImageConfig(LayerConfig):
+ """One image delivered directly to the configured vision model.
+
+ Callers provide either an HTTP(S) URL or base64 bytes. Opaque Dify file
+ references belong to the sandbox download flow and are not accepted here.
+ """
+
+ delivery: Literal["multimodal"] = "multimodal"
+ type: Literal["image"] = "image"
+ filename: str = Field(min_length=1)
+ mime_type: str = Field(pattern=r"^image/[A-Za-z0-9.+-]+$")
+ format: str = Field(min_length=1)
+ url: str | None = None
+ base64_data: str | None = None
+ detail: Literal["low", "high"] | None = None
+
+ model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+ @model_validator(mode="after")
+ def _validate_transport(self) -> Self:
+ if (self.url is None) == (self.base64_data is None):
+ raise ValueError("exactly one of url or base64_data must be provided")
+ if self.url is not None and not self.url.startswith(("http://", "https://")):
+ raise ValueError("url must use http or https")
+ if self.base64_data is not None:
+ try:
+ base64.b64decode(self.base64_data, validate=True)
+ except (ValueError, binascii.Error) as exc:
+ raise ValueError("base64_data must be valid base64") from exc
+ return self
+
+
+class DifyUserPromptDownloadConfig(LayerConfig):
+ delivery: Literal["download"] = "download"
+ type: DifyUserPromptFileType
+ transfer_method: Literal["remote_url", "local_file", "tool_file", "datasource_file"]
+ url: str | None = Field(default=None, min_length=1)
+ reference: str | None = Field(default=None, min_length=1)
+
+ model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+ @model_validator(mode="after")
+ def _validate_locator(self) -> Self:
+ if self.transfer_method == "remote_url":
+ if self.url is None or self.reference is not None:
+ raise ValueError("remote_url requires url and must not include reference")
+ elif self.reference is None or self.url is not None:
+ raise ValueError("reference is required and url must not be provided")
+ return self
+
+
+type DifyUserPromptFileConfig = Annotated[
+ DifyUserPromptImageConfig | DifyUserPromptDownloadConfig, Field(discriminator="delivery")
+]
+
+
+class DifyUserPromptLayerConfig(LayerConfig):
+ """User text plus images that should be sent in the same model turn."""
+
+ text: str = Field(min_length=1)
+ files: list[DifyUserPromptFileConfig] = Field(default_factory=list)
+
+ model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+
+__all__ = [
+ "DIFY_USER_PROMPT_LAYER_TYPE_ID",
+ "DifyUserPromptDownloadConfig",
+ "DifyUserPromptFileConfig",
+ "DifyUserPromptFileType",
+ "DifyUserPromptImageConfig",
+ "DifyUserPromptLayerConfig",
+]
diff --git a/dify-agent/src/dify_agent/layers/user_prompt/layer.py b/dify-agent/src/dify_agent/layers/user_prompt/layer.py
new file mode 100644
index 00000000000..edcdf0a45bc
--- /dev/null
+++ b/dify-agent/src/dify_agent/layers/user_prompt/layer.py
@@ -0,0 +1,91 @@
+"""Turn serialized Agent App images into pydantic-ai user content."""
+
+from __future__ import annotations
+
+import base64
+import json
+from dataclasses import dataclass
+from typing import ClassVar, assert_never
+
+from pydantic_ai.messages import BinaryContent, ImageUrl, UserContent
+from typing_extensions import Self, override
+
+from agenton.layers import EmptyRuntimeState, NoLayerDeps, PydanticAILayer
+from dify_agent.layers.user_prompt.configs import (
+ DIFY_USER_PROMPT_LAYER_TYPE_ID,
+ DifyUserPromptDownloadConfig,
+ DifyUserPromptImageConfig,
+ DifyUserPromptLayerConfig,
+)
+
+
+@dataclass(slots=True)
+class DifyUserPromptLayer(PydanticAILayer[NoLayerDeps, object, DifyUserPromptLayerConfig, EmptyRuntimeState]):
+ """State-free layer for a text prompt and directly attached images."""
+
+ type_id: ClassVar[str | None] = DIFY_USER_PROMPT_LAYER_TYPE_ID
+ config: DifyUserPromptLayerConfig
+
+ @classmethod
+ @override
+ def from_config(cls, config: DifyUserPromptLayerConfig) -> Self:
+ return cls(config=DifyUserPromptLayerConfig.model_validate(config))
+
+ @property
+ @override
+ def user_prompts(self) -> list[UserContent]:
+ images: list[UserContent] = []
+ downloads: list[DifyUserPromptDownloadConfig] = []
+ for file in self.config.files:
+ match file.delivery:
+ case "multimodal":
+ images.append(_to_image_content(file))
+ case "download":
+ downloads.append(file)
+ case _:
+ assert_never(file)
+ return [_append_file_downloads(self.config.text, downloads), *images]
+
+
+def _append_file_downloads(text: str, files: list[DifyUserPromptDownloadConfig]) -> str:
+ if not files:
+ return text
+ locators = [file.model_dump(mode="json", exclude={"delivery", "type"}, exclude_none=True) for file in files]
+ payload = json.dumps(locators, ensure_ascii=False, separators=(",", ":"))
+ return (
+ f"{text}\n"
+ "User provided files: use dify-agent file download with the listed transfer_method and reference/url "
+ "to get the files and investigate them\n"
+ f"{payload}"
+ )
+
+
+def _to_image_content(file: DifyUserPromptImageConfig) -> ImageUrl | BinaryContent:
+ vendor_metadata: dict[str, str] = {"filename": file.filename}
+ if file.detail is not None:
+ vendor_metadata["detail"] = file.detail
+ identifier = _identifier_from_filename(file.filename, file.format)
+ if file.url is not None:
+ return ImageUrl(
+ url=file.url,
+ media_type=file.mime_type,
+ identifier=identifier,
+ vendor_metadata=vendor_metadata,
+ )
+ assert file.base64_data is not None
+ return BinaryContent(
+ data=base64.b64decode(file.base64_data, validate=True),
+ media_type=file.mime_type,
+ identifier=identifier,
+ vendor_metadata=vendor_metadata,
+ )
+
+
+def _identifier_from_filename(filename: str, file_format: str) -> str:
+ suffix = f".{file_format}"
+ if filename.lower().endswith(suffix.lower()):
+ return filename[: -len(suffix)] or "image"
+ return filename
+
+
+__all__ = ["DifyUserPromptLayer"]
diff --git a/dify-agent/src/dify_agent/runtime/compositor_factory.py b/dify-agent/src/dify_agent/runtime/compositor_factory.py
index f56b37e3c17..a22df749ea6 100644
--- a/dify-agent/src/dify_agent/runtime/compositor_factory.py
+++ b/dify-agent/src/dify_agent/runtime/compositor_factory.py
@@ -10,6 +10,7 @@ layer, and the Dify plugin/knowledge business-layer family:
- ``dify.execution_context`` for shared tenant/user/run daemon context,
- ``dify.runtime`` for operation-scoped RuntimeLease acquisition,
- ``dify.shell`` for command/file capabilities from the active RuntimeLease,
+- ``dify.user_prompt`` for structured text-plus-image user turns,
- ``dify.plugin.llm`` for plugin-backed model selection,
- ``dify.plugin.tools`` for prepared plugin tool exposure, and
- ``dify.core.tools`` for API-routed Dify tool exposure, and
@@ -55,6 +56,7 @@ from dify_agent.layers.runtime.configs import DifyRuntimeLayerConfig
from dify_agent.layers.runtime.layer import DifyRuntimeLayer
from dify_agent.layers.shell.configs import DifyShellLayerConfig
from dify_agent.layers.shell.layer import DifyShellLayer
+from dify_agent.layers.user_prompt.layer import DifyUserPromptLayer
from dify_agent.runtime_backend import RuntimeBackendProfile
type DifyAgentLayerProvider = LayerProvider[Any]
@@ -74,6 +76,7 @@ def create_default_layer_providers(
"""Return the server provider set of safe config-constructible layers."""
providers: list[DifyAgentLayerProvider] = [
LayerProvider.from_layer_type(PromptLayer),
+ LayerProvider.from_layer_type(DifyUserPromptLayer),
LayerProvider.from_layer_type(PydanticAIHistoryLayer),
LayerProvider.from_layer_type(DifyOutputLayer),
LayerProvider.from_layer_type(DifyAskHumanLayer),
diff --git a/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py b/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py
new file mode 100644
index 00000000000..3c8927bd732
--- /dev/null
+++ b/dify-agent/tests/local/dify_agent/layers/user_prompt/test_layer.py
@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import base64
+
+import pytest
+from pydantic import ValidationError
+from pydantic_ai.messages import BinaryContent, ImageUrl
+
+from dify_agent.layers.user_prompt import DifyUserPromptLayerConfig
+from dify_agent.layers.user_prompt.layer import DifyUserPromptLayer
+from dify_agent.runtime.compositor_factory import create_default_layer_providers
+
+
[email protected]("include_image", [False, True])
[email protected]("download_type", ["image", "document"])
+def test_user_prompt_layer_injects_file_locators(include_image: bool, download_type: str) -> None:
+ config = DifyUserPromptLayerConfig.model_validate(
+ {
+ "text": "Inspect these files.",
+ "files": [
+ {
+ "delivery": "download",
+ "type": download_type,
+ "transfer_method": "local_file",
+ "reference": "dify-file-ref:upload-file-1",
+ },
+ {
+ "delivery": "download",
+ "type": "document",
+ "transfer_method": "remote_url",
+ "url": "https://example.com/说明.pdf",
+ },
+ {
+ "delivery": "download",
+ "type": "document",
+ "transfer_method": "tool_file",
+ "reference": "dify-file-ref:tool-file-1",
+ },
+ {
+ "delivery": "download",
+ "type": "document",
+ "transfer_method": "datasource_file",
+ "reference": "dify-file-ref:datasource-file-1",
+ },
+ *(
+ [
+ {
+ "delivery": "multimodal",
+ "type": "image",
+ "filename": "earth.png",
+ "mime_type": "image/png",
+ "format": "png",
+ "url": "https://files.example.com/earth.png",
+ }
+ ]
+ if include_image
+ else []
+ ),
+ ],
+ }
+ )
+ layer = DifyUserPromptLayer.from_config(DifyUserPromptLayerConfig.model_validate_json(config.model_dump_json()))
+
+ prompts = layer.user_prompts
+
+ assert prompts[0] == (
+ "Inspect these files.\n"
+ "User provided files: use dify-agent file download with the listed transfer_method and reference/url "
+ "to get the files and investigate them\n"
+ '[{"transfer_method":"local_file","reference":"dify-file-ref:upload-file-1"},'
+ '{"transfer_method":"remote_url","url":"https://example.com/说明.pdf"},'
+ '{"transfer_method":"tool_file","reference":"dify-file-ref:tool-file-1"},'
+ '{"transfer_method":"datasource_file","reference":"dify-file-ref:datasource-file-1"}]'
+ )
+ assert layer.config.text == "Inspect these files."
+ assert "locators" not in layer.config.model_dump()
+ assert layer.user_prompts == prompts
+ assert len(prompts) == (2 if include_image else 1)
+ if include_image:
+ assert isinstance(prompts[1], ImageUrl)
+ assert prompts[1].url == "https://files.example.com/earth.png"
+
+
+ "locator",
+ [
+ {"transfer_method": "remote_url", "reference": "file-1"},
+ {"transfer_method": "local_file", "url": "https://example.com/file.pdf"},
+ {"transfer_method": "unsupported", "reference": "file-1"},
+ {"transfer_method": "local_file"},
+ {"transfer_method": "remote_url"},
+ {"transfer_method": "local_file", "reference": ""},
+ {"transfer_method": "remote_url", "url": ""},
+ {"transfer_method": "remote_url", "url": "https://example.com/file.pdf", "reference": "file-1"},
+ {"transfer_method": "tool_file", "url": "https://example.com/file.pdf", "reference": "file-1"},
+ ],
+)
+def test_user_prompt_layer_rejects_invalid_locators(locator: dict[str, str]) -> None:
+ with pytest.raises(ValidationError):
+ DifyUserPromptLayerConfig.model_validate(
+ {"text": "Inspect it.", "files": [{"delivery": "download", "type": "document", **locator}]}
+ )
+
+
+def test_user_prompt_layer_preserves_text_without_locators() -> None:
+ layer = DifyUserPromptLayer.from_config(DifyUserPromptLayerConfig(text=" Original prompt.\n"))
+
+ assert layer.user_prompts == [" Original prompt.\n"]
+
+
+def test_user_prompt_layer_restores_image_url_content() -> None:
+ layer = DifyUserPromptLayer.from_config(
+ DifyUserPromptLayerConfig.model_validate(
+ {
+ "text": "What is in this image?",
+ "files": [
+ {
+ "delivery": "multimodal",
+ "type": "image",
+ "filename": "earth.png",
+ "mime_type": "image/png",
+ "format": "png",
+ "url": "https://files.example.com/earth.png?sign=secret",
+ "detail": "high",
+ }
+ ],
+ }
+ )
+ )
+
+ prompts = layer.user_prompts
+
+ assert prompts[0] == "What is in this image?"
+ assert isinstance(prompts[1], ImageUrl)
+ assert prompts[1].url == "https://files.example.com/earth.png?sign=secret"
+ assert prompts[1].media_type == "image/png"
+ assert prompts[1].vendor_metadata == {"filename": "earth.png", "detail": "high"}
+
+
+def test_user_prompt_layer_restores_inline_binary_content() -> None:
+ payload = base64.b64encode(b"image-bytes").decode()
+ layer = DifyUserPromptLayer.from_config(
+ DifyUserPromptLayerConfig.model_validate(
+ {
+ "text": "Describe it.",
+ "files": [
+ {
+ "delivery": "multimodal",
+ "type": "image",
+ "filename": "inline.png",
+ "mime_type": "image/png",
+ "format": "png",
+ "base64_data": payload,
+ }
+ ],
+ }
+ )
+ )
+
+ content = layer.user_prompts[1]
+
+ assert isinstance(content, BinaryContent)
+ assert content.data == b"image-bytes"
+ assert content.media_type == "image/png"
+
+
+ "file_payload",
+ [
+ {
+ "delivery": "multimodal",
+ "type": "image",
+ "filename": "missing.png",
+ "mime_type": "image/png",
+ "format": "png",
+ },
+ {
+ "delivery": "multimodal",
+ "type": "image",
+ "filename": "ambiguous.png",
+ "mime_type": "image/png",
+ "format": "png",
+ "url": "https://files.example.com/ambiguous.png",
+ "base64_data": "aW1hZ2U=",
+ },
+ ],
+)
+def test_user_prompt_file_requires_exactly_one_transport(file_payload: dict[str, str]) -> None:
+ with pytest.raises(ValidationError, match="exactly one"):
+ DifyUserPromptLayerConfig.model_validate({"text": "Describe it.", "files": [file_payload]})
+
+
+def test_default_compositor_registers_user_prompt_layer() -> None:
+ provider = next(provider for provider in create_default_layer_providers() if provider.type_id == "dify.user_prompt")
+
+ layer = provider.create_layer({"text": "Describe it."})
+
+ assert isinstance(layer, DifyUserPromptLayer)
diff --git a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py
index ba1493701fb..ec4ca696882 100644
--- a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py
+++ b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py
@@ -72,6 +72,7 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Pat
plugin_module = importlib.import_module("dify_agent.layers.dify_plugin")
ask_human_module = importlib.import_module("dify_agent.layers.ask_human")
output_module = importlib.import_module("dify_agent.layers.output")
+ user_prompt_module = importlib.import_module("dify_agent.layers.user_prompt")
assert agenton_layers.ExitIntent is not None
assert agenton_layers.LayerConfig is not None
@@ -93,6 +94,17 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Pat
assert plugin_module.DifyPluginLLMLayerConfig is not None
assert ask_human_module.DifyAskHumanLayerConfig is not None
assert output_module.DifyOutputLayerConfig is not None
+ assert user_prompt_module.DifyUserPromptLayerConfig is not None
+ assert user_prompt_module.DifyUserPromptFileConfig is not None
+ download = user_prompt_module.DifyUserPromptDownloadConfig(
+ type="image", transfer_method="local_file", reference="file-1"
+ )
+ image = user_prompt_module.DifyUserPromptImageConfig(
+ filename="image.png", mime_type="image/png", format="png", url="https://example.com/image.png"
+ )
+ config = user_prompt_module.DifyUserPromptLayerConfig(text="Inspect it.", files=[download, image])
+ assert user_prompt_module.DifyUserPromptLayerConfig.model_validate_json(config.model_dump_json()) == config
+ assert [file.delivery for file in config.files] == ["download", "multimodal"]
unexpectedly_installed = []
for dependency_name in sorted(server_only_dependency_names):