Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(workflow): adopt Graphon 0.8 execution orchestration (#40277)
-LAN-823f40e71fc9
diff --git a/.github/workflows/main-ci.yml b/.github/workflows/main-ci.ymlindex 96dafb9bb50..ec658b9a559 100644--- a/.github/workflows/main-ci.yml+++ b/.github/workflows/main-ci.yml@@ -34,6 +34,8 @@ jobs:with:cancel_others: 'true'concurrent_skipping: same_content_newer+ # Coverage checks must be generated for each new commit, even with an unchanged tree.+ skip_after_successful_duplicate: 'false'# Check which paths were changed to determine which tests to runcheck-changes:diff --git a/api/.env.example b/api/.env.exampleindex f6ad2db9fdc..f10f15f0b91 100644--- a/api/.env.example+++ b/api/.env.example@@ -612,15 +612,9 @@ MAX_VARIABLE_SIZE=204800# Maximum concurrent node-builder LLM calls per workflow generation requestWORKFLOW_GENERATOR_NODE_BUILDER_MAX_WORKERS=6-# GraphEngine Worker Pool Configuration-# Minimum number of workers per GraphEngine instance (default: 1)-GRAPH_ENGINE_MIN_WORKERS=3-# Maximum number of workers per GraphEngine instance (default: 10)+# Engine Worker Pool Configuration+# Number of workers per Engine instance (default: 10)GRAPH_ENGINE_MAX_WORKERS=10-# Pending task threshold that triggers worker scale up (default: 0)-GRAPH_ENGINE_SCALE_UP_THRESHOLD=0-# Seconds of idle time before scaling down workers (default: 5.0)-GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0# Workflow storage configuration# Options: rdbms, hybriddiff --git a/api/.importlinter b/api/.importlinterindex 336ea2ed16c..1c775479ed0 100644--- a/api/.importlinter+++ b/api/.importlinter@@ -34,7 +34,6 @@ ignore_imports =core.app.apps.advanced_chat.app_generator -> services.conversation_servicecore.app.apps.advanced_chat.app_generator -> services.errors.conversationcore.app.apps.advanced_chat.app_generator -> services.workflow_draft_variable_service- core.app.apps.advanced_chat.app_runner -> services.conversation_variable_updatercore.app.apps.agent_app.app_generator -> services.agent.composer_servicecore.app.apps.agent_app.app_generator -> services.agent.workspace_servicecore.app.apps.agent_app.app_generator -> services.conversation_service@@ -57,12 +56,14 @@ ignore_imports =core.app.apps.workflow.app_generator -> controllers.console.app.workflowcore.app.apps.workflow.app_generator -> services.snippet_generate_servicecore.app.apps.workflow.app_generator -> services.workflow_draft_variable_service+ # Legacy debug callers delegate input hydration to the application owner during core migration.+ core.app.apps.workflow_app_runner -> services.workflow_node_variables+ core.workflow.workflow_entry -> services.workflow_node_variablescore.app.apps.workflow_app_runner -> tasks.mail_human_input_delivery_taskcore.app.features.annotation_reply.annotation_reply -> services.annotation_servicecore.app.features.annotation_reply.annotation_reply -> services.knowledge.dataset_servicecore.app.llm.quota -> services.credit_pool_servicecore.app.task_pipeline.message_cycle_manager -> services.annotation_service- core.app.workflow.layers.persistence -> services.workflow.inspector_eventscore.helper.credential_utils -> services.enterprise.plugin_manager_servicecore.helper.credential_utils -> services.system_feature_servicecore.mcp.auth_client -> servicesdiff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.pyindex 796fb3bda7e..c4171da78ab 100644--- a/api/configs/feature/__init__.py+++ b/api/configs/feature/__init__.py@@ -965,28 +965,12 @@ class WorkflowConfig(BaseSettings):default=400_000,)- # GraphEngine Worker Pool Configuration- GRAPH_ENGINE_MIN_WORKERS: PositiveInt = Field(- description="Minimum number of workers per GraphEngine instance",- default=3,- )-+ # Engine Worker Pool ConfigurationGRAPH_ENGINE_MAX_WORKERS: PositiveInt = Field(- description="Maximum number of workers per GraphEngine instance",+ description="Number of workers per Engine instance",default=10,)- GRAPH_ENGINE_SCALE_UP_THRESHOLD: NonNegativeInt = Field(- description="Pending task threshold that triggers worker scale up",- default=0,- )-- GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME: float = Field(- description="Seconds of idle time before scaling down workers",- default=5.0,- ge=0.1,- )-class WorkflowNodeExecutionConfig(BaseSettings):"""diff --git a/api/context/execution_context.py b/api/context/execution_context.pyindex c0b4c90c4b9..c32bc3cf913 100644--- a/api/context/execution_context.py+++ b/api/context/execution_context.py@@ -13,6 +13,8 @@ from typing import Any, Protocol, final, override, runtime_checkablefrom pydantic import BaseModel+from libs.contextvars import use_contextvars+class AppContext(Protocol):"""@@ -63,8 +65,9 @@ class ExecutionContext:"""Generic execution context used by application-layer adapters.- It restores captured `contextvars` and optionally enters an application- context before the worker executes graph logic.+ It applies captured `contextvars` and optionally enters an application+ context before the worker executes graph logic, then restores the caller's+ bindings after application-context cleanup."""def __init__(@@ -96,28 +99,27 @@ class ExecutionContext:@contextmanagerdef enter(self) -> Generator[None]:"""Enter this execution context."""- if self._context_vars:- for var, val in self._context_vars.items():- var.set(val)-- if self._app_context is not None:- with self._app_context.enter():+ with use_contextvars(self._context_vars):+ if self._app_context is not None:+ with self._app_context.enter():+ yield+ else:yield- else:- yielddef __enter__(self) -> "ExecutionContext":"""Enter the execution context."""cm = self.enter()- self._local.cm = cmcm.__enter__()+ if not hasattr(self._local, "stack"):+ self._local.stack = []+ self._local.stack.append(cm)return selfdef __exit__(self, *args: Any) -> None:"""Exit the execution context."""- cm = getattr(self._local, "cm", None)- if cm is not None:- cm.__exit__(*args)+ stack = getattr(self._local, "stack", None)+ if stack:+ stack.pop().__exit__(*args)class NullAppContext(AppContext):diff --git a/api/context/flask_app_context.py b/api/context/flask_app_context.pyindex 8216aeb6509..7faae70c385 100644--- a/api/context/flask_app_context.py+++ b/api/context/flask_app_context.py@@ -11,6 +11,7 @@ from typing import Any, final, overridefrom flask import Flask, current_app, gfrom context.execution_context import AppContext, IExecutionContext, register_context_capturer+from libs.contextvars import use_contextvars@final@@ -96,8 +97,8 @@ class FlaskExecutionContext:Flask-specific execution context.This is a specialized version of ExecutionContext that includes Flask app- context. It provides the same interface as ExecutionContext but with- Flask-specific implementation.+ context. It provides the same interface as ExecutionContext and restores+ the caller's bindings after Flask context cleanup."""def __init__(@@ -117,7 +118,6 @@ class FlaskExecutionContext:self._app_context = FlaskAppContext(flask_app)self._context_vars = context_varsself._user = user- self._flask_app = flask_appself._local = threading.local()@property@@ -137,37 +137,23 @@ class FlaskExecutionContext:def __enter__(self) -> "FlaskExecutionContext":"""Enter the Flask execution context."""- # Restore non-Flask context variables to avoid leaking Flask tokens across threads- for var, val in self._context_vars.items():- var.set(val)-- # Enter Flask app context- cm = self._app_context.enter()- self._local.cm = cm+ cm = self.enter()cm.__enter__()-- # Restore user in new app context- if self._user is not None:- g._login_user = self._user-+ if not hasattr(self._local, "stack"):+ self._local.stack = []+ self._local.stack.append(cm)return selfdef __exit__(self, *args: Any) -> None:"""Exit the Flask execution context."""- cm = getattr(self._local, "cm", None)- if cm is not None:- cm.__exit__(*args)+ stack = getattr(self._local, "stack", None)+ if stack:+ stack.pop().__exit__(*args)@contextmanagerdef enter(self) -> Generator[None]:"""Enter Flask execution context as context manager."""- # Restore non-Flask context variables to avoid leaking Flask tokens across threads- for var, val in self._context_vars.items():- var.set(val)-- # Enter Flask app context- with self._flask_app.app_context():- # Restore user in new app context+ with use_contextvars(self._context_vars), self._app_context.enter():if self._user is not None:g._login_user = self._useryielddiff --git a/api/controllers/console/app/completion.py b/api/controllers/console/app/completion.pyindex 647c2b55c80..b097914b5b8 100644--- a/api/controllers/console/app/completion.py+++ b/api/controllers/console/app/completion.py@@ -223,6 +223,8 @@ class CompletionMessageStopApi(Resource):@get_app_model(mode=AppMode.COMPLETION)def post(self, current_user_id: str, app_model: App, task_id: str):AppTaskService.stop_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,task_id=task_id,invoke_from=InvokeFrom.DEBUGGER,user_id=current_user_id,@@ -615,6 +617,8 @@ def _generate_chat_message_response(def _stop_chat_message(*, current_user_id: str, app_model: App, task_id: str):AppTaskService.stop_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,task_id=task_id,invoke_from=InvokeFrom.DEBUGGER,user_id=current_user_id,diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.pyindex 8e0bbb84a1d..f8a448a91d9 100644--- a/api/controllers/console/app/workflow.py+++ b/api/controllers/console/app/workflow.py@@ -52,7 +52,6 @@ from controllers.console.wraps import ()from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpErrorfrom core.app.app_config.features.file_upload.manager import FileUploadConfigManager-from core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.app.file_access import DatabaseFileAccessControllerfrom core.db.session_factory import session_factory@@ -83,7 +82,6 @@ from fields.workflow_run_fields import (from graphon.enums import NodeTypefrom graphon.file import Filefrom graphon.file import helpers as file_helpers-from graphon.graph_engine.manager import GraphEngineManagerfrom graphon.model_runtime.utils.encoders import jsonable_encoderfrom graphon.variables import SecretVariable, SegmentType, VariableBasefrom graphon.variables.exc import VariableError@@ -99,6 +97,7 @@ from repositories.workflow_collaboration_repository import WORKFLOW_ONLINE_USERSfrom services.agent.retirement_service import WorkflowAgentRetirementServicefrom services.app.access import resolve_app_access_filterfrom services.app_generate_service import AppGenerateService+from services.app_task_service import AppTaskServicefrom services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundErrorfrom services.errors.llm import InvokeRateLimitErrorfrom services.workflow_ref_service import WorkflowRefService@@ -1228,12 +1227,13 @@ class WorkflowTaskStopApi(Resource):"""Stop workflow task"""- # Stop using both mechanisms for backward compatibility- # Legacy stop flag mechanism (without user check)- AppQueueManager.set_stop_flag_no_user_check(task_id)-- # New graph engine command channel mechanism- GraphEngineManager(redis_client).send_stop_command(task_id)+ AppTaskService.stop_workflow_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,+ task_id=task_id,+ app_mode=AppMode.value_of(app_model.mode),+ owner=None,+ )return {"result": "success"}diff --git a/api/controllers/console/explore/completion.py b/api/controllers/console/explore/completion.pyindex c0056de1974..0505501436a 100644--- a/api/controllers/console/explore/completion.py+++ b/api/controllers/console/explore/completion.py@@ -36,6 +36,7 @@ from graphon.model_runtime.errors.invoke import InvokeErrorfrom libs import helperfrom libs.external_api import error_responsesfrom machinery.context import RequestContext+from models.enums import CreatorUserRolefrom models.model import AppModefrom services.account_errors import AccountNotFoundErrorfrom services.app_definition_query_service import AppDefinitionUnavailableError@@ -165,7 +166,6 @@ class CompletionStopApi(Resource):task_id=task_id,invoke_from=InvokeFrom.EXPLORE,user_id=request_context.account_id,- app_mode=AppMode.value_of(app_mode),)return SimpleResultResponse(result="success").model_dump(mode="json"), HTTPStatus.OK@@ -253,11 +253,19 @@ class ChatStopApi(Resource):if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:raise NotChatAppError()- application_services().app_tasks.stop_task(- task_id=task_id,- invoke_from=InvokeFrom.EXPLORE,- user_id=request_context.account_id,- app_mode=app_mode,- )+ if app_mode == AppMode.ADVANCED_CHAT:+ application_services().app_tasks.stop_workflow_task(+ tenant_id=installed_app.app_owner_tenant_id,+ app_id=installed_app.app_id,+ task_id=task_id,+ app_mode=app_mode,+ owner=(CreatorUserRole.ACCOUNT, request_context.account_id),+ )+ else:+ application_services().app_tasks.stop_task(+ task_id=task_id,+ invoke_from=InvokeFrom.EXPLORE,+ user_id=request_context.account_id,+ )return SimpleResultResponse(result="success").model_dump(mode="json"), HTTPStatus.OKdiff --git a/api/controllers/console/explore/trial.py b/api/controllers/console/explore/trial.pyindex acdb57863a2..7bc8b59bb56 100644--- a/api/controllers/console/explore/trial.py+++ b/api/controllers/console/explore/trial.py@@ -614,7 +614,13 @@ class TrialAppWorkflowTaskStopApi(Resource):if trial_app.app_mode != "workflow":raise NotWorkflowAppError()- application_services().app_tasks.stop_workflow_task_no_user_check(task_id=task_id)+ application_services().app_tasks.stop_workflow_task(+ tenant_id=trial_app.tenant_id,+ app_id=trial_app.app_id,+ task_id=task_id,+ app_mode=AppMode.WORKFLOW,+ owner=None,+ )return dump_response(SimpleResultResponse, {"result": "success"})@@ -700,12 +706,21 @@ class TrialChatTaskStopApi(Resource):if trial_app.app_mode not in {"chat", "agent-chat", "agent", "advanced-chat"}:raise NotChatAppError()- application_services().app_tasks.stop_task(- task_id=task_id,- invoke_from=InvokeFrom.EXPLORE,- user_id=request_context.account_id,- app_mode=AppMode.value_of(trial_app.app_mode),- )+ app_mode = AppMode.value_of(trial_app.app_mode)+ if app_mode == AppMode.ADVANCED_CHAT:+ application_services().app_tasks.stop_workflow_task(+ tenant_id=trial_app.tenant_id,+ app_id=trial_app.app_id,+ task_id=task_id,+ app_mode=app_mode,+ owner=(CreatorUserRole.ACCOUNT, request_context.account_id),+ )+ else:+ application_services().app_tasks.stop_task(+ task_id=task_id,+ invoke_from=InvokeFrom.EXPLORE,+ user_id=request_context.account_id,+ )return SimpleResultResponse(result="success").model_dump(mode="json"), 200diff --git a/api/controllers/console/explore/workflow.py b/api/controllers/console/explore/workflow.pyindex 872761b6c4d..ec54d0517fd 100644--- a/api/controllers/console/explore/workflow.py+++ b/api/controllers/console/explore/workflow.py@@ -30,6 +30,7 @@ from graphon.model_runtime.errors.invoke import InvokeErrorfrom libs import helperfrom libs.external_api import error_responsesfrom machinery.context import RequestContext+from models.model import AppModefrom services.account_errors import AccountNotFoundErrorfrom services.app_definition_query_service import AppDefinitionUnavailableErrorfrom services.errors.llm import InvokeRateLimitError@@ -109,6 +110,12 @@ class InstalledAppWorkflowTaskStopApi(Resource):if installed_app.app_mode != "workflow":raise NotWorkflowAppError()- application_services().app_tasks.stop_workflow_task_no_user_check(task_id=task_id)+ application_services().app_tasks.stop_workflow_task(+ tenant_id=installed_app.app_owner_tenant_id,+ app_id=installed_app.app_id,+ task_id=task_id,+ app_mode=AppMode.WORKFLOW,+ owner=None,+ )return SimpleResultResponse(result="success").model_dump(mode="json")diff --git a/api/controllers/console/human_input_form.py b/api/controllers/console/human_input_form.pyindex e3ca2c33fb5..ebe6113aebb 100644--- a/api/controllers/console/human_input_form.py+++ b/api/controllers/console/human_input_form.py@@ -4,7 +4,7 @@ Console/Studio Human Input Form APIs.import jsonimport logging-from collections.abc import Generator+from collections.abc import Generator, Sequencefrom typing import Anyfrom flask import Response, jsonify, request@@ -31,6 +31,7 @@ from core.app.apps.common.workflow_response_converter import WorkflowResponseConfrom core.app.apps.message_generator import MessageGeneratorfrom core.app.apps.workflow.app_generator import WorkflowAppGeneratorfrom core.workflow.human_input_policy import HumanInputSurface, is_recipient_type_allowed_for_surface+from core.workflow.nodes.human_input.entities import FormInputConfigfrom extensions.ext_database import dbfrom libs.external_api import error_responsesfrom libs.login import login_required@@ -68,8 +69,10 @@ register_response_schema_models()-def _jsonify_form_definition(form: Form) -> Response:+def _jsonify_form_definition(form: Form, *, inputs: Sequence[FormInputConfig] | None = None) -> Response:payload = form.get_definition().model_dump()+ if inputs is not None:+ payload["inputs"] = [form_input.model_dump(mode="json") for form_input in inputs]payload["expiration_time"] = int(form.expiration_time.timestamp())return Response(json.dumps(payload, ensure_ascii=False), mimetype="application/json")@@ -108,7 +111,7 @@ class ConsoleHumanInputFormApi(Resource):self._ensure_console_access(form, current_tenant_id)- return _jsonify_form_definition(form)+ return _jsonify_form_definition(form, inputs=service.resolve_form_inputs(form))@error_responses(FormExpiredError, FormSubmittedError, InvalidFormDataError)@account_initialization_required@@ -214,12 +217,12 @@ class ConsoleWorkflowEventsApi(Resource):else:msg_generator = MessageGenerator()- generator: BaseAppGenerator+ generator: type[BaseAppGenerator]match app.mode:case AppMode.ADVANCED_CHAT:- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGeneratorcase AppMode.WORKFLOW:- generator = WorkflowAppGenerator()+ generator = WorkflowAppGeneratorcase _:raise InvalidArgumentError(f"cannot subscribe to workflow run, workflow_run_id={workflow_run.id}")diff --git a/api/controllers/console/snippets/snippet_workflow.py b/api/controllers/console/snippets/snippet_workflow.pyindex fd281287850..e0960043209 100644--- a/api/controllers/console/snippets/snippet_workflow.py+++ b/api/controllers/console/snippets/snippet_workflow.py@@ -44,10 +44,8 @@ from controllers.console.wraps import (setup_required,with_current_user,)-from core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.db.session_factory import session_factory-from extensions.ext_redis import redis_clientfrom fields.workflow_run_fields import (WorkflowRunDetailResponse,WorkflowRunNodeExecutionListResponse,@@ -57,14 +55,15 @@ from fields.workflow_run_fields import (workflow_run_pagination_response_source,workflow_run_response_source,)-from graphon.graph_engine.manager import GraphEngineManagerfrom libs import helperfrom libs.external_api import error_responsesfrom libs.helper import TimestampFieldfrom libs.login import current_account_with_tenant, login_requiredfrom models import Account+from models.model import AppModefrom models.snippet import CustomizedSnippetfrom services.agent.workflow_publish_service import WorkflowAgentPublishService+from services.app_task_service import AppTaskServicefrom services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundErrorfrom services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseErrorfrom services.snippet_generate_service import SnippetGenerateService@@ -856,17 +855,13 @@ class SnippetWorkflowTaskStopApi(Resource):@get_snippet@edit_permission_requireddef post(self, snippet: CustomizedSnippet, task_id: str):- """- Stop a running snippet workflow task.-- Uses both the legacy stop flag mechanism and the graph engine- command channel for backward compatibility.- """- # Stop using both mechanisms for backward compatibility- # Legacy stop flag mechanism (without user check)- AppQueueManager.set_stop_flag_no_user_check(task_id)-- # New graph engine command channel mechanism- GraphEngineManager(redis_client).send_stop_command(task_id)+ """Stop a running or paused task belonging to this snippet."""+ AppTaskService.stop_workflow_task(+ tenant_id=snippet.tenant_id,+ app_id=snippet.id,+ task_id=task_id,+ app_mode=AppMode.WORKFLOW,+ owner=None,+ )return {"result": "success"}diff --git a/api/controllers/inner_api/plugin/plugin.py b/api/controllers/inner_api/plugin/plugin.pyindex 65103cee9bd..c10ec649ca4 100644--- a/api/controllers/inner_api/plugin/plugin.py+++ b/api/controllers/inner_api/plugin/plugin.py@@ -40,6 +40,7 @@ from libs.helper import length_prefixed_responsefrom models import Account, Tenantfrom models.model import EndUserfrom services.file_request_service import FileRequestService+from services.workflow_run_agg import WorkflowRunAgg@inner_api_ns.route("/invoke/llm")@@ -245,6 +246,7 @@ class PluginInvokeToolApi(Resource):def generator():return PluginToolBackwardsInvocation.convert_to_event_stream(PluginToolBackwardsInvocation.invoke_tool(+ execution_driver=WorkflowRunAgg.run,session=session,tenant_id=tenant_model.id,user_id=user_model.id,@@ -343,6 +345,7 @@ class PluginInvokeAppApi(Resource):@with_sessiondef post(self, session: Session, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeApp):response = PluginAppBackwardsInvocation.invoke_app(+ execution_driver=WorkflowRunAgg.run,session=session,app_id=payload.app_id,user_id=user_model.id,diff --git a/api/controllers/openapi/app_run.py b/api/controllers/openapi/app_run.pyindex dfc431dfe15..7ef165744f3 100644--- a/api/controllers/openapi/app_run.py+++ b/api/controllers/openapi/app_run.py@@ -62,7 +62,6 @@ from controllers.service_api.app.error import (TriggerWorkflowServiceModeUnavailableError,)from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError-from core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.app.entities.task_entities import MessageEndStreamResponse, StreamEventfrom core.errors.error import (@@ -71,8 +70,7 @@ from core.errors.error import (ProviderTokenNotInitError,QuotaExceededError,)-from extensions.ext_redis import redis_client-from graphon.graph_engine.manager import GraphEngineManager+from extensions.ext_application_services import application_servicesfrom graphon.model_runtime.errors.invoke import InvokeErrorfrom libs import helperfrom models.model import App, AppMode@@ -453,10 +451,13 @@ class AppRunTaskStopApi(Resource):returns=(200, TaskStopResponse, "Task stopped"),)def post(self, ctx: Context, app_id: str, task_id: str):- if isinstance(ctx.subject, ResourceAccessSubject):- owner = redis_client.get(AppQueueManager._generate_task_belong_cache_key(task_id))- if owner != f"end-user-{ctx.end_user.id}".encode():- raise NotFound("Task not found")- AppQueueManager.set_stop_flag_no_user_check(task_id)- GraphEngineManager(redis_client).send_stop_command(task_id)+ if not application_services().app_tasks.stop_app_task(+ tenant_id=ctx.app.tenant_id,+ app_id=ctx.app.id,+ task_id=task_id,+ app_mode=AppMode.value_of(ctx.app.mode),+ caller_role=ctx.subject.caller_role,+ caller_id=ctx.caller.id,+ ):+ raise NotFound("Task not found")return TaskStopResponse(result="success")diff --git a/api/controllers/openapi/workflow_events.py b/api/controllers/openapi/workflow_events.pyindex 1dbbcd5cc96..f986744b759 100644--- a/api/controllers/openapi/workflow_events.py+++ b/api/controllers/openapi/workflow_events.py@@ -127,8 +127,8 @@ class OpenApiWorkflowEventsApi(Resource):event_generator = _generate_finished_eventselse:msg_generator = MessageGenerator()- generator: BaseAppGenerator- generator = AdvancedChatAppGenerator() if app_mode == AppMode.ADVANCED_CHAT else WorkflowAppGenerator()+ generator: type[BaseAppGenerator]+ generator = AdvancedChatAppGenerator if app_mode == AppMode.ADVANCED_CHAT else WorkflowAppGeneratorinclude_state_snapshot = query.include_state_snapshotcontinue_on_pause = query.continue_on_pausediff --git a/api/controllers/service_api/app/completion.py b/api/controllers/service_api/app/completion.pyindex 02e65545780..a568fc68c1f 100644--- a/api/controllers/service_api/app/completion.py+++ b/api/controllers/service_api/app/completion.py@@ -321,6 +321,8 @@ class CompletionStopApi(Resource):raise AppUnavailableError()AppTaskService.stop_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,task_id=task_id,invoke_from=InvokeFrom.SERVICE_API,user_id=end_user.id,@@ -520,6 +522,8 @@ class ChatStopApi(Resource):raise NotChatAppError()AppTaskService.stop_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,task_id=task_id,invoke_from=InvokeFrom.SERVICE_API,user_id=end_user.id,diff --git a/api/controllers/service_api/app/workflow.py b/api/controllers/service_api/app/workflow.pyindex 9d351220543..4410413c168 100644--- a/api/controllers/service_api/app/workflow.py+++ b/api/controllers/service_api/app/workflow.py@@ -39,7 +39,6 @@ from controllers.service_api.schema import ()from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_tokenfrom controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError-from core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.errors.error import (ModelCurrentlyNotSupportError,@@ -50,12 +49,10 @@ from core.helper.trace_id_helper import get_external_trace_id, get_trace_sessionfrom enums import CloudPlan, DeploymentEditionfrom extensions.ext_application_services import application_servicesfrom extensions.ext_database import db-from extensions.ext_redis import redis_clientfrom fields.base import ResponseModelfrom fields.end_user_fields import SimpleEndUserfrom fields.member_fields import SimpleAccountResponsefrom graphon.enums import WorkflowExecutionStatus-from graphon.graph_engine.manager import GraphEngineManagerfrom graphon.model_runtime.errors.invoke import InvokeErrorfrom libs import helperfrom libs.external_api import error_responses@@ -63,6 +60,7 @@ from libs.helper import dump_response, to_timestampfrom models.model import App, AppMode, EndUserfrom repositories.factory import DifyAPIRepositoryFactoryfrom services.app_generate_service import AppGenerateService+from services.app_task_service import AppTaskServicefrom services.billing_service import BillingServicefrom services.errors.app import (IsDraftWorkflowError,@@ -588,12 +586,13 @@ class WorkflowTaskStopApi(Resource):if app_mode != AppMode.WORKFLOW:raise NotWorkflowAppError()- # Stop using both mechanisms for backward compatibility- # Legacy stop flag mechanism (without user check)- AppQueueManager.set_stop_flag_no_user_check(task_id)-- # New graph engine command channel mechanism- GraphEngineManager(redis_client).send_stop_command(task_id)+ AppTaskService.stop_workflow_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,+ task_id=task_id,+ app_mode=app_mode,+ owner=None,+ )return SimpleResultResponse(result="success").model_dump()diff --git a/api/controllers/service_api/app/workflow_events.py b/api/controllers/service_api/app/workflow_events.pyindex 75c21914c51..1aaf2bbf122 100644--- a/api/controllers/service_api/app/workflow_events.py+++ b/api/controllers/service_api/app/workflow_events.py@@ -136,11 +136,11 @@ class WorkflowEventsApi(Resource):event_generator = _generate_finished_eventselse:msg_generator = MessageGenerator()- generator: BaseAppGenerator+ generator: type[BaseAppGenerator]if app_mode == AppMode.ADVANCED_CHAT:- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGeneratorelif app_mode == AppMode.WORKFLOW:- generator = WorkflowAppGenerator()+ generator = WorkflowAppGeneratorelse:raise NotWorkflowAppError()diff --git a/api/controllers/web/completion.py b/api/controllers/web/completion.pyindex 21687ea0876..4918b61ea03 100644--- a/api/controllers/web/completion.py+++ b/api/controllers/web/completion.py@@ -187,6 +187,8 @@ class CompletionStopApi(WebApiResource):raise NotCompletionAppError()AppTaskService.stop_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,task_id=task_id,invoke_from=InvokeFrom.WEB_APP,user_id=end_user.id,@@ -302,6 +304,8 @@ class ChatStopApi(WebApiResource):raise NotChatAppError()AppTaskService.stop_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,task_id=task_id,invoke_from=InvokeFrom.WEB_APP,user_id=end_user.id,diff --git a/api/controllers/web/workflow.py b/api/controllers/web/workflow.pyindex c93c8d99721..0672eeb155d 100644--- a/api/controllers/web/workflow.py+++ b/api/controllers/web/workflow.py@@ -18,20 +18,18 @@ from controllers.web.error import ()from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpErrorfrom controllers.web.wraps import WebApiResource-from core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.errors.error import (ModelCurrentlyNotSupportError,ProviderTokenNotInitError,QuotaExceededError,)-from extensions.ext_redis import redis_client-from graphon.graph_engine.manager import GraphEngineManagerfrom graphon.model_runtime.errors.invoke import InvokeErrorfrom libs import helperfrom libs.external_api import error_responsesfrom models.model import App, AppMode, EndUserfrom services.app_generate_service import AppGenerateService+from services.app_task_service import AppTaskServicefrom services.errors.app import (TriggerWorkflowServiceModeUnavailableError as TriggerWorkflowServiceModeUnavailableServiceError,)@@ -139,11 +137,12 @@ class WorkflowTaskStopApi(WebApiResource):if app_mode != AppMode.WORKFLOW:raise NotWorkflowAppError()- # Stop using both mechanisms for backward compatibility- # Legacy stop flag mechanism (without user check)- AppQueueManager.set_stop_flag_no_user_check(task_id)-- # New graph engine command channel mechanism- GraphEngineManager(redis_client).send_stop_command(task_id)+ AppTaskService.stop_workflow_task(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,+ task_id=task_id,+ app_mode=app_mode,+ owner=None,+ )return SimpleResultResponse(result="success").model_dump(mode="json")diff --git a/api/controllers/web/workflow_events.py b/api/controllers/web/workflow_events.pyindex b8f2328f0f6..d74ebcfcdd5 100644--- a/api/controllers/web/workflow_events.py+++ b/api/controllers/web/workflow_events.py@@ -78,12 +78,12 @@ class WorkflowEventsApi(WebApiResource):else:app_mode = AppMode.value_of(app_model.mode)msg_generator = MessageGenerator()- generator: BaseAppGenerator+ generator: type[BaseAppGenerator]match app_mode:case AppMode.ADVANCED_CHAT:- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGeneratorcase AppMode.WORKFLOW:- generator = WorkflowAppGenerator()+ generator = WorkflowAppGeneratorcase _:raise InvalidArgumentError(f"cannot subscribe to workflow run, workflow_run_id={workflow_run.id}")diff --git a/api/core/agent/base_agent_runner.py b/api/core/agent/base_agent_runner.pyindex 37fb7d3539e..db45e465131 100644--- a/api/core/agent/base_agent_runner.py+++ b/api/core/agent/base_agent_runner.py@@ -2,7 +2,7 @@ import jsonimport loggingimport uuidfrom decimal import Decimal-from typing import cast+from typing import TYPE_CHECKING, castfrom sqlalchemy import func, selectfrom sqlalchemy.orm import Session@@ -44,6 +44,9 @@ from graphon.model_runtime.model_providers.base.large_language_model import Largfrom models.enums import CreatorUserRolefrom models.model import Conversation, Message, MessageAgentThought, MessageFile, load_annotation_reply_config+if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+logger = logging.getLogger(__name__)_file_access_controller = DatabaseFileAccessController()@@ -52,6 +55,7 @@ class BaseAgentRunner(AppRunner):def __init__(self,*,+ execution_driver: "WorkflowRunDriver",session: Session,tenant_id: str,application_generate_entity: AgentChatAppGenerateEntity,@@ -66,6 +70,7 @@ class BaseAgentRunner(AppRunner):memory: TokenBufferMemory | None = None,prompt_messages: list[PromptMessage] | None = None,):+ self._execution_driver = execution_driverself.tenant_id = tenant_idself.application_generate_entity = application_generate_entityself.conversation = conversation@@ -143,6 +148,7 @@ class BaseAgentRunner(AppRunner):convert tool to prompt message tool"""tool_entity = ToolManager.get_agent_tool_runtime(+ execution_driver=self._execution_driver,tenant_id=self.tenant_id,app_id=self.app_config.app_id,agent_tool=tool,diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.pyindex 27d0d8ae55c..1b3f52f6bf5 100644--- a/api/core/app/apps/advanced_chat/app_generator.py+++ b/api/core/app/apps/advanced_chat/app_generator.py@@ -4,7 +4,7 @@ import contextvarsimport loggingimport threadingimport uuid-from collections.abc import Generator, Mapping, Sequence+from collections.abc import Generator, Mappingfrom typing import TYPE_CHECKING, Any, Literal, overloadfrom flask import Flask, current_app@@ -18,6 +18,7 @@ from constants import UUID_NILif TYPE_CHECKING:from controllers.console.app.workflow import LoopNodeRunPayload+ from core.app.apps.workflow_app_runner import WorkflowRunDriverfrom core.app.app_config.features.file_upload.manager import FileUploadConfigManagerfrom core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManagerfrom core.app.apps.advanced_chat.app_runner import AdvancedChatAppRunner@@ -40,7 +41,7 @@ from core.app.entities.task_entities import (ChatbotAppBlockingResponse,ChatbotAppStreamResponse,)-from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, PauseStatePersistenceLayer+from core.app.layers.pause_state_persist_layer import PauseStateLayerConfigfrom core.helper.trace_id_helper import extract_external_trace_id_from_args, extract_trace_session_id_from_argsfrom core.ops.ops_trace_manager import TraceQueueManagerfrom core.prompt.utils.get_thread_messages_length import get_thread_messages_length@@ -48,10 +49,9 @@ from core.repositories import DifyCoreRepositoryFactoryfrom core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepositoryfrom extensions.ext_database import dbfrom factories import file_factory-from graphon.filters import ResponseStreamFilter-from graphon.graph_engine.layers import GraphEngineLayer+from graphon.engine.filter import ResponseStreamFilterfrom graphon.model_runtime.errors.invoke import InvokeAuthorizationError-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoaderfrom libs.flask_utils import preserve_flask_contextsfrom models import Account, App, Conversation, EndUser, Message, Workflow, WorkflowNodeExecutionTriggeredFrom@@ -73,6 +73,9 @@ def _extract_trace_session_id_from_debug_args(args: Mapping[str, Any] | Any) ->class AdvancedChatAppGenerator(MessageBasedAppGenerator):+ def __init__(self, *, execution_driver: WorkflowRunDriver) -> None:+ self._execution_driver = execution_driver+_dialogue_count: int@overload@@ -281,7 +284,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):application_generate_entity: AdvancedChatAppGenerateEntity,workflow_execution_repository: WorkflowExecutionRepository,workflow_node_execution_repository: WorkflowNodeExecutionRepository,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,pause_state_config: PauseStateLayerConfig | None = None,response_stream_filter: ResponseStreamFilter | None = None,):@@ -520,8 +523,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):stream: bool = True,variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER,pause_state_config: PauseStateLayerConfig | None = None,- graph_runtime_state: GraphRuntimeState | None = None,- graph_engine_layers: Sequence[GraphEngineLayer] = (),+ graph_runtime_state: RuntimeState | None = None,response_stream_filter: ResponseStreamFilter | None = None,) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]:"""@@ -574,18 +576,6 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):message_id=message.id,)- graph_layers: list[GraphEngineLayer] = list(graph_engine_layers)- resolved_response_stream_filter = response_stream_filter or ResponseStreamFilter()- if pause_state_config is not None:- graph_layers.append(- PauseStatePersistenceLayer(- session_factory=pause_state_config.session_factory,- generate_entity=application_generate_entity,- state_owner_user_id=pause_state_config.state_owner_user_id,- response_stream_filter=resolved_response_stream_filter,- )- )-# new thread with request context and contextvarscontext = contextvars.copy_context()@@ -601,9 +591,9 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):"variable_loader": variable_loader,"workflow_execution_repository": workflow_execution_repository,"workflow_node_execution_repository": workflow_node_execution_repository,- "graph_engine_layers": tuple(graph_layers),+ "pause_state_config": pause_state_config,"graph_runtime_state": graph_runtime_state,- "response_stream_filter": resolved_response_stream_filter,+ "response_stream_filter": response_stream_filter,},)@@ -656,9 +646,9 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):variable_loader: VariableLoader,workflow_execution_repository: WorkflowExecutionRepository,workflow_node_execution_repository: WorkflowNodeExecutionRepository,- graph_engine_layers: Sequence[GraphEngineLayer] = (),- graph_runtime_state: GraphRuntimeState | None = None,+ graph_runtime_state: RuntimeState | None = None,response_stream_filter: ResponseStreamFilter | None = None,+ pause_state_config: PauseStateLayerConfig | None = None,):"""Generate worker in a new thread.@@ -712,6 +702,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):runner = AdvancedChatAppRunner(application_generate_entity=application_generate_entity,+ execution_driver=self._execution_driver,queue_manager=queue_manager,conversation=conversation,message=message,@@ -720,16 +711,15 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):workflow=workflow,system_user_id=system_user_id,app=app,- workflow_execution_repository=workflow_execution_repository,- workflow_node_execution_repository=workflow_node_execution_repository,- graph_engine_layers=graph_engine_layers,graph_runtime_state=graph_runtime_state,response_stream_filter=response_stream_filter,)try:with active_workflow_task(application_generate_entity.task_id):- runner.run()+ self._execution_driver(+ runner, pause_state_config, workflow_execution_repository, workflow_node_execution_repository+ )except GenerateTaskStoppedError:passexcept InvokeAuthorizationError:diff --git a/api/core/app/apps/advanced_chat/app_runner.py b/api/core/app/apps/advanced_chat/app_runner.pyindex 30470c0383b..f350345c764 100644--- a/api/core/app/apps/advanced_chat/app_runner.py+++ b/api/core/app/apps/advanced_chat/app_runner.py@@ -1,7 +1,7 @@import loggingimport time-from collections.abc import Mapping, Sequence-from typing import Any, cast+from collections.abc import Mapping+from typing import Any, cast, overridefrom sqlalchemy import selectfrom sqlalchemy.orm import Session@@ -14,11 +14,10 @@ from core.app.apps.workflow.command_channels import (StopFlagCommandChannel,)from core.app.apps.workflow.stop_aware_ready_queue import attach_stop_aware_ready_queue-from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner+from core.app.apps.workflow_app_runner import PreparedWorkflowRun, WorkflowBasedAppRunner, WorkflowRunDriverfrom core.app.entities.app_invoke_entities import (AdvancedChatAppGenerateEntity,AppGenerateEntity,- DifyRunContext,InvokeFrom,get_credit_usage_app_type,)@@ -28,14 +27,10 @@ from core.app.entities.queue_entities import (QueueTextChunkEvent,)from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature-from core.app.layers.conversation_variable_persist_layer import ConversationVariablePersistenceLayer-from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer-from core.db.session_factory import create_session, session_factory+from core.db.session_factory import create_sessionfrom core.moderation.base import ModerationErrorfrom core.moderation.input_moderation import InputModeration-from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepositoryfrom core.workflow.node_factory import get_default_root_node_id-from core.workflow.nodes.agent_v2.workspace_retirement_layer import build_workflow_agent_workspace_retirement_layerfrom core.workflow.system_variables import (build_bootstrap_variables,build_system_variables,@@ -44,19 +39,16 @@ from core.workflow.system_variables import (from core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_poolfrom core.workflow.workflow_entry import WorkflowEntryfrom extensions.ext_redis import redis_client-from extensions.otel import WorkflowAppRunnerHandler, trace_spanfrom extensions.workflow_warm_shutdown import WORKFLOW_WARM_SHUTDOWN_ABORT_REASON, celery_warm_shutdown_started+from graphon.engine.command import RedisChannel+from graphon.engine.filter import ResponseStreamFilterfrom graphon.enums import WorkflowType-from graphon.filters import ResponseStreamFilter-from graphon.graph_engine.command_channels import RedisChannel-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variable_loader import VariableLoaderfrom graphon.variables.variables import Variablefrom models import Workflowfrom models.model import App, Conversation, Message, MessageAnnotationfrom models.workflow import ConversationVariable-from services.conversation_variable_updater import ConversationVariableUpdaterlogger = logging.getLogger(__name__)@@ -78,17 +70,15 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):workflow: Workflow,system_user_id: str,app: App,- workflow_execution_repository: WorkflowExecutionRepository,- workflow_node_execution_repository: WorkflowNodeExecutionRepository,- graph_engine_layers: Sequence[GraphEngineLayer] = (),- graph_runtime_state: GraphRuntimeState | None = None,+ execution_driver: WorkflowRunDriver | None = None,+ graph_runtime_state: RuntimeState | None = None,response_stream_filter: ResponseStreamFilter | None = None,):super().__init__(queue_manager=queue_manager,+ execution_driver=execution_driver,variable_loader=variable_loader,app_id=application_generate_entity.app_config.app_id,- graph_engine_layers=graph_engine_layers,)self.application_generate_entity = application_generate_entityself.conversation = conversation@@ -97,13 +87,11 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):self._workflow = workflowself.system_user_id = system_user_idself._app = app- self._workflow_execution_repository = workflow_execution_repository- self._workflow_node_execution_repository = workflow_node_execution_repositoryself._resume_graph_runtime_state = graph_runtime_stateself._response_stream_filter = response_stream_filter- @trace_span(WorkflowAppRunnerHandler)- def run(self):+ @override+ def prepare(self) -> PreparedWorkflowRun | None:app_config = self.application_generate_entity.app_configapp_config = cast(AdvancedChatAppConfig, app_config)@@ -144,6 +132,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):user_from=user_from,app_type=get_credit_usage_app_type(app_config.app_mode),trace_session_id=self.application_generate_entity.extras.get("trace_session_id"),+ call_depth=self.application_generate_entity.call_depth,)elif self.application_generate_entity.single_iteration_run or self.application_generate_entity.single_loop_run:# Handle single iteration or single loop run@@ -168,7 +157,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):message_id=self.message.id,)if stop:- return+ return Noneself.application_generate_entity.inputs = new_inputsself.application_generate_entity.query = new_query@@ -193,7 +182,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):text=annotation_reply.content,stopped_by=QueueStopEvent.StopBy.ANNOTATION_REPLY,)- return+ return None# Initialize conversation variablesconversation_variables = self._initialize_conversation_variables()@@ -213,7 +202,9 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):add_node_inputs_to_pool(variable_pool, node_id=root_node_id, inputs=new_inputs)# init graph- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.time())+ graph_runtime_state = RuntimeState(+ workflow_id=self._workflow.id, variable_pool=variable_pool, start_at=time.time()+ )graph = self._init_graph(graph_config=self._workflow.graph_dict,graph_runtime_state=graph_runtime_state,@@ -225,6 +216,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):root_node_id=root_node_id,app_type=get_credit_usage_app_type(app_config.app_mode),trace_session_id=self.application_generate_entity.extras.get("trace_session_id"),+ call_depth=self.application_generate_entity.call_depth,)# RUN WORKFLOW@@ -245,16 +237,9 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):)workflow_entry = WorkflowEntry(- tenant_id=self._workflow.tenant_id,- app_id=self._workflow.app_id,- workflow_id=self._workflow.id,graph=graph,graph_config=self._workflow.graph_dict,- user_id=self.application_generate_entity.user_id,- user_from=user_from,- invoke_from=invoke_from,call_depth=self.application_generate_entity.call_depth,- variable_pool=variable_pool,graph_runtime_state=graph_runtime_state,command_channel=command_channel,response_stream_filter=self._response_stream_filter,@@ -262,44 +247,14 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):self._queue_manager.graph_runtime_state = graph_runtime_state- persistence_layer = WorkflowPersistenceLayer(- application_generate_entity=self.application_generate_entity,- workflow_info=PersistenceWorkflowInfo(- workflow_id=self._workflow.id,- workflow_type=WorkflowType(self._workflow.type),- version=self._workflow.version,- graph_data=self._workflow.graph_dict,- ),- workflow_execution_repository=self._workflow_execution_repository,- workflow_node_execution_repository=self._workflow_node_execution_repository,- trace_manager=self.application_generate_entity.trace_manager,+ return PreparedWorkflowRun(+ entry=workflow_entry,+ application_layers=self._graph_engine_layers,+ generate_entity=self.application_generate_entity,+ workflow_type=WorkflowType(self._workflow.type),+ workflow_version=self._workflow.version,)- workflow_entry.graph_engine.layer(persistence_layer)- workflow_entry.graph_engine.layer(- build_workflow_agent_workspace_retirement_layer(- dify_run_context=DifyRunContext(- tenant_id=self._workflow.tenant_id,- app_id=self._workflow.app_id,- user_id=self.application_generate_entity.user_id,- user_from=user_from,- invoke_from=invoke_from,- app_type=get_credit_usage_app_type(app_config.app_mode),- trace_session_id=self.application_generate_entity.extras.get("trace_session_id"),- )- )- )- conversation_variable_layer = ConversationVariablePersistenceLayer(- ConversationVariableUpdater(session_factory.get_session_maker())- )- workflow_entry.graph_engine.layer(conversation_variable_layer)- for layer in self._graph_engine_layers:- workflow_entry.graph_engine.layer(layer)-- generator = workflow_entry.run()- for event in generator:- self._handle_event(workflow_entry, event)-def handle_input_moderation(self,app_record: App,diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.pyindex f837c9f8426..3d9581f06e7 100644--- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py+++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py@@ -74,7 +74,6 @@ from core.base.tts import AppGeneratorTTSPublisherfrom core.db.session_factory import session_factoryfrom core.ops.entities.trace_entity import TraceTaskNamefrom core.ops.ops_trace_manager import TraceQueueManager, TraceTask-from core.repositories.human_input_repository import HumanInputFormRepositoryImplfrom core.workflow.file_reference import resolve_file_record_idfrom core.workflow.nodes.human_input.pause_reason import HumanInputRequiredfrom core.workflow.system_variables import build_system_variables@@ -82,13 +81,13 @@ from graphon.enums import WorkflowExecutionStatusfrom graphon.model_runtime.entities.llm_entities import LLMUsagefrom graphon.model_runtime.utils.encoders import jsonable_encoderfrom graphon.nodes import BuiltinNodeTypes-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom libs.datetime_utils import naive_utc_nowfrom models import Account, Conversation, EndUser, Message, MessageFilefrom models.enums import CreatorUserRole, MessageFileBelongsTo, MessageStatusfrom models.execution_extra_content import HumanInputContentfrom models.model import AppMode-from models.workflow import Workflow+from models.workflow import Workflow, WorkflowRunlogger = logging.getLogger(__name__)@@ -209,7 +208,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):self._recorded_files: list[Mapping[str, Any]] = []self._workflow_run_id: str = ""self._draft_var_saver_factory = draft_var_saver_factory- self._graph_runtime_state: GraphRuntimeState | None = None+ self._graph_runtime_state: RuntimeState | None = Noneself._message_saved_on_pause = Falseself._seed_graph_runtime_state_from_queue_manager()@@ -452,6 +451,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):workflow_run_id=run_id,workflow_id=self._workflow_id,reason=event.reason,+ node_execution_snapshots=event.node_execution_snapshots,)yield workflow_start_resp@@ -709,18 +709,28 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):)for reason in event.reasons:if isinstance(reason, HumanInputRequired):- self._persist_human_input_extra_content(form_id=reason.form_id, node_id=reason.node_id)+ self._persist_human_input_extra_content(form_id=reason.form_id)yield from responses- resolved_state: GraphRuntimeState | None = None+ resolved_state: RuntimeState | None = Nonetry:resolved_state = self._ensure_graph_runtime_initialized()except ValueError:resolved_state = Nonewith self._database_session() as session:+ # Serialize the message's pause state with durable stop on the run.+ workflow_run_status = session.scalar(+ select(WorkflowRun.status)+ .where(+ WorkflowRun.id == self._workflow_run_id,+ WorkflowRun.tenant_id == self._workflow_tenant_id,+ WorkflowRun.app_id == self._application_generate_entity.app_config.app_id,+ )+ .with_for_update()+ )self._save_message(session=session, graph_runtime_state=resolved_state)message = self._get_message(session=session)- if message is not None:+ if message is not None and workflow_run_status == WorkflowExecutionStatus.PAUSED:message.status = MessageStatus.PAUSEDself._message_saved_on_pause = Trueself._base_task_pipeline.queue_manager.publish(QueueAdvancedChatMessageEndEvent(), PublishFrom.TASK_PIPELINE)@@ -757,7 +767,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):self,event: QueueStopEvent,*,- graph_runtime_state: GraphRuntimeState | None = None,+ graph_runtime_state: RuntimeState | None = None,trace_manager: TraceQueueManager | None = None,**kwargs,) -> Generator[StreamResponse]:@@ -798,7 +808,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):self,event: QueueAdvancedChatMessageEndEvent,*,- graph_runtime_state: GraphRuntimeState | None = None,+ graph_runtime_state: RuntimeState | None = None,**kwargs,) -> Generator[StreamResponse]:"""Handle advanced chat message end events."""@@ -843,7 +853,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):self, event: QueueHumanInputFormFilledEvent, **kwargs) -> Generator[StreamResponse]:"""Handle human input form filled events."""- self._persist_human_input_extra_content(node_id=event.node_id)+ self._persist_human_input_extra_content(form_id=event.form_id)yield self._workflow_response_converter.human_input_form_filled_to_stream_response(event=event, task_id=self._application_generate_entity.task_id)@@ -856,22 +866,10 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):event=event, task_id=self._application_generate_entity.task_id)- def _persist_human_input_extra_content(self, *, node_id: str | None = None, form_id: str | None = None) -> None:+ def _persist_human_input_extra_content(self, *, form_id: str) -> None:if not self._workflow_run_id or not self._message_id:return- if form_id is None:- if node_id is None:- return- form_id = self._load_human_input_form_id(node_id=node_id)- if form_id is None:- logger.warning(- "HumanInput form not found for workflow run %s node %s",- self._workflow_run_id,- node_id,- )- return-with self._database_session() as session:exists_stmt = select(HumanInputContent).where(HumanInputContent.workflow_run_id == self._workflow_run_id,@@ -888,16 +886,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):)session.add(content)- def _load_human_input_form_id(self, *, node_id: str) -> str | None:- form_repository = HumanInputFormRepositoryImpl(- tenant_id=self._workflow_tenant_id,- workflow_execution_id=self._workflow_run_id,- )- form = form_repository.get_form(node_id)- if form is None:- return None- return form.id-def _handle_agent_log_event(self, event: QueueAgentLogEvent, **kwargs) -> Generator[StreamResponse]:"""Handle agent log events."""yield self._workflow_response_converter.handle_agent_log(@@ -1038,7 +1026,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):if self._conversation_name_generate_thread:logger.debug("Conversation name generation running as daemon thread")- def _save_message(self, *, session: Session, graph_runtime_state: GraphRuntimeState | None = None):+ def _save_message(self, *, session: Session, graph_runtime_state: RuntimeState | None = None):message = self._get_message(session=session)if message is None:returndiff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.pyindex 5f123b01e6f..8cc709004d7 100644--- a/api/core/app/apps/agent_chat/app_generator.py+++ b/api/core/app/apps/agent_chat/app_generator.py@@ -3,7 +3,7 @@ import loggingimport threadingimport uuidfrom collections.abc import Generator, Mapping-from typing import Any, Literal, overload+from typing import TYPE_CHECKING, Any, Literal, overloadfrom flask import Flask, current_appfrom pydantic import ValidationError@@ -31,10 +31,16 @@ from models import Account, App, EndUserfrom models.model import load_annotation_reply_configfrom services.conversation_service import ConversationService+if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+logger = logging.getLogger(__name__)class AgentChatAppGenerator(MessageBasedAppGenerator):+ def __init__(self, *, execution_driver: "WorkflowRunDriver") -> None:+ self._execution_driver = execution_driver+@overloaddef generate(self,@@ -272,7 +278,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):message = self._get_message(message_id)# chatbot app- runner = AgentChatAppRunner()+ runner = AgentChatAppRunner(execution_driver=self._execution_driver)with session_factory.create_session() as session:runner.run(application_generate_entity=application_generate_entity,diff --git a/api/core/app/apps/agent_chat/app_runner.py b/api/core/app/apps/agent_chat/app_runner.pyindex ddfc2dac96a..b67565f5d9e 100644--- a/api/core/app/apps/agent_chat/app_runner.py+++ b/api/core/app/apps/agent_chat/app_runner.py@@ -1,5 +1,5 @@import logging-from typing import cast+from typing import TYPE_CHECKING, castfrom sqlalchemy import selectfrom sqlalchemy.orm import Session@@ -22,6 +22,9 @@ from graphon.model_runtime.entities.model_entities import ModelFeature, ModelProfrom graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModelfrom models.model import App, Conversation, Message+if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+logger = logging.getLogger(__name__)@@ -30,6 +33,9 @@ class AgentChatAppRunner(AppRunner):Agent Application Runner"""+ def __init__(self, *, execution_driver: "WorkflowRunDriver") -> None:+ self._execution_driver = execution_driver+def run(self,application_generate_entity: AgentChatAppGenerateEntity,@@ -225,6 +231,7 @@ class AgentChatAppRunner(AppRunner):raise ValueError(f"Invalid agent strategy: {agent_entity.strategy}")runner = runner_cls(+ execution_driver=self._execution_driver,session=session,tenant_id=app_config.tenant_id,application_generate_entity=application_generate_entity,diff --git a/api/core/app/apps/base_app_queue_manager.py b/api/core/app/apps/base_app_queue_manager.pyindex 24909a79896..03c517c782d 100644--- a/api/core/app/apps/base_app_queue_manager.py+++ b/api/core/app/apps/base_app_queue_manager.py@@ -26,7 +26,7 @@ from core.app.entities.queue_entities import (WorkflowQueueMessage,)from extensions.ext_redis import RedisClientWrapper, redis_client-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatelogger = logging.getLogger(__name__)@@ -64,7 +64,7 @@ class AppQueueManager(ABC):q: queue.Queue[WorkflowQueueMessage | MessageQueueMessage | None] = queue.Queue()self._q = q- self._graph_runtime_state: GraphRuntimeState | None = None+ self._graph_runtime_state: RuntimeState | None = Noneself._stopped_cache: TTLCache[tuple, bool] = TTLCache(maxsize=1, ttl=1)self._cache_lock = threading.Lock()self._listener_segment_completed = threading.Event()@@ -152,12 +152,12 @@ class AppQueueManager(ABC):self.publish(QueueErrorEvent(error=e), pub_from)@property- def graph_runtime_state(self) -> GraphRuntimeState | None:+ def graph_runtime_state(self) -> RuntimeState | None:"""Retrieve the attached graph runtime state, if available."""return self._graph_runtime_state@graph_runtime_state.setter- def graph_runtime_state(self, graph_runtime_state: GraphRuntimeState | None) -> None:+ def graph_runtime_state(self, graph_runtime_state: RuntimeState | None) -> None:"""Attach the live graph runtime state reference for downstream consumers."""self._graph_runtime_state = graph_runtime_state@@ -188,22 +188,20 @@ class AppQueueManager(ABC):@classmethoddef set_stop_flag(cls, task_id: str, invoke_from: InvokeFrom, user_id: str, *, redis: RedisClientWrapper | None = None- ) -> None:- """- Set task stop flag- :return:- """+ ) -> bool:+ """Set the task stop flag only for its owner; return whether it was accepted."""client = redis if redis is not None else redis_clientresult: Any | None = client.get(cls._generate_task_belong_cache_key(task_id))if result is None:- return+ return Falseuser_prefix = "account" if invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} else "end-user"if result.decode("utf-8") != f"{user_prefix}-{user_id}":- return+ return Falsestopped_cache_key = cls._generate_stopped_cache_key(task_id)client.setex(stopped_cache_key, 600, 1)+ return True@classmethoddef set_stop_flag_no_user_check(cls, task_id: str) -> None:diff --git a/api/core/app/apps/common/graph_runtime_state_support.py b/api/core/app/apps/common/graph_runtime_state_support.pyindex 2a90fbdad0e..8f2f56cd3d9 100644--- a/api/core/app/apps/common/graph_runtime_state_support.py+++ b/api/core/app/apps/common/graph_runtime_state_support.py@@ -1,11 +1,11 @@-"""Shared helpers for managing GraphRuntimeState across task pipelines."""+"""Shared helpers for managing RuntimeState across task pipelines."""from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom core.workflow.system_variables import SystemVariableKey, get_system_text-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStateif TYPE_CHECKING:from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline@@ -13,7 +13,7 @@ if TYPE_CHECKING:class GraphRuntimeStateSupport:"""- Mixin that centralises common GraphRuntimeState access patterns used by task pipelines.+ Mixin that centralises common RuntimeState access patterns used by task pipelines.Subclasses are expected to provide:* `_base_task_pipeline` – exposing the queue manager with an optional cached runtime state.@@ -21,16 +21,16 @@ class GraphRuntimeStateSupport:"""_base_task_pipeline: BasedGenerateTaskPipeline- _graph_runtime_state: GraphRuntimeState | None = None+ _graph_runtime_state: RuntimeState | None = Nonedef _ensure_graph_runtime_initialized(self,- graph_runtime_state: GraphRuntimeState | None = None,- ) -> GraphRuntimeState:+ graph_runtime_state: RuntimeState | None = None,+ ) -> RuntimeState:"""Validate and return the active graph runtime state."""return self._resolve_graph_runtime_state(graph_runtime_state)- def _extract_workflow_run_id(self, graph_runtime_state: GraphRuntimeState) -> str:+ def _extract_workflow_run_id(self, graph_runtime_state: RuntimeState) -> str:workflow_run_id = get_system_text(graph_runtime_state.variable_pool, SystemVariableKey.WORKFLOW_EXECUTION_ID)if not workflow_run_id:raise ValueError("workflow_execution_id missing from runtime state")@@ -38,8 +38,8 @@ class GraphRuntimeStateSupport:def _resolve_graph_runtime_state(self,- graph_runtime_state: GraphRuntimeState | None = None,- ) -> GraphRuntimeState:+ graph_runtime_state: RuntimeState | None = None,+ ) -> RuntimeState:"""Return the cached runtime state or bootstrap it from the queue manager."""if graph_runtime_state is not None:self._graph_runtime_state = graph_runtime_statediff --git a/api/core/app/apps/common/workflow_response_converter.py b/api/core/app/apps/common/workflow_response_converter.pyindex 205810adc9f..abfefa9bfeb 100644--- a/api/core/app/apps/common/workflow_response_converter.py+++ b/api/core/app/apps/common/workflow_response_converter.py@@ -2,7 +2,6 @@ import jsonimport loggingimport timefrom collections.abc import Mapping, Sequence-from dataclasses import dataclassfrom datetime import datetimefrom typing import Any, NewType, TypedDict@@ -11,6 +10,7 @@ from sqlalchemy.orm import Sessionfrom core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom, WorkflowAppGenerateEntityfrom core.app.entities.queue_entities import (+ NodeExecutionSnapshot,QueueAgentLogEvent,QueueHumanInputFormFilledEvent,QueueHumanInputFormTimeoutEvent,@@ -72,7 +72,7 @@ from graphon.enums import (WorkflowNodeExecutionStatus,)from graphon.file import FILE_MODEL_IDENTITY, File-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom graphon.variables.segments import ArrayFileSegment, FileSegment, Segmentfrom graphon.variables.variables import Variablefrom graphon.workflow_type_encoder import WorkflowRuntimeTypeConverter@@ -108,19 +108,6 @@ class EndUserCreatedByDict(TypedDict):CreatedByDict = AccountCreatedByDict | EndUserCreatedByDict-@dataclass(slots=True)-class _NodeSnapshot:- """In-memory cache for node metadata between start and completion events."""-- title: str- index: int- start_at: datetime- iteration_id: str = ""- """Empty string means the node is not executing inside an iteration."""- loop_id: str = ""- """Empty string means the node is not executing inside a loop."""--class WorkflowResponseConverter:_truncator: BaseTruncator@@ -142,7 +129,7 @@ class WorkflowResponseConverter:else:self._truncator = VariableTruncator.default()- self._node_snapshots: dict[NodeExecutionId, _NodeSnapshot] = {}+ self._node_snapshots: dict[NodeExecutionId, NodeExecutionSnapshot] = {}self._workflow_execution_id: str | None = Noneself._workflow_started_at: datetime | None = None@@ -173,8 +160,14 @@ class WorkflowResponseConverter:# ------------------------------------------------------------------# Node snapshot helpers# ------------------------------------------------------------------- def _store_snapshot(self, event: QueueNodeStartedEvent) -> _NodeSnapshot:- snapshot = _NodeSnapshot(+ def _store_snapshot(self, event: QueueNodeStartedEvent) -> NodeExecutionSnapshot:+ # The engine may have persisted this start before resumption metadata was loaded.+ snapshot = self._get_snapshot(event.node_execution_id)+ if snapshot is not None:+ return snapshot++ snapshot = NodeExecutionSnapshot(+ execution_id=event.node_execution_id,title=event.node_title,index=event.node_run_index,start_at=event.start_at,@@ -185,16 +178,16 @@ class WorkflowResponseConverter:self._node_snapshots[node_execution_id] = snapshotreturn snapshot- def _get_snapshot(self, node_execution_id: str) -> _NodeSnapshot | None:+ def _get_snapshot(self, node_execution_id: str) -> NodeExecutionSnapshot | None:return self._node_snapshots.get(NodeExecutionId(node_execution_id))- def _pop_snapshot(self, node_execution_id: str) -> _NodeSnapshot | None:+ def _pop_snapshot(self, node_execution_id: str) -> NodeExecutionSnapshot | None:return self._node_snapshots.pop(NodeExecutionId(node_execution_id), None)@staticmethoddef _merge_metadata(base_metadata: Mapping[WorkflowNodeExecutionMetadataKey, Any] | None,- snapshot: _NodeSnapshot | None,+ snapshot: NodeExecutionSnapshot | None,) -> Mapping[WorkflowNodeExecutionMetadataKey, Any] | None:if not base_metadata and not snapshot:return base_metadata@@ -241,11 +234,16 @@ class WorkflowResponseConverter:workflow_run_id: str,workflow_id: str,reason: WorkflowStartReason,+ node_execution_snapshots: Sequence[NodeExecutionSnapshot] = (),) -> WorkflowStartStreamResponse:run_id = self._ensure_workflow_run_id(workflow_run_id)started_at = naive_utc_now()self._workflow_started_at = started_atself._application_generate_entity.workflow_execution_timing.start()+ if reason == WorkflowStartReason.RESUMPTION:+ self._node_snapshots.update(+ (NodeExecutionId(snapshot.execution_id), snapshot) for snapshot in node_execution_snapshots+ )return WorkflowStartStreamResponse(task_id=task_id,@@ -265,7 +263,7 @@ class WorkflowResponseConverter:task_id: str,workflow_id: str,status: WorkflowExecutionStatus,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,error: str | None = None,exceptions_count: int = 0,) -> WorkflowFinishStreamResponse:@@ -322,7 +320,7 @@ class WorkflowResponseConverter:*,event: QueueWorkflowPausedEvent,task_id: str,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,) -> list[StreamResponse]:run_id = self._ensure_workflow_run_id()started_at = self._workflow_started_at@@ -429,6 +427,7 @@ class WorkflowResponseConverter:) -> HumanInputFormFilledResponse:run_id = self._ensure_workflow_run_id()data = HumanInputFormFilledResponse.Data(+ form_id=event.form_id,node_id=event.node_id,node_title=event.node_title,rendered_content=event.rendered_content,@@ -450,6 +449,7 @@ class WorkflowResponseConverter:task_id=task_id,workflow_run_id=run_id,data=HumanInputFormTimeoutResponse.Data(+ form_id=event.form_id,node_id=event.node_id,node_title=event.node_title,expiration_time=to_utc_timestamp(event.expiration_time),@@ -512,10 +512,10 @@ class WorkflowResponseConverter:event: QueueNodeStartedEvent,task_id: str,) -> NodeStartStreamResponse | None:+ snapshot = self._store_snapshot(event)if event.node_type in {BuiltinNodeTypes.ITERATION, BuiltinNodeTypes.LOOP}:return Nonerun_id = self._ensure_workflow_run_id()- snapshot = self._store_snapshot(event)response = NodeStartStreamResponse(task_id=task_id,diff --git a/api/core/app/apps/execution_coordinator.py b/api/core/app/apps/execution_coordinator.pyindex f8ef6347f83..7a28e3c8a34 100644--- a/api/core/app/apps/execution_coordinator.py+++ b/api/core/app/apps/execution_coordinator.py@@ -8,8 +8,7 @@ from enum import Enum, autofrom configs import dify_configfrom extensions.ext_redis import RedisClientWrapper, redis_client-from graphon.graph_engine.command_channels import RedisChannel-from graphon.graph_engine.manager import GraphEngineManager+from graphon.engine.command import AbortCommand, RedisChannellogger = logging.getLogger(__name__)@@ -31,6 +30,20 @@ def app_task_stop_flag_key(task_id: str) -> str:return f"generate_task_stopped:{task_id}"+def send_abort_command(task_id: str, reason: str | None = None, *, redis: RedisClientWrapper | None = None) -> None:+ """Send an abort command to ``task_id``, using the global Redis client when none is injected."""+ if not task_id:+ return++ try:+ RedisChannel(redis if redis is not None else redis_client, app_task_command_channel_key(task_id)).send_command(+ AbortCommand(reason=reason or "User requested stop")+ )+ except Exception:+ # The legacy stop flag remains the fallback when Redis is unavailable.+ logger.exception("Failed to send Engine abort command for task %s", task_id)++def set_app_task_stop_flag(task_id: str, *, redis: RedisClientWrapper | None = None) -> None:if not task_id:return@@ -67,10 +80,7 @@ def clear_app_task_cancellation_signals(task_id: str) -> None:channel_key = app_task_command_channel_key(task_id)try:- # fetch_commands() drains the queue and its pending marker together; the- # explicit delete covers a queue whose marker was already consumed.discarded = RedisChannel(redis_client, channel_key).fetch_commands()- redis_client.delete(channel_key)if discarded:logger.info("Discarded %s stale GraphEngine command(s) for app task %s",@@ -212,14 +222,7 @@ class AppExecutionCoordinator:self._attempt_id,)- try:- GraphEngineManager(redis_client).send_stop_command(self._task_id, reason=reason)- except Exception:- logger.exception(- "Failed to send stop command for app execution task=%s attempt=%s",- self._task_id,- self._attempt_id,- )+ send_abort_command(self._task_id, reason=reason)def _detach_watchdog_locked(self) -> threading.Timer | None:watchdog = self._watchdogdiff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.pyindex b2459247c9d..3b4773bc748 100644--- a/api/core/app/apps/pipeline/pipeline_generator.py+++ b/api/core/app/apps/pipeline/pipeline_generator.py@@ -7,7 +7,7 @@ import threadingimport timeimport uuidfrom collections.abc import Generator, Mapping-from typing import Any, Literal, cast, overload+from typing import TYPE_CHECKING, Any, Literal, cast, overloadfrom flask import Flask, current_appfrom pydantic import ValidationError@@ -56,11 +56,21 @@ from services.data_source.provider_service import DatasourceProviderServicefrom services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxyfrom services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService+if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+logger = logging.getLogger(__name__)class PipelineGenerator(BaseAppGenerator):- def __init__(self, *, documents: PipelineDocumentStore, datasource_providers: DatasourceProviderService) -> None:+ def __init__(+ self,+ *,+ execution_driver: "WorkflowRunDriver",+ documents: PipelineDocumentStore,+ datasource_providers: DatasourceProviderService,+ ) -> None:+ self._execution_driver = execution_driverself._documents = documentsself._datasource_providers = datasource_providers@@ -631,18 +641,19 @@ class PipelineGenerator(BaseAppGenerator):system_user_id = application_generate_entity.user_id# workflow apprunner = PipelineRunner(+ execution_driver=self._execution_driver,application_generate_entity=application_generate_entity,queue_manager=queue_manager,workflow_thread_pool_id=workflow_thread_pool_id,variable_loader=variable_loader,workflow=workflow,system_user_id=system_user_id,- workflow_execution_repository=workflow_execution_repository,- workflow_node_execution_repository=workflow_node_execution_repository,documents=self._documents,)- runner.run()+ self._execution_driver(+ runner, None, workflow_execution_repository, workflow_node_execution_repository+ )except GenerateTaskStoppedError:passexcept InvokeAuthorizationError:diff --git a/api/core/app/apps/pipeline/pipeline_runner.py b/api/core/app/apps/pipeline/pipeline_runner.pyindex 5a3f6142201..ae0909c1a86 100644--- a/api/core/app/apps/pipeline/pipeline_runner.py+++ b/api/core/app/apps/pipeline/pipeline_runner.py@@ -1,31 +1,29 @@import loggingimport time-from typing import Protocol, cast+from typing import Protocol, cast, overridefrom sqlalchemy import selectfrom sqlalchemy.orm import Sessionfrom core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.apps.pipeline.pipeline_config_manager import PipelineConfig-from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner+from core.app.apps.workflow_app_runner import PreparedWorkflowRun, WorkflowBasedAppRunner, WorkflowRunDriverfrom core.app.entities.app_invoke_entities import (InvokeFrom,RagPipelineGenerateEntity,UserFrom,build_dify_run_context,)-from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayerfrom core.credit_usage import CreditUsageAppTypefrom core.db.session_factory import create_session-from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepositoryfrom core.workflow.node_factory import DifyGraphInitContext, DifyNodeFactory, get_default_root_node_idfrom core.workflow.system_variables import build_bootstrap_variables, build_system_variablesfrom core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_poolfrom core.workflow.workflow_entry import WorkflowEntry+from graphon.engine_events import EngineEvent, GraphRunFailedEventfrom graphon.enums import WorkflowTypefrom graphon.graph import Graph-from graphon.graph_events import GraphEngineEvent, GraphRunFailedEvent-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variable_loader import VariableLoaderfrom graphon.variables.variables import RAGPipelineVariable, RAGPipelineVariableInputfrom models.dataset import Dataset, Pipeline@@ -57,10 +55,9 @@ class PipelineRunner(WorkflowBasedAppRunner):variable_loader: VariableLoader,workflow: Workflow,system_user_id: str,- workflow_execution_repository: WorkflowExecutionRepository,- workflow_node_execution_repository: WorkflowNodeExecutionRepository,documents: PipelineDocumentStore,workflow_thread_pool_id: str | None = None,+ execution_driver: WorkflowRunDriver | None = None,) -> None:""":param application_generate_entity: application generate entity@@ -69,6 +66,7 @@ class PipelineRunner(WorkflowBasedAppRunner):"""super().__init__(queue_manager=queue_manager,+ execution_driver=execution_driver,variable_loader=variable_loader,app_id=application_generate_entity.app_config.app_id,)@@ -76,17 +74,14 @@ class PipelineRunner(WorkflowBasedAppRunner):self.workflow_thread_pool_id = workflow_thread_pool_idself._workflow = workflowself._sys_user_id = system_user_id- self._workflow_execution_repository = workflow_execution_repository- self._workflow_node_execution_repository = workflow_node_execution_repositoryself._documents = documentsdef _get_app_id(self) -> str:return self.application_generate_entity.app_config.app_id- def run(self) -> None:- """- Run application- """+ @override+ def prepare(self) -> PreparedWorkflowRun:+ """Prepare graph execution and passive application observers."""app_config = self.application_generate_entity.app_configapp_config = cast(PipelineConfig, app_config)invoke_from = self.application_generate_entity.invoke_from@@ -199,7 +194,9 @@ class PipelineRunner(WorkflowBasedAppRunner):workflow.graph_dict)add_node_inputs_to_pool(variable_pool, node_id=root_node_id, inputs=inputs)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id=workflow.id, variable_pool=variable_pool, start_at=time.perf_counter()+ )# init graphgraph = self._init_rag_pipeline_graph(@@ -212,46 +209,34 @@ class PipelineRunner(WorkflowBasedAppRunner):# RUN WORKFLOWworkflow_entry = WorkflowEntry(- tenant_id=workflow.tenant_id,- app_id=workflow.app_id,- workflow_id=workflow.id,graph=graph,graph_config=workflow.graph_dict,- user_id=self.application_generate_entity.user_id,- user_from=user_from,- invoke_from=invoke_from,call_depth=self.application_generate_entity.call_depth,graph_runtime_state=graph_runtime_state,- variable_pool=variable_pool,)self._queue_manager.graph_runtime_state = graph_runtime_state- persistence_layer = WorkflowPersistenceLayer(- application_generate_entity=self.application_generate_entity,- workflow_info=PersistenceWorkflowInfo(- workflow_id=workflow.id,- workflow_type=WorkflowType(workflow.type),- version=workflow.version,- graph_data=workflow.graph_dict,- ),- workflow_execution_repository=self._workflow_execution_repository,- workflow_node_execution_repository=self._workflow_node_execution_repository,- trace_manager=self.application_generate_entity.trace_manager,+ self._dataset_workspace_id = dataset_workspace_id+ self._dataset_id = dataset_id+ self._document_id = document_id+ return PreparedWorkflowRun(+ entry=workflow_entry,+ application_layers=self._graph_engine_layers,+ generate_entity=self.application_generate_entity,+ workflow_type=WorkflowType(workflow.type),+ workflow_version=workflow.version,)- workflow_entry.graph_engine.layer(persistence_layer)-- generator = workflow_entry.run()-- for event in generator:- self._update_document_status(- event,- workspace_id=dataset_workspace_id,- dataset_id=dataset_id,- document_id=document_id,- )- self._handle_event(workflow_entry, event)+ @override+ def handle_event(self, workflow_entry: WorkflowEntry, event: EngineEvent, **kwargs) -> None:+ self._update_document_status(+ event,+ workspace_id=self._dataset_workspace_id,+ dataset_id=self._dataset_id,+ document_id=self._document_id,+ )+ super().handle_event(workflow_entry, event, **kwargs)def get_workflow(self, session: Session, pipeline: Pipeline, workflow_id: str) -> Workflow | None:"""@@ -270,7 +255,7 @@ class PipelineRunner(WorkflowBasedAppRunner):def _init_rag_pipeline_graph(self,workflow: Workflow,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,start_node_id: str | None = None,user_from: UserFrom = UserFrom.ACCOUNT,invoke_from: InvokeFrom = InvokeFrom.SERVICE_API,@@ -319,6 +304,7 @@ class PipelineRunner(WorkflowBasedAppRunner):app_type=CreditUsageAppType.RAG_PIPELINE,)graph_init_context = DifyGraphInitContext(+ execution_driver=self._execution_driver,workflow_id=workflow.id,graph_config=graph_config,run_context=run_context,@@ -327,7 +313,7 @@ class PipelineRunner(WorkflowBasedAppRunner):node_factory = DifyNodeFactory.from_graph_init_context(graph_init_context=graph_init_context,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)if start_node_id is None:start_node_id = get_default_root_node_id(graph_config)@@ -340,7 +326,7 @@ class PipelineRunner(WorkflowBasedAppRunner):def _update_document_status(self,- event: GraphEngineEvent,+ event: EngineEvent,*,workspace_id: str,dataset_id: str,diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.pyindex 8d7f7c4ce33..e067f07b3cb 100644--- a/api/core/app/apps/workflow/app_generator.py+++ b/api/core/app/apps/workflow/app_generator.py@@ -31,7 +31,7 @@ from core.app.entities.task_entities import (WorkflowAppPausedBlockingResponse,WorkflowAppStreamResponse,)-from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, PauseStatePersistenceLayer+from core.app.layers.pause_state_persist_layer import PauseStateLayerConfigfrom core.db.session_factory import session_factoryfrom core.helper.trace_id_helper import (extract_external_trace_id_from_args,@@ -45,10 +45,10 @@ from core.trigger.constants import is_trigger_node_typefrom core.workflow.node_factory import get_default_root_node_idfrom extensions.ext_database import dbfrom factories import file_factory-from graphon.filters import ResponseStreamFilter-from graphon.graph_engine.layers import GraphEngineLayer+from graphon.engine.filter import ResponseStreamFilter+from graphon.engine.layer import Layerfrom graphon.model_runtime.errors.invoke import InvokeAuthorizationError-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoaderfrom libs.flask_utils import preserve_flask_contextsfrom models.account import Account@@ -59,6 +59,7 @@ from services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraif TYPE_CHECKING:from controllers.console.app.workflow import LoopNodeRunPayload+ from core.app.apps.workflow_app_runner import WorkflowRunDriverlogger = logging.getLogger(__name__)@@ -70,6 +71,9 @@ def _extract_trace_session_id_from_debug_args(args: Mapping[str, Any] | Any) ->class WorkflowAppGenerator(BaseAppGenerator):+ def __init__(self, *, execution_driver: WorkflowRunDriver) -> None:+ self._execution_driver = execution_driver+@staticmethoddef _ensure_snippet_start_node_in_worker(*, session: Session, workflow: Workflow) -> Workflow:"""Re-apply snippet virtual Start injection after worker reloads workflow from DB."""@@ -103,7 +107,7 @@ class WorkflowAppGenerator(BaseAppGenerator):workflow_run_id: str | uuid.UUID | None = None,triggered_from: WorkflowRunTriggeredFrom | None = None,root_node_id: str | None = None,- graph_engine_layers: Sequence[GraphEngineLayer] = (),+ graph_engine_layers: Sequence[Layer] = (),pause_state_config: PauseStateLayerConfig | None = None,) -> Generator[Mapping[str, Any] | str]: ...@@ -121,7 +125,7 @@ class WorkflowAppGenerator(BaseAppGenerator):workflow_run_id: str | uuid.UUID | None = None,triggered_from: WorkflowRunTriggeredFrom | None = None,root_node_id: str | None = None,- graph_engine_layers: Sequence[GraphEngineLayer] = (),+ graph_engine_layers: Sequence[Layer] = (),pause_state_config: PauseStateLayerConfig | None = None,) -> Mapping[str, Any]: ...@@ -139,7 +143,7 @@ class WorkflowAppGenerator(BaseAppGenerator):workflow_run_id: str | uuid.UUID | None = None,triggered_from: WorkflowRunTriggeredFrom | None = None,root_node_id: str | None = None,- graph_engine_layers: Sequence[GraphEngineLayer] = (),+ graph_engine_layers: Sequence[Layer] = (),pause_state_config: PauseStateLayerConfig | None = None,) -> Mapping[str, Any] | Generator[Mapping[str, Any] | str]: ...@@ -156,7 +160,7 @@ class WorkflowAppGenerator(BaseAppGenerator):workflow_run_id: str | uuid.UUID | None = None,triggered_from: WorkflowRunTriggeredFrom | None = None,root_node_id: str | None = None,- graph_engine_layers: Sequence[GraphEngineLayer] = (),+ graph_engine_layers: Sequence[Layer] = (),pause_state_config: PauseStateLayerConfig | None = None,) -> Mapping[str, Any] | Generator[Mapping[str, Any] | str]:with self._bind_file_access_scope(tenant_id=app_model.tenant_id, user=user, invoke_from=invoke_from):@@ -276,10 +280,10 @@ class WorkflowAppGenerator(BaseAppGenerator):workflow: Workflow,user: Account | EndUser,application_generate_entity: WorkflowAppGenerateEntity,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,workflow_execution_repository: WorkflowExecutionRepository,workflow_node_execution_repository: WorkflowNodeExecutionRepository,- graph_engine_layers: Sequence[GraphEngineLayer] = (),+ graph_engine_layers: Sequence[Layer] = (),pause_state_config: PauseStateLayerConfig | None = None,variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER,response_stream_filter: ResponseStreamFilter | None = None,@@ -329,8 +333,8 @@ class WorkflowAppGenerator(BaseAppGenerator):streaming: bool = True,variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER,root_node_id: str | None = None,- graph_engine_layers: Sequence[GraphEngineLayer] = (),- graph_runtime_state: GraphRuntimeState | None = None,+ graph_engine_layers: Sequence[Layer] = (),+ graph_runtime_state: RuntimeState | None = None,pause_state_config: PauseStateLayerConfig | None = None,response_stream_filter: ResponseStreamFilter | None = None,) -> Mapping[str, Any] | Generator[str | Mapping[str, Any]]:@@ -351,8 +355,6 @@ class WorkflowAppGenerator(BaseAppGenerator):user=user,invoke_from=invoke_from,):- graph_layers: list[GraphEngineLayer] = list(graph_engine_layers)-# init queue managerqueue_manager = WorkflowAppQueueManager(task_id=application_generate_entity.task_id,@@ -361,17 +363,6 @@ class WorkflowAppGenerator(BaseAppGenerator):app_mode=app_model.mode,)- resolved_response_stream_filter = response_stream_filter or ResponseStreamFilter()- if pause_state_config is not None:- graph_layers.append(- PauseStatePersistenceLayer(- session_factory=pause_state_config.session_factory,- generate_entity=application_generate_entity,- state_owner_user_id=pause_state_config.state_owner_user_id,- response_stream_filter=resolved_response_stream_filter,- )- )-# new thread with request context and contextvarscontext = contextvars.copy_context()@@ -389,9 +380,10 @@ class WorkflowAppGenerator(BaseAppGenerator):"root_node_id": root_node_id,"workflow_execution_repository": workflow_execution_repository,"workflow_node_execution_repository": workflow_node_execution_repository,- "graph_engine_layers": tuple(graph_layers),+ "graph_engine_layers": graph_engine_layers,+ "pause_state_config": pause_state_config,"graph_runtime_state": graph_runtime_state,- "response_stream_filter": resolved_response_stream_filter,+ "response_stream_filter": response_stream_filter,},)@@ -621,9 +613,10 @@ class WorkflowAppGenerator(BaseAppGenerator):workflow_execution_repository: WorkflowExecutionRepository,workflow_node_execution_repository: WorkflowNodeExecutionRepository,root_node_id: str | None = None,- graph_engine_layers: Sequence[GraphEngineLayer] = (),- graph_runtime_state: GraphRuntimeState | None = None,+ graph_engine_layers: Sequence[Layer] = (),+ graph_runtime_state: RuntimeState | None = None,response_stream_filter: ResponseStreamFilter | None = None,+ pause_state_config: PauseStateLayerConfig | None = None,) -> None:"""Generate worker in a new thread.@@ -669,12 +662,11 @@ class WorkflowAppGenerator(BaseAppGenerator):runner = WorkflowAppRunner(application_generate_entity=application_generate_entity,+ execution_driver=self._execution_driver,queue_manager=queue_manager,variable_loader=variable_loader,workflow=workflow,system_user_id=system_user_id,- workflow_execution_repository=workflow_execution_repository,- workflow_node_execution_repository=workflow_node_execution_repository,root_node_id=root_node_id,graph_engine_layers=graph_engine_layers,graph_runtime_state=graph_runtime_state,@@ -683,7 +675,9 @@ class WorkflowAppGenerator(BaseAppGenerator):try:with active_workflow_task(application_generate_entity.task_id):- runner.run()+ self._execution_driver(+ runner, pause_state_config, workflow_execution_repository, workflow_node_execution_repository+ )except GenerateTaskStoppedError:logger.warning("Task stopped", exc_info=True)passdiff --git a/api/core/app/apps/workflow/app_runner.py b/api/core/app/apps/workflow/app_runner.pyindex df545d0e345..5752e14b987 100644--- a/api/core/app/apps/workflow/app_runner.py+++ b/api/core/app/apps/workflow/app_runner.py@@ -1,7 +1,7 @@import loggingimport timefrom collections.abc import Sequence-from typing import cast+from typing import cast, overridefrom core.app.apps.base_app_queue_manager import AppQueueManagerfrom core.app.apps.execution_coordinator import app_task_command_channel_key@@ -12,29 +12,24 @@ from core.app.apps.workflow.command_channels import (StopFlagCommandChannel,)from core.app.apps.workflow.stop_aware_ready_queue import attach_stop_aware_ready_queue-from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner+from core.app.apps.workflow_app_runner import PreparedWorkflowRun, WorkflowBasedAppRunner, WorkflowRunDriverfrom core.app.entities.app_invoke_entities import (- DifyRunContext,InvokeFrom,WorkflowAppGenerateEntity,get_credit_usage_app_type,)-from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer-from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepositoryfrom core.workflow.node_factory import get_default_root_node_id-from core.workflow.nodes.agent_v2.workspace_retirement_layer import build_workflow_agent_workspace_retirement_layerfrom core.workflow.snippet_start import get_compatible_start_aliasesfrom core.workflow.system_variables import build_bootstrap_variables, build_system_variablesfrom core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_poolfrom core.workflow.workflow_entry import WorkflowEntryfrom extensions.ext_redis import redis_client-from extensions.otel import WorkflowAppRunnerHandler, trace_spanfrom extensions.workflow_warm_shutdown import WORKFLOW_WARM_SHUTDOWN_ABORT_REASON, celery_warm_shutdown_started+from graphon.engine.command import RedisChannel+from graphon.engine.filter import ResponseStreamFilter+from graphon.engine.layer import Layerfrom graphon.enums import WorkflowType-from graphon.filters import ResponseStreamFilter-from graphon.graph_engine.command_channels import RedisChannel-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variable_loader import VariableLoaderfrom libs.datetime_utils import naive_utc_nowfrom models.workflow import Workflow@@ -56,14 +51,14 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):workflow: Workflow,system_user_id: str,root_node_id: str | None = None,- workflow_execution_repository: WorkflowExecutionRepository,- workflow_node_execution_repository: WorkflowNodeExecutionRepository,- graph_engine_layers: Sequence[GraphEngineLayer] = (),- graph_runtime_state: GraphRuntimeState | None = None,+ execution_driver: WorkflowRunDriver | None = None,+ graph_engine_layers: Sequence[Layer] = (),+ graph_runtime_state: RuntimeState | None = None,response_stream_filter: ResponseStreamFilter | None = None,):super().__init__(queue_manager=queue_manager,+ execution_driver=execution_driver,variable_loader=variable_loader,app_id=application_generate_entity.app_config.app_id,graph_engine_layers=graph_engine_layers,@@ -72,16 +67,12 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):self._workflow = workflowself._sys_user_id = system_user_idself._root_node_id = root_node_id- self._workflow_execution_repository = workflow_execution_repository- self._workflow_node_execution_repository = workflow_node_execution_repositoryself._resume_graph_runtime_state = graph_runtime_stateself._response_stream_filter = response_stream_filter- @trace_span(WorkflowAppRunnerHandler)- def run(self):- """- Run application- """+ @override+ def prepare(self) -> PreparedWorkflowRun | None:+ """Prepare graph execution and passive application observers."""app_config = self.application_generate_entity.app_configapp_config = cast(WorkflowAppConfig, app_config)invoke_from = self.application_generate_entity.invoke_from@@ -106,6 +97,7 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):root_node_id=self._root_node_id,app_type=get_credit_usage_app_type(app_config.app_mode),trace_session_id=self.application_generate_entity.extras.get("trace_session_id"),+ call_depth=self.application_generate_entity.call_depth,)elif self.application_generate_entity.single_iteration_run or self.application_generate_entity.single_loop_run:graph, variable_pool, graph_runtime_state = self._prepare_single_node_execution(@@ -147,7 +139,9 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):),)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id=self._workflow.id, variable_pool=variable_pool, start_at=time.perf_counter()+ )graph = self._init_graph(graph_config=self._workflow.graph_dict,graph_runtime_state=graph_runtime_state,@@ -159,6 +153,7 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):root_node_id=root_node_id,app_type=get_credit_usage_app_type(app_config.app_mode),trace_session_id=self.application_generate_entity.extras.get("trace_session_id"),+ call_depth=self.application_generate_entity.call_depth,)# RUN WORKFLOW@@ -181,52 +176,18 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):self._queue_manager.graph_runtime_state = graph_runtime_stateworkflow_entry = WorkflowEntry(- tenant_id=self._workflow.tenant_id,- app_id=self._workflow.app_id,- workflow_id=self._workflow.id,graph=graph,graph_config=self._workflow.graph_dict,- user_id=self.application_generate_entity.user_id,- user_from=user_from,- invoke_from=invoke_from,call_depth=self.application_generate_entity.call_depth,- variable_pool=variable_pool,graph_runtime_state=graph_runtime_state,command_channel=command_channel,response_stream_filter=self._response_stream_filter,)- persistence_layer = WorkflowPersistenceLayer(- application_generate_entity=self.application_generate_entity,- workflow_info=PersistenceWorkflowInfo(- workflow_id=self._workflow.id,- workflow_type=WorkflowType(self._workflow.type),- version=self._workflow.version,- graph_data=self._workflow.graph_dict,- ),- workflow_execution_repository=self._workflow_execution_repository,- workflow_node_execution_repository=self._workflow_node_execution_repository,- trace_manager=self.application_generate_entity.trace_manager,+ return PreparedWorkflowRun(+ entry=workflow_entry,+ application_layers=self._graph_engine_layers,+ generate_entity=self.application_generate_entity,+ workflow_type=WorkflowType(self._workflow.type),+ workflow_version=self._workflow.version,)-- workflow_entry.graph_engine.layer(persistence_layer)- workflow_entry.graph_engine.layer(- build_workflow_agent_workspace_retirement_layer(- dify_run_context=DifyRunContext(- tenant_id=self._workflow.tenant_id,- app_id=self._workflow.app_id,- user_id=self.application_generate_entity.user_id,- user_from=user_from,- invoke_from=invoke_from,- app_type=get_credit_usage_app_type(app_config.app_mode),- trace_session_id=self.application_generate_entity.extras.get("trace_session_id"),- )- )- )- for layer in self._graph_engine_layers:- workflow_entry.graph_engine.layer(layer)-- generator = workflow_entry.run()-- for event in generator:- self._handle_event(workflow_entry, event)diff --git a/api/core/app/apps/workflow/command_channels.py b/api/core/app/apps/workflow/command_channels.pyindex 6dab7bebbf4..234a5e4307a 100644--- a/api/core/app/apps/workflow/command_channels.py+++ b/api/core/app/apps/workflow/command_channels.py@@ -5,8 +5,7 @@ from collections.abc import Callable, Sequencefrom typing import final, overridefrom core.app.apps.execution_coordinator import is_app_task_stop_flag_set-from graphon.graph_engine.command_channels import CommandChannel-from graphon.graph_engine.entities.commands import AbortCommand, GraphEngineCommand+from graphon.engine.command import AbortCommand, Command, CommandChannellogger = logging.getLogger(__name__)ShutdownStateGetter = Callable[[], bool]@@ -23,23 +22,23 @@ class CombinedCommandChannel:raise ValueError("command_channels must not be empty")self._command_channels = tuple(command_channels)- def fetch_commands(self) -> list[GraphEngineCommand]:- commands: list[GraphEngineCommand] = []+ def fetch_commands(self) -> list[Command]:+ commands: list[Command] = []for channel in self._command_channels:try:commands.extend(channel.fetch_commands())except Exception:- logger.exception("Failed to fetch GraphEngine commands from %s", channel.__class__.__name__)+ logger.exception("Failed to fetch Engine commands from %s", channel.__class__.__name__)return commands- def send_command(self, command: GraphEngineCommand) -> None:+ def send_command(self, command: Command) -> None:"""Send commands through the first channel, which is the runner's primary command sink."""self._command_channels[0].send_command(command)@finalclass CelerySignalCommandChannel(CommandChannel):- """Translate process-local Celery shutdown state into one GraphEngine abort command."""+ """Translate process-local Celery shutdown state into one Engine abort command."""_shutdown_state_getter: ShutdownStateGetter_abort_reason: str@@ -56,7 +55,7 @@ class CelerySignalCommandChannel(CommandChannel):self._abort_emitted = False@override- def fetch_commands(self) -> list[GraphEngineCommand]:+ def fetch_commands(self) -> list[Command]:if self._abort_emitted or not self._shutdown_state_getter():return []@@ -64,7 +63,7 @@ class CelerySignalCommandChannel(CommandChannel):return [AbortCommand(reason=self._abort_reason)]@override- def send_command(self, command: GraphEngineCommand) -> None:+ def send_command(self, command: Command) -> None:_ = command@@ -87,7 +86,7 @@ class StopFlagCommandChannel(CommandChannel):self._abort_emitted = False@override- def fetch_commands(self) -> list[GraphEngineCommand]:+ def fetch_commands(self) -> list[Command]:if self._abort_emitted or not is_app_task_stop_flag_set(self._task_id):return []@@ -95,5 +94,5 @@ class StopFlagCommandChannel(CommandChannel):return [AbortCommand(reason=self._abort_reason)]@override- def send_command(self, command: GraphEngineCommand) -> None:+ def send_command(self, command: Command) -> None:_ = commanddiff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.pyindex b0065259d9e..1ea0fb11277 100644--- a/api/core/app/apps/workflow/generate_task_pipeline.py+++ b/api/core/app/apps/workflow/generate_task_pipeline.py@@ -62,7 +62,7 @@ from core.workflow.system_variables import build_system_variablesfrom extensions.ext_database import dbfrom graphon.entities import WorkflowStartReasonfrom graphon.enums import WorkflowExecutionStatus-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom models import Accountfrom models.enums import CreatorUserRolefrom models.model import EndUser@@ -118,7 +118,7 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):user=user,system_variables=self._workflow_system_variables,)- self._graph_runtime_state: GraphRuntimeState | None = self._base_task_pipeline.queue_manager.graph_runtime_state+ self._graph_runtime_state: RuntimeState | None = self._base_task_pipeline.queue_manager.graph_runtime_statedef process(self,@@ -340,6 +340,7 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):workflow_run_id=run_id,workflow_id=self._workflow.id,reason=event.reason,+ node_execution_snapshots=event.node_execution_snapshots,)yield start_respdiff --git a/api/core/app/apps/workflow/stop_aware_ready_queue.py b/api/core/app/apps/workflow/stop_aware_ready_queue.pyindex dadc402da1e..ef5edb193ac 100644--- a/api/core/app/apps/workflow/stop_aware_ready_queue.py+++ b/api/core/app/apps/workflow/stop_aware_ready_queue.py@@ -5,15 +5,17 @@ from __future__ import annotationsfrom typing import finalfrom core.app.apps.execution_coordinator import is_app_task_stop_flag_set-from graphon.graph_engine.ready_queue import ReadyQueue, ReadyTask-from graphon.runtime.graph_runtime_state import GraphExecutionProtocol, GraphRuntimeState+from graphon.engine.ready_queue import ReadyTask+from graphon.runtime import RuntimeState+from graphon.runtime.execution import GraphExecution+from graphon.runtime.ready_queue import ReadyQueue@finalclass StopAwareReadyQueue:"""Reject newly ready nodes once the run has been stopped.- GraphEngine drain still enqueues successors after abort. Drop those puts so+ Graph execution still enqueues successors after abort. Drop those puts solater nodes do not start; the in-flight node can finish."""@@ -22,7 +24,7 @@ class StopAwareReadyQueue:inner: ReadyQueue,*,task_id: str,- graph_execution: GraphExecutionProtocol,+ graph_execution: GraphExecution,) -> None:self._inner = innerself._task_id = task_id@@ -45,8 +47,8 @@ class StopAwareReadyQueue:def qsize(self) -> int:return self._inner.qsize()- def drain(self) -> list[ReadyTask]:- return self._inner.drain()+ def take_all(self) -> list[ReadyTask]:+ return self._inner.take_all()def dumps(self) -> str:return self._inner.dumps()@@ -55,7 +57,7 @@ class StopAwareReadyQueue:self._inner.loads(data)-def attach_stop_aware_ready_queue(graph_runtime_state: GraphRuntimeState, *, task_id: str) -> None:+def attach_stop_aware_ready_queue(graph_runtime_state: RuntimeState, *, task_id: str) -> None:"""Install stop-aware enqueue policy on an existing runtime state."""current = graph_runtime_state.ready_queueif isinstance(current, StopAwareReadyQueue):diff --git a/api/core/app/apps/workflow_app_runner.py b/api/core/app/apps/workflow_app_runner.pyindex 6d10ab721f0..70ad6483ad0 100644--- a/api/core/app/apps/workflow_app_runner.py+++ b/api/core/app/apps/workflow_app_runner.py@@ -1,15 +1,23 @@import loggingimport time-from collections.abc import Mapping, Sequence+from collections.abc import Callable, Mapping, Sequence+from dataclasses import dataclassfrom typing import Any, castfrom pydantic import BaseModel, ConfigDict, ValidationErrorfrom core.app.apps.base_app_queue_manager import AppQueueManager, PublishFromfrom core.app.entities.agent_strategy import AgentStrategyInfo-from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_context+from core.app.entities.app_invoke_entities import (+ AdvancedChatAppGenerateEntity,+ InvokeFrom,+ UserFrom,+ WorkflowAppGenerateEntity,+ build_dify_run_context,+)from core.app.entities.queue_entities import (AppQueueEvent,+ NodeExecutionSnapshot,QueueAgentLogEvent,QueueHumanInputFormFilledEvent,QueueHumanInputFormTimeoutEvent,@@ -34,9 +42,11 @@ from core.app.entities.queue_entities import (QueueWorkflowStartedEvent,QueueWorkflowSucceededEvent,)+from core.app.layers.pause_state_persist_layer import PauseStateLayerConfigfrom core.credit_usage import CreditUsageAppTypefrom core.rag.entities import RetrievalSourceMetadata-from core.repositories.human_input_repository import HumanInputFormSubmissionRepository+from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository+from core.repositories.human_input_repository import HumanInputFormRecordfrom core.workflow.node_factory import (DifyGraphInitContext,DifyNodeFactory,@@ -44,8 +54,9 @@ from core.workflow.node_factory import (resolve_workflow_node_class,)from core.workflow.nodes.agent.events import NodeRunAgentLogEvent-from core.workflow.nodes.human_input.boundary import enrich_graph_pause_reasons-from core.workflow.nodes.human_input.pause_reason import HumanInputRequired+from core.workflow.nodes.human_input.callback import DifyHITLCallback+from core.workflow.nodes.human_input.enums import HumanInputFormStatus+from core.workflow.nodes.human_input.pause_reason import HumanInputRequired, PauseReasonfrom core.workflow.system_variables import (build_bootstrap_variables,default_system_variables,@@ -56,20 +67,16 @@ from core.workflow.system_variables import (from core.workflow.variable_pool_initializer import add_variables_to_poolfrom core.workflow.workflow_entry import WorkflowEntryfrom core.workflow.workflow_run_outputs import project_node_outputs_for_workflow_run-from graphon.entities.base_node_data import BaseNodeData-from graphon.entities.graph_config import NodeConfigDict-from graphon.entities.pause_reason import HitlRequired-from graphon.enums import BuiltinNodeTypes-from graphon.graph import Graph-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import (- GraphEngineEvent,+from graphon.engine.layer import Layer+from graphon.engine_events import (+ EngineEvent,GraphRunAbortedEvent,GraphRunFailedEvent,GraphRunPartialSucceededEvent,GraphRunPausedEvent,GraphRunStartedEvent,GraphRunSucceededEvent,+ NodeEvent,NodeRunExceptionEvent,NodeRunFailedEvent,NodeRunHumanInputFormFilledEvent,@@ -89,9 +96,17 @@ from graphon.graph_events import (NodeRunStreamChunkEvent,NodeRunSucceededEvent,)-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.entities.base_node_data import BaseNodeData+from graphon.entities.graph_config import NodeConfigDict+from graphon.entities.pause_reason import HitlRequired+from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKey, WorkflowType+from graphon.graph import Graph+from graphon.graph.scoping import resolve_container_id+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader, load_into_variable_pool+from graphon.variables.factory import build_segmentfrom models.workflow import Workflow+from services.workflow_node_variables import load_additional_node_variablesfrom tasks.mail_human_input_delivery_task import dispatch_human_input_email_tasklogger = logging.getLogger(__name__)@@ -104,15 +119,6 @@ class _WorkflowGraphNodeData(BaseNodeData):iteration_id: str | None = Noneloop_id: str | None = None- def parent_id_for(self, key: str) -> str | None:- match key:- case "iteration_id":- return self.iteration_id- case "loop_id":- return self.loop_id- case _:- return None-class _WorkflowGraphNodeConfig(BaseModel):"""Top-level node wrapper used for runner graph filtering."""@@ -158,19 +164,55 @@ class _WorkflowGraphConfig(BaseModel):return self.model_dump(mode="python", exclude_unset=True)-class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor names differ from private state.+def _merge_container_metadata(+ lifecycle_metadata: Mapping[str, Any], node_metadata: Mapping[str, Any]+) -> dict[str, Any]:+ # Keep lifecycle details and ancestor ownership together, preserving the+ # existing node-result precedence when both sources contain the same key.+ return {**lifecycle_metadata, **node_metadata}+++@dataclass(frozen=True, slots=True)+class PreparedWorkflowRun:+ """Prepared engine and workflow facts handed to the persistence-owning orchestrator."""++ entry: WorkflowEntry+ workflow_type: WorkflowType+ workflow_version: str+ generate_entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity+ application_layers: Sequence[Layer]+ """Layers that also receive terminal failures raised after the engine closes."""+++type WorkflowRunDriver = Callable[+ [+ WorkflowBasedAppRunner,+ PauseStateLayerConfig | None,+ WorkflowExecutionRepository,+ WorkflowNodeExecutionRepository,+ ],+ None,+]+++class WorkflowBasedAppRunner:def __init__(self,*,queue_manager: AppQueueManager,variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER,app_id: str,- graph_engine_layers: Sequence[GraphEngineLayer] = (),+ graph_engine_layers: Sequence[Layer] = (),+ execution_driver: WorkflowRunDriver | None = None,):self._queue_manager = queue_managerself._variable_loader = variable_loaderself._app_id = app_idself._graph_engine_layers = graph_engine_layers+ self._execution_driver = execution_driver++ def prepare(self) -> PreparedWorkflowRun | None:+ raise NotImplementedError@staticmethoddef _resolve_user_from(invoke_from: InvokeFrom) -> UserFrom:@@ -181,9 +223,10 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesdef _init_graph(self,graph_config: Mapping[str, Any],- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,user_from: UserFrom,invoke_from: InvokeFrom,+ call_depth: int,workflow_id: str = "",tenant_id: str = "",user_id: str = "",@@ -207,17 +250,18 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namestrace_session_id=trace_session_id,)graph_init_context = DifyGraphInitContext(+ execution_driver=self._execution_driver,workflow_id=workflow_id,graph_config=graph_config,run_context=run_context,- call_depth=0,+ call_depth=call_depth,)# Use the provided graph_runtime_state for consistent state managementnode_factory = DifyNodeFactory.from_graph_init_context(graph_init_context=graph_init_context,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)if root_node_id is None:@@ -240,7 +284,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesuser_id: str,app_type: CreditUsageAppType | None = None,trace_session_id: str | None = None,- ) -> tuple[Graph, VariablePool, GraphRuntimeState]:+ ) -> tuple[Graph, VariablePool, RuntimeState]:"""Prepare graph, variable pool, and runtime state for single node execution(either single iteration or single loop).@@ -265,7 +309,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesenvironment_variables=workflow.environment_variables,),)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.time())+ graph_runtime_state = RuntimeState(workflow_id=workflow.id, variable_pool=variable_pool, start_at=time.time())# Determine which type of single node execution and get graph/variable_poolif single_iteration_run:@@ -274,7 +318,6 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_id=single_iteration_run.node_id,user_inputs=dict(single_iteration_run.inputs),graph_runtime_state=graph_runtime_state,- node_type_filter_key="iteration_id",node_type_label="iteration",user_id=user_id,app_type=app_type,@@ -286,7 +329,6 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_id=single_loop_run.node_id,user_inputs=dict(single_loop_run.inputs),graph_runtime_state=graph_runtime_state,- node_type_filter_key="loop_id",node_type_label="loop",user_id=user_id,app_type=app_type,@@ -296,7 +338,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesraise ValueError("Neither single_iteration_run nor single_loop_run is specified")# Return the graph, variable_pool, and the same graph_runtime_state used during graph creation- # This ensures all nodes in the graph reference the same GraphRuntimeState instance+ # This ensures all nodes in the graph reference the same RuntimeState instancereturn graph, variable_pool, graph_runtime_statedef _get_graph_and_variable_pool_for_single_node_run(@@ -304,8 +346,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesworkflow: Workflow,node_id: str,user_inputs: dict[str, Any],- graph_runtime_state: GraphRuntimeState,- node_type_filter_key: str, # 'iteration_id' or 'loop_id'+ graph_runtime_state: RuntimeState,node_type_label: str = "node", # 'iteration' or 'loop' for error messages*,user_id: str = "",@@ -322,7 +363,6 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_id: The node ID to executeuser_inputs: User inputs for the nodegraph_runtime_state: The graph runtime state- node_type_filter_key: The key to filter nodes ('iteration_id' or 'loop_id')node_type_label: Label for error messages ('iteration' or 'loop')Returns:@@ -333,18 +373,29 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_user_inputs = dict(user_inputs)- # filter nodes only in the specified node type (iteration or loop)- main_node_config = next((node for node in source_graph_config.nodes if node.id == node_id), None)- start_node_id = main_node_config.data.start_node_id if main_node_config else None- selected_node_configs = [- node- for node in source_graph_config.nodes- if node.id == node_id- or node.data.parent_id_for(node_type_filter_key) == node_id- or (start_node_id and node.id == start_node_id)- ]+ graph_config = source_graph_config.to_graph_config()- node_ids = [node.id for node in selected_node_configs]+ # filter nodes only in the specified node type (iteration or loop)+ all_node_configs = graph_config["nodes"]+ nodes_by_id = {node["id"]: node for node in all_node_configs}+ container_ids = {node["id"]: resolve_container_id(node, nodes_by_id=nodes_by_id) for node in all_node_configs}+ node_ids = {node_id}+ while True:+ child_node_ids = {node["id"] for node in all_node_configs if container_ids[node["id"]] in node_ids}+ child_node_ids -= node_ids+ if not child_node_ids:+ break+ node_ids.update(child_node_ids)+ # Preserve ownership resolved with the complete ancestry before removing it+ # from the debug subtree. Nested legacy IDs can otherwise become ambiguous.+ selected_node_configs = []+ for node in all_node_configs:+ if node["id"] not in node_ids:+ continue+ selected_node = dict(node)+ selected_node["data"] = dict(node["data"])+ selected_node["data"]["container_id"] = container_ids[node["id"]]+ selected_node_configs.append(_WorkflowGraphNodeConfig.model_validate(selected_node))# filter edges only in the specified node typeselected_edge_configs = [@@ -356,7 +407,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_configs = [node.to_graph_node_config() for node in selected_node_configs]edge_configs = [edge.model_dump(mode="python", exclude_unset=True) for edge in selected_edge_configs]graph_config = {- **source_graph_config.to_graph_config(),+ **graph_config,"nodes": node_configs,"edges": edge_configs,}@@ -374,6 +425,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namestrace_session_id=trace_session_id,)graph_init_context = DifyGraphInitContext(+ execution_driver=self._execution_driver,workflow_id=workflow.id,graph_config=graph_config,run_context=run_context,@@ -382,7 +434,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_factory = DifyNodeFactory.from_graph_init_context(graph_init_context=graph_init_context,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)target_node_config = None@@ -420,7 +472,7 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesgraph_config=graph_config, config=target_node_config.to_typed_node_config())except NotImplementedError:- variable_mapping = {}+ variable_mapping = dict[str, Sequence[str]]()variable_mapping = inject_default_system_variable_mappings(node_id=target_node_config.id,node_type=node_type,@@ -434,6 +486,14 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesvariable_mapping=variable_mapping,user_inputs=node_user_inputs,)+ variable_mapping = load_additional_node_variables(+ node_config=target_node_config.to_typed_node_config(),+ graph_config=graph_config,+ variable_mapping=variable_mapping,+ variable_loader=self._variable_loader,+ variable_pool=variable_pool,+ user_inputs=node_user_inputs,+ )WorkflowEntry.mapping_user_inputs_to_variable_pool(variable_mapping=variable_mapping,@@ -444,7 +504,11 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor names# init graph after constructor-time context has been loadedgraph = Graph.init(- graph_config=graph_config, node_factory=node_factory, root_node_id=node_id, skip_validation=True+ graph_config=graph_config,+ node_factory=node_factory,+ root_node_id=node_id,+ container_id=container_ids[node_id],+ skip_validation=True,)if not graph:@@ -464,15 +528,104 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor nameslogger.warning("Invalid agent strategy payload for node %s", event.node_id, exc_info=True)return None- def _handle_event(self, workflow_entry: WorkflowEntry, event: GraphEngineEvent):+ def publish_human_input_results(+ self,+ workflow_entry: WorkflowEntry,+ pending_forms: Mapping[str, HitlRequired],+ forms: Mapping[str, HumanInputFormRecord],+ published_form_ids: set[str],+ ) -> None:+ """Publish materialized completions supplied by run orchestration."""+ for form_id, reason in pending_forms.items():+ form = forms[form_id]+ node_id = reason.node_id+ node_title = reason.node_title or form.definition.node_title or form.node_id+ if form.status == HumanInputFormStatus.TIMEOUT:+ self._publish_event(+ QueueHumanInputFormTimeoutEvent(+ form_id=form_id,+ node_id=node_id,+ node_type=BuiltinNodeTypes.HUMAN_INPUT,+ node_title=node_title,+ expiration_time=form.expiration_time,+ )+ )+ published_form_ids.add(form_id)+ elif form.status == HumanInputFormStatus.SUBMITTED:+ action = next(+ (item for item in form.definition.user_actions if item.id == form.selected_action_id), None+ )+ if action is None:+ raise ValueError(f"Submitted human input form has no matching action: {form_id}")+ submitted_data = {name: build_segment(value) for name, value in (form.submitted_data or {}).items()}+ self._publish_event(+ QueueHumanInputFormFilledEvent(+ form_id=form_id,+ node_id=node_id,+ node_type=BuiltinNodeTypes.HUMAN_INPUT,+ node_title=node_title,+ rendered_content=DifyHITLCallback.render_form_content_with_outputs(+ form.rendered_content,+ submitted_data,+ [item.output_variable_name for item in form.definition.inputs],+ form.definition.inputs,+ ),+ action_id=action.id,+ action_text=action.title,+ submitted_data=submitted_data,+ )+ )+ published_form_ids.add(form_id)++ def handle_event(+ self,+ workflow_entry: WorkflowEntry,+ event: EngineEvent,+ *,+ node_run_index: int = 1,+ node_execution_snapshots: tuple[NodeExecutionSnapshot, ...] = (),+ pause_reasons: Sequence[PauseReason] = (),+ ) -> None:"""Handle event:param workflow_entry: workflow entry:param event: event"""+ node_metadata: dict[str, Any] = (+ {str(key): value for key, value in event.node_run_result.metadata.items()}+ if isinstance(event, NodeEvent)+ else {}+ )+ iteration_id = node_metadata.get(WorkflowNodeExecutionMetadataKey.ITERATION_ID.value)+ loop_id = node_metadata.get(WorkflowNodeExecutionMetadataKey.LOOP_ID.value)match event:+ case NodeRunHumanInputFormFilledEvent():+ self._publish_event(+ QueueHumanInputFormFilledEvent(+ form_id=event.id,+ node_id=event.node_id,+ node_type=event.node_type,+ node_title=event.node_title,+ rendered_content=event.rendered_content,+ action_id=event.action_id,+ action_text=event.action_text,+ submitted_data=event.submitted_data,+ )+ )+ case NodeRunHumanInputFormTimeoutEvent():+ self._publish_event(+ QueueHumanInputFormTimeoutEvent(+ form_id=event.id,+ node_id=event.node_id,+ node_type=event.node_type,+ node_title=event.node_title,+ expiration_time=event.expiration_time,+ )+ )case GraphRunStartedEvent():- self._publish_event(QueueWorkflowStartedEvent(reason=event.reason))+ self._publish_event(+ QueueWorkflowStartedEvent(reason=event.reason, node_execution_snapshots=node_execution_snapshots)+ )case GraphRunSucceededEvent():self._publish_event(QueueWorkflowSucceededEvent(outputs=event.outputs))case GraphRunPartialSucceededEvent():@@ -491,45 +644,17 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor names))case GraphRunPausedEvent():- runtime_state = workflow_entry.graph_engine.graph_runtime_statepaused_nodes = list(- dict.fromkeys(reason.node_id for reason in event.reasons if isinstance(reason, HitlRequired))- )- enriched_reasons = enrich_graph_pause_reasons(- reasons=event.reasons,- form_repository=HumanInputFormSubmissionRepository(),- variable_pool=runtime_state.variable_pool,+ dict.fromkeys(reason.node_id for reason in pause_reasons if isinstance(reason, HumanInputRequired)))- self._enqueue_human_input_notifications(enriched_reasons)+ self._enqueue_human_input_notifications(pause_reasons)self._publish_event(QueueWorkflowPausedEvent(- reasons=enriched_reasons,+ reasons=list(pause_reasons),outputs=event.outputs,paused_nodes=paused_nodes,))- case NodeRunHumanInputFormFilledEvent():- self._publish_event(- QueueHumanInputFormFilledEvent(- node_execution_id=event.id,- node_id=event.node_id,- node_type=event.node_type,- node_title=event.node_title,- rendered_content=event.rendered_content,- action_id=event.action_id,- action_text=event.action_text,- submitted_data=event.submitted_data,- )- )- case NodeRunHumanInputFormTimeoutEvent():- self._publish_event(- QueueHumanInputFormTimeoutEvent(- node_id=event.node_id,- node_type=event.node_type,- node_title=event.node_title,- expiration_time=event.expiration_time,- )- )case NodeRunRetryEvent():node_run_result = event.node_run_resultinputs = node_run_result.inputs@@ -547,8 +672,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_title=event.node_title,node_type=event.node_type,start_at=event.start_at,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,inputs=inputs,process_data=process_data,outputs=outputs,@@ -563,12 +688,13 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesself._publish_event(QueueNodeStartedEvent(node_execution_id=event.id,+ node_run_index=node_run_index,node_id=event.node_id,node_title=event.node_title,node_type=event.node_type,start_at=event.start_at,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,agent_strategy=self._build_agent_strategy_info(event),provider_type=event.provider_type,provider_id=event.provider_id,@@ -584,15 +710,15 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesoutputs=node_run_result.outputs,)execution_metadata = node_run_result.metadata- if event.node_type == BuiltinNodeTypes.ANSWER and (event.in_iteration_id or event.in_loop_id):+ if event.node_type == BuiltinNodeTypes.ANSWER and (iteration_id or loop_id):answer = outputs.get("answer")if isinstance(answer, str) and answer:self._publish_event(QueueTextChunkEvent(text=answer,from_variable_selector=[event.node_id, "answer"],- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,))self._publish_event(@@ -606,8 +732,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesprocess_data=process_data,outputs=outputs,execution_metadata=execution_metadata,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,))case NodeRunFailedEvent():@@ -628,8 +754,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesoutputs=outputs,error=event.node_run_result.error or "Unknown error",execution_metadata=event.node_run_result.metadata,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,))case NodeRunExceptionEvent():@@ -650,8 +776,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesoutputs=outputs,error=event.node_run_result.error or "Unknown error",execution_metadata=event.node_run_result.metadata,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,))case NodeRunStreamChunkEvent():@@ -659,8 +785,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesQueueTextChunkEvent(text=event.chunk,from_variable_selector=list(event.selector),- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,))case NodeRunReasoningChunkEvent():@@ -669,8 +795,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesreasoning=event.chunk,from_node_id=event.node_id,is_final=event.is_final,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,))case NodeRunRetrieverResourceEvent():@@ -679,8 +805,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesretriever_resources=[RetrievalSourceMetadata.model_validate(resource) for resource in event.retriever_resources],- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ in_iteration_id=iteration_id,+ in_loop_id=loop_id,))case NodeRunAgentLogEvent():@@ -705,9 +831,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_type=event.node_type,node_title=event.node_title,start_at=event.start_at,- node_run_index=workflow_entry.graph_engine.graph_runtime_state.node_run_steps,inputs=event.inputs,- metadata=event.metadata,+ metadata=_merge_container_metadata(event.metadata, node_metadata),))case NodeRunIterationNextEvent():@@ -718,7 +843,6 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_type=event.node_type,node_title=event.node_title,index=event.index,- node_run_index=workflow_entry.graph_engine.graph_runtime_state.node_run_steps,output=event.pre_iteration_output,))@@ -730,10 +854,9 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_type=event.node_type,node_title=event.node_title,start_at=event.start_at,- node_run_index=workflow_entry.graph_engine.graph_runtime_state.node_run_steps,inputs=event.inputs,outputs=event.outputs,- metadata=event.metadata,+ metadata=_merge_container_metadata(event.metadata, node_metadata),steps=event.steps,error=event.error if isinstance(event, NodeRunIterationFailedEvent) else None,)@@ -746,9 +869,8 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_type=event.node_type,node_title=event.node_title,start_at=event.start_at,- node_run_index=workflow_entry.graph_engine.graph_runtime_state.node_run_steps,inputs=event.inputs,- metadata=event.metadata,+ metadata=_merge_container_metadata(event.metadata, node_metadata),))case NodeRunLoopNextEvent():@@ -759,7 +881,6 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_type=event.node_type,node_title=event.node_title,index=event.index,- node_run_index=workflow_entry.graph_engine.graph_runtime_state.node_run_steps,output=event.pre_loop_output,))@@ -771,10 +892,9 @@ class WorkflowBasedAppRunner: # noqa: B903 # Runtime owner; constructor namesnode_type=event.node_type,node_title=event.node_title,start_at=event.start_at,- node_run_index=workflow_entry.graph_engine.graph_runtime_state.node_run_steps,inputs=event.inputs,outputs=event.outputs,- metadata=event.metadata,+ metadata=_merge_container_metadata(event.metadata, node_metadata),steps=event.steps,error=event.error if isinstance(event, NodeRunLoopFailedEvent) else None,)diff --git a/api/core/app/entities/queue_entities.py b/api/core/app/entities/queue_entities.pyindex 88a3cf6e550..2ed3a0902a8 100644--- a/api/core/app/entities/queue_entities.py+++ b/api/core/app/entities/queue_entities.py@@ -85,7 +85,6 @@ class QueueIterationStartEvent(AppQueueEvent):node_title: strstart_at: datetime- node_run_index: intinputs: Mapping[str, object] = Field(default_factory=dict)metadata: Mapping[str, object] = Field(default_factory=dict)@@ -102,7 +101,6 @@ class QueueIterationNextEvent(AppQueueEvent):node_id: strnode_type: NodeTypenode_title: str- node_run_index: intoutput: Any = None # output for the current iteration@@ -119,7 +117,6 @@ class QueueIterationCompletedEvent(AppQueueEvent):node_title: strstart_at: datetime- node_run_index: intinputs: Mapping[str, object] = Field(default_factory=dict)outputs: Mapping[str, object] = Field(default_factory=dict)metadata: Mapping[str, object] = Field(default_factory=dict)@@ -140,7 +137,6 @@ class QueueLoopStartEvent(AppQueueEvent):node_title: strstart_at: datetime- node_run_index: intinputs: Mapping[str, object] = Field(default_factory=dict)metadata: Mapping[str, object] = Field(default_factory=dict)@@ -157,7 +153,6 @@ class QueueLoopNextEvent(AppQueueEvent):node_id: strnode_type: NodeTypenode_title: str- node_run_index: intoutput: Any = None # output for the current loop@@ -174,7 +169,6 @@ class QueueLoopCompletedEvent(AppQueueEvent):node_title: strstart_at: datetime- node_run_index: intinputs: Mapping[str, object] = Field(default_factory=dict)outputs: Mapping[str, object] = Field(default_factory=dict)metadata: Mapping[str, object] = Field(default_factory=dict)@@ -283,12 +277,24 @@ class QueueAdvancedChatMessageEndEvent(AppQueueEvent):event: QueueEvent = QueueEvent.ADVANCED_CHAT_MESSAGE_END+class NodeExecutionSnapshot(BaseModel):+ """Materialized node metadata supplied by the run owner when resuming streaming."""++ execution_id: str+ title: str+ index: int+ start_at: datetime+ iteration_id: str = ""+ loop_id: str = ""++class QueueWorkflowStartedEvent(AppQueueEvent):"""QueueWorkflowStartedEvent entity."""event: QueueEvent = QueueEvent.WORKFLOW_STARTED# Always present; mirrors GraphRunStartedEvent.reason for downstream consumers.reason: WorkflowStartReason = WorkflowStartReason.INITIAL+ node_execution_snapshots: tuple[NodeExecutionSnapshot, ...] = ()class QueueWorkflowSucceededEvent(AppQueueEvent):@@ -331,7 +337,7 @@ class QueueNodeStartedEvent(AppQueueEvent):node_id: strnode_title: strnode_type: NodeType- node_run_index: int = 1 # FIXME(-LAN-): may not used+ node_run_index: int = 1in_iteration_id: str | None = Nonein_loop_id: str | None = Nonestart_at: datetime@@ -526,7 +532,7 @@ class QueueHumanInputFormFilledEvent(AppQueueEvent):event: QueueEvent = QueueEvent.HUMAN_INPUT_FORM_FILLED- node_execution_id: str+ form_id: strnode_id: strnode_type: NodeTypenode_title: str@@ -546,6 +552,7 @@ class QueueHumanInputFormTimeoutEvent(AppQueueEvent):event: QueueEvent = QueueEvent.HUMAN_INPUT_FORM_TIMEOUT+ form_id: strnode_id: strnode_type: NodeTypenode_title: strdiff --git a/api/core/app/entities/task_entities.py b/api/core/app/entities/task_entities.pyindex 3ffea921433..b3cd0b3ec3d 100644--- a/api/core/app/entities/task_entities.py+++ b/api/core/app/entities/task_entities.py@@ -344,6 +344,7 @@ class HumanInputFormFilledResponse(StreamResponse):Data entity"""+ form_id: strnode_id: strnode_title: strrendered_content: str@@ -363,6 +364,7 @@ class HumanInputFormTimeoutResponse(StreamResponse):Data entity"""+ form_id: strnode_id: strnode_title: strexpiration_time: intdiff --git a/api/core/app/layers/conversation_variable_persist_layer.py b/api/core/app/layers/conversation_variable_persist_layer.pyindex 9125236af61..d9048d57cbe 100644--- a/api/core/app/layers/conversation_variable_persist_layer.py+++ b/api/core/app/layers/conversation_variable_persist_layer.py@@ -12,24 +12,20 @@ from typing import overridefrom core.workflow.system_variables import SystemVariableKey, get_system_textfrom core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import GraphEngineEvent, NodeRunVariableUpdatedEvent+from graphon.engine.layer import Layer+from graphon.engine_events import EngineEvent, NodeRunVariableUpdatedEventfrom services.conversation_variable_updater import ConversationVariableUpdaterlogger = logging.getLogger(__name__)-class ConversationVariablePersistenceLayer(GraphEngineLayer):+class ConversationVariablePersistenceLayer(Layer):def __init__(self, conversation_variable_updater: ConversationVariableUpdater) -> None:super().__init__()self._conversation_variable_updater = conversation_variable_updater@override- def on_graph_start(self) -> None:- pass-- @override- def on_event(self, event: GraphEngineEvent) -> None:+ def on_event(self, event: EngineEvent) -> None:if not isinstance(event, NodeRunVariableUpdatedEvent):return@@ -38,7 +34,7 @@ class ConversationVariablePersistenceLayer(GraphEngineLayer):logger.warning("Conversation variable selector invalid. selector=%s", selector)return- conversation_id = get_system_text(self.graph_runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)+ conversation_id = get_system_text(self.runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)if conversation_id is None:return@@ -46,7 +42,3 @@ class ConversationVariablePersistenceLayer(GraphEngineLayer):returnself._conversation_variable_updater.update(conversation_id=conversation_id, variable=event.variable)-- @override- def on_graph_end(self, error: Exception | None) -> None:- passdiff --git a/api/core/app/layers/execution_context_layer.py b/api/core/app/layers/execution_context_layer.pynew file mode 100644index 00000000000..9af7d2142d0--- /dev/null+++ b/api/core/app/layers/execution_context_layer.py@@ -0,0 +1,28 @@+from collections.abc import Generator+from contextlib import AbstractContextManager, contextmanager+from typing import override++from context import capture_current_context+from graphon.engine.layer import Layer+from graphon.file.runtime import peek_workflow_file_runtime, use_workflow_file_runtime+from graphon.nodes.base.node import Node+++class ExecutionContextLayer(Layer):+ """Restore the host's Flask and contextvars state for each worker activation."""++ def __init__(self) -> None:+ super().__init__()+ self._context = capture_current_context()++ @contextmanager+ def enter_context(self) -> Generator[None]:+ """Create a fresh manager for a node worker or child-frame construction."""+ file_runtime = peek_workflow_file_runtime()+ # Flask installs its app adapter; node execution uses the engine's adapter.+ with self._context, use_workflow_file_runtime(file_runtime):+ yield++ @override+ def node_run_context(self, node: Node, *, parent_execution_id: str | None = None) -> AbstractContextManager[None]:+ return self.enter_context()diff --git a/api/core/app/layers/pause_state_persist_layer.py b/api/core/app/layers/pause_state_persist_layer.pyindex b65728ab3b0..a228611bc59 100644--- a/api/core/app/layers/pause_state_persist_layer.py+++ b/api/core/app/layers/pause_state_persist_layer.py@@ -1,21 +1,14 @@from dataclasses import dataclass-from typing import Annotated, Literal, Self, override+from typing import Annotated, Literal, Selffrom pydantic import BaseModel, Fieldfrom sqlalchemy import Enginefrom sqlalchemy.orm import Session, sessionmakerfrom core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity-from core.repositories.human_input_repository import HumanInputFormSubmissionRepository-from core.workflow.nodes.human_input.boundary import enrich_graph_pause_reasons-from core.workflow.system_variables import SystemVariableKey, get_system_text-from graphon.filters import ResponseStreamFilter-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import GraphEngineEvent, GraphRunPausedEvent+from graphon.engine.filter import ResponseStreamFilterfrom models.model import AppMode-from repositories.api_workflow_run_repository import APIWorkflowRunRepositoryfrom repositories.entities.workflow_execution import WorkflowExecutionTiming-from repositories.factory import DifyAPIRepositoryFactory# Wrapper types for `WorkflowAppGenerateEntity` and@@ -74,111 +67,7 @@ class WorkflowResumptionContext(BaseModel):@dataclass(frozen=True)class PauseStateLayerConfig:- """Configuration container for instantiating pause persistence layers."""+ """Persistence configuration for resumable workflow runs."""session_factory: Engine | sessionmaker[Session]state_owner_user_id: str---class PauseStatePersistenceLayer(GraphEngineLayer):- def __init__(- self,- session_factory: Engine | sessionmaker[Session],- generate_entity: WorkflowAppGenerateEntity | AdvancedChatAppGenerateEntity,- state_owner_user_id: str,- response_stream_filter: ResponseStreamFilter,- ):- """Create a PauseStatePersistenceLayer.-- The `state_owner_user_id` is used when creating state file for pause.- It generally should id of the creator of workflow.-- `response_stream_filter` must be the exact same instance that- `WorkflowEntry` is using to stream this run's events — this layer- dumps its state on pause, and a different instance would silently- persist the wrong (empty) filter state.- """- if isinstance(session_factory, Engine):- session_factory = sessionmaker(session_factory)- super().__init__()- self._session_maker = session_factory- self._state_owner_user_id = state_owner_user_id- self._generate_entity = generate_entity- self._response_stream_filter = response_stream_filter-- def _get_repo(self) -> APIWorkflowRunRepository:- return DifyAPIRepositoryFactory.create_api_workflow_run_repository(self._session_maker)-- @override- def on_graph_start(self) -> None:- """- Called when graph execution starts.-- This is called after the engine has been initialized but before any nodes- are executed. Layers can use this to set up resources or log start information.- """- pass-- @override- def on_event(self, event: GraphEngineEvent) -> None:- """- Called for every event emitted by the engine.-- This method receives all events generated during graph execution, including:- - Graph lifecycle events (start, success, failure)- - Node execution events (start, success, failure, retry)- - Stream events for response nodes- - Container events (iteration, loop)-- Args:- event: The event emitted by the engine- """- if not isinstance(event, GraphRunPausedEvent):- return-- entity_wrapper: _GenerateEntityUnion- if isinstance(self._generate_entity, WorkflowAppGenerateEntity):- entity_wrapper = _WorkflowGenerateEntityWrapper(entity=self._generate_entity)- else:- entity_wrapper = _AdvancedChatAppGenerateEntityWrapper(entity=self._generate_entity)-- state = WorkflowResumptionContext(- serialized_graph_runtime_state=self.graph_runtime_state.dumps(),- elapsed_time=self._generate_entity.workflow_execution_timing.finish(),- generate_entity=entity_wrapper,- serialized_response_stream_filter_state=self._response_stream_filter.dumps(),- )-- workflow_run_id = get_system_text(- self.graph_runtime_state.variable_pool,- SystemVariableKey.WORKFLOW_EXECUTION_ID,- )- assert workflow_run_id is not None- # NOTE(QuantumGhost): Dify owns the pause-reason semantics that cross the- # persistence boundary. Graphon session ids are translated back to form ids- # here so repository/model layers only handle Dify-owned pause reasons.- pause_reasons = enrich_graph_pause_reasons(- reasons=event.reasons,- form_repository=HumanInputFormSubmissionRepository(),- variable_pool=self.graph_runtime_state.variable_pool,- )- repo = self._get_repo()- repo.create_workflow_pause(- workflow_run_id=workflow_run_id,- state_owner_user_id=self._state_owner_user_id,- state=state.dumps(),- pause_reasons=pause_reasons,- )-- @override- def on_graph_end(self, error: Exception | None) -> None:- """- Called when graph execution ends.-- This is called after all nodes have been executed or when execution is- aborted. Layers can use this to clean up resources or log final state.-- Args:- error: The exception that caused execution to fail, or None if successful- """- passdiff --git a/api/core/app/layers/suspend_layer.py b/api/core/app/layers/suspend_layer.pydeleted file mode 100644index 3e28303a7d3..00000000000--- a/api/core/app/layers/suspend_layer.py+++ /dev/null@@ -1,32 +0,0 @@-from typing import override--from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import GraphEngineEvent, GraphRunPausedEvent---class SuspendLayer(GraphEngineLayer):- """ """-- def __init__(self) -> None:- super().__init__()- self._paused = False-- @override- def on_graph_start(self):- self._paused = False-- @override- def on_event(self, event: GraphEngineEvent):- """- Handle the paused event, stash runtime state into storage and wait for resume.- """- if isinstance(event, GraphRunPausedEvent):- self._paused = True-- @override- def on_graph_end(self, error: Exception | None):- """ """- self._paused = False-- def is_paused(self) -> bool:- return self._pauseddiff --git a/api/core/app/layers/timeslice_layer.py b/api/core/app/layers/timeslice_layer.pyindex 094c21944df..098fd174719 100644--- a/api/core/app/layers/timeslice_layer.py+++ b/api/core/app/layers/timeslice_layer.py@@ -4,16 +4,15 @@ from typing import ClassVar, overridefrom apscheduler.schedulers.background import BackgroundScheduler # type: ignore-from graphon.graph_engine.entities.commands import CommandType, GraphEngineCommand-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import GraphEngineEvent+from graphon.engine.command import PauseCommand+from graphon.engine.layer import Layerfrom services.workflow.entities import WorkflowScheduleCFSPlanEntityfrom services.workflow.scheduler import CFSPlanScheduler, SchedulerCommandlogger = logging.getLogger(__name__)-class TimeSliceLayer(GraphEngineLayer):+class TimeSliceLayer(Layer):"""CFS plan scheduler to control the timeslice of the workflow."""@@ -51,14 +50,7 @@ class TimeSliceLayer(GraphEngineLayer):return# send command to pause the workflow- self.command_channel.send_command(- GraphEngineCommand(- command_type=CommandType.PAUSE,- payload={- "reason": SchedulerCommand.RESOURCE_LIMIT_REACHED,- },- )- )+ self.command_channel.send_command(PauseCommand(reason=SchedulerCommand.RESOURCE_LIMIT_REACHED))except Exception:logger.exception("scheduler error during check if the workflow need to be suspended")@@ -79,10 +71,6 @@ class TimeSliceLayer(GraphEngineLayer):id=self.schedule_id,)- @override- def on_event(self, event: GraphEngineEvent):- pass-@overridedef on_graph_end(self, error: Exception | None) -> None:self.stopped = Truediff --git a/api/core/app/layers/trigger_post_layer.py b/api/core/app/layers/trigger_post_layer.pyindex b30603b860d..95431350807 100644--- a/api/core/app/layers/trigger_post_layer.py+++ b/api/core/app/layers/trigger_post_layer.py@@ -6,9 +6,9 @@ from pydantic import TypeAdapterfrom core.db.session_factory import session_factoryfrom core.workflow.system_variables import SystemVariableKey, get_system_text-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import (- GraphEngineEvent,+from graphon.engine.layer import Layer+from graphon.engine_events import (+ EngineEvent,GraphRunAbortedEvent,GraphRunFailedEvent,GraphRunPausedEvent,@@ -21,12 +21,12 @@ from tasks.workflow_cfs_scheduler.cfs_scheduler import AsyncWorkflowCFSPlanEntitlogger = logging.getLogger(__name__)-class TriggerPostLayer(GraphEngineLayer):+class TriggerPostLayer(Layer):"""Trigger post layer."""- _STATUS_MAP: ClassVar[dict[type[GraphEngineEvent], WorkflowTriggerStatus]] = {+ _STATUS_MAP: ClassVar[dict[type[EngineEvent], WorkflowTriggerStatus]] = {GraphRunSucceededEvent: WorkflowTriggerStatus.SUCCEEDED,GraphRunFailedEvent: WorkflowTriggerStatus.FAILED,GraphRunAbortedEvent: WorkflowTriggerStatus.FAILED,@@ -45,11 +45,7 @@ class TriggerPostLayer(GraphEngineLayer):self.cfs_plan_scheduler_entity = cfs_plan_scheduler_entity@override- def on_graph_start(self):- pass-- @override- def on_event(self, event: GraphEngineEvent):+ def on_event(self, event: EngineEvent):"""Update trigger log with success or failure."""@@ -62,19 +58,20 @@ class TriggerPostLayer(GraphEngineLayer):return# Calculate elapsed time- elapsed_time = (datetime.now(UTC) - self.start_time).total_seconds()+ now = datetime.now(UTC)+ elapsed_time = (now - self.start_time).total_seconds()# Extract relevant data from result- outputs = self.graph_runtime_state.outputs+ outputs = self.runtime_state.outputs# BASICLY, workflow_execution_id is the same as workflow_run_idworkflow_run_id = get_system_text(- self.graph_runtime_state.variable_pool,+ self.runtime_state.variable_pool,SystemVariableKey.WORKFLOW_EXECUTION_ID,)assert workflow_run_id, "Workflow run id is not set"- total_tokens = self.graph_runtime_state.total_tokens+ total_tokens = self.runtime_state.total_tokens# Update trigger log with successtrigger_log.status = self._STATUS_MAP[type(event)]@@ -89,10 +86,7 @@ class TriggerPostLayer(GraphEngineLayer):trigger_log.elapsed_time += elapsed_timetrigger_log.total_tokens = total_tokens- trigger_log.finished_at = datetime.now(UTC)+ trigger_log.finished_at = nowrepo.update(trigger_log)session.commit()-- @override- def on_graph_end(self, error: Exception | None) -> None:- pass+ self.start_time = nowdiff --git a/api/core/app/workflow/file_runtime.py b/api/core/app/workflow/file_runtime.pyindex 7bdabd93717..886ef5e52d0 100644--- a/api/core/app/workflow/file_runtime.py+++ b/api/core/app/workflow/file_runtime.py@@ -18,10 +18,10 @@ from core.workflow.file_reference import parse_file_referencefrom extensions.ext_storage import storagefrom graphon.file import FileTransferMethodfrom graphon.file.protocols import WorkflowFileRuntimeProtocol-from graphon.file.runtime import set_workflow_file_runtimefrom graphon.http.protocols import HttpResponseProtocolif TYPE_CHECKING:+ from dify_app import DifyAppfrom graphon.file import File@@ -215,5 +215,10 @@ class DifyWorkflowFileRuntime(WorkflowFileRuntimeProtocol):raise ValueError(f"Tool file {tool_file_id} not found")-def bind_dify_workflow_file_runtime() -> None:- set_workflow_file_runtime(DifyWorkflowFileRuntime(file_access_controller=DatabaseFileAccessController()))+def create_dify_workflow_file_runtime() -> DifyWorkflowFileRuntime:+ return DifyWorkflowFileRuntime(file_access_controller=DatabaseFileAccessController())+++def init_app(app: DifyApp) -> None:+ """Bind host-side file rendering for each Flask request, task, and CLI context."""+ app.extensions["workflow_file_runtime"] = create_dify_workflow_file_runtime()diff --git a/api/core/app/workflow/layers/__init__.py b/api/core/app/workflow/layers/__init__.pyindex 945f75303c7..8e69b33a940 100644--- a/api/core/app/workflow/layers/__init__.py+++ b/api/core/app/workflow/layers/__init__.py@@ -1,10 +1 @@-"""Workflow-level GraphEngine layers that depend on outer infrastructure."""--from .observability import ObservabilityLayer-from .persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer--__all__ = [- "ObservabilityLayer",- "PersistenceWorkflowInfo",- "WorkflowPersistenceLayer",-]+"""Workflow Engine observability layer."""diff --git a/api/core/app/workflow/layers/observability.py b/api/core/app/workflow/layers/observability.pyindex 317dab91ad3..2c3fff1bae6 100644--- a/api/core/app/workflow/layers/observability.py+++ b/api/core/app/workflow/layers/observability.py@@ -1,5 +1,5 @@"""-Observability layer for GraphEngine.+Observability layer for Engine.This layer creates OpenTelemetry spans for node execution, enabling distributedtracing of workflow execution. It establishes OTel context during node execution@@ -8,8 +8,9 @@ associates with the node span."""import logging+from collections.abc import Generator+from contextlib import contextmanagerfrom contextvars import Token-from dataclasses import dataclassfrom typing import cast, final, overridefrom opentelemetry import context as context_api@@ -25,22 +26,16 @@ from extensions.otel.parser import ()from extensions.otel.runtime import is_instrument_flag_enabledfrom extensions.otel.semconv import DifySpanAttributes+from graphon.engine.layer import Layer+from graphon.engine_events import EngineEvent, GraphRunAbortedEvent, NodeEventfrom graphon.enums import BuiltinNodeTypes, NodeType-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import GraphEngineEvent, GraphNodeEventBase, GraphRunAbortedEventfrom graphon.nodes.base.node import Nodelogger = logging.getLogger(__name__)-@dataclass(slots=True)-class _NodeSpanContext:- span: "Span"- token: Token[context_api.Context]--@final-class ObservabilityLayer(GraphEngineLayer):+class ObservabilityLayer(Layer):"""Layer that creates OpenTelemetry spans for node execution.@@ -48,11 +43,15 @@ class ObservabilityLayer(GraphEngineLayer):- Creates a span when a node starts execution- Establishes OTel context so automatic instrumentation associates with the span- Sets complete attributes and status when node execution ends++ Node spans survive worker handoffs within one engine attempt only. A+ persisted pause ends them; the restored engine creates new spans without+ restoring this layer's previous node-to-span mapping or parent span contexts."""def __init__(self) -> None:super().__init__()- self._node_contexts: dict[str, _NodeSpanContext] = {}+ self._node_spans: dict[str, Span] = {}self._parsers: dict[NodeType, NodeOTelParser] = {}self._default_parser: NodeOTelParser = cast(NodeOTelParser, DefaultNodeOTelParser())self._is_disabled: bool = False@@ -84,47 +83,40 @@ class ObservabilityLayer(GraphEngineLayer):return self._parsers.get(node.node_type, self._default_parser)@override- def on_graph_start(self) -> None:- """Called when graph execution starts."""- self._node_contexts.clear()-- @override- def on_node_run_start(self, node: Node) -> None:- """- Called when a node starts execution.+ @contextmanager+ def node_run_context(self, node: Node, *, parent_execution_id: str | None = None) -> Generator[None]:+ """Activate a retained span only for this worker's execution segment.- Creates a span and establishes OTel context for automatic instrumentation.+ Container spans outlive worker activations. Context tokens must not:+ suspension and final resume can execute in different contexts/threads."""- if self._is_disabled:+ if self._is_disabled or self._tracer is None:+ yieldreturn+ token: Token[context_api.Context] | None = Nonetry:- if not self._tracer:- return-- execution_id = node.execution_id- if not execution_id:- return-- parent_context = context_api.get_current()- span = self._tracer.start_span(- f"{node.title}",- kind=SpanKind.INTERNAL,- context=parent_context,- )-- new_context = set_span_in_context(span)- token = context_api.attach(new_context)-- self._node_contexts[execution_id] = _NodeSpanContext(span=span, token=token)-+ if execution_id := node.execution_id:+ span = self._node_spans.get(execution_id)+ if span is None:+ parent_context = context_api.get_current()+ parent_span = self._node_spans.get(parent_execution_id) if parent_execution_id else None+ if parent_span is not None:+ parent_context = set_span_in_context(parent_span, parent_context)+ span = self._tracer.start_span(node.title, kind=SpanKind.INTERNAL, context=parent_context)+ self._node_spans[execution_id] = span+ token = context_api.attach(set_span_in_context(span))except Exception as e:- logger.warning("Failed to create OpenTelemetry span for node %s: %s", node.id, e)+ logger.warning("Failed to activate OpenTelemetry span for node %s: %s", node.id, e)++ try:+ yield+ finally:+ if token is not None:+ context_api.detach(token)@override- def on_node_run_end(- self, node: Node, error: Exception | None, result_event: GraphNodeEventBase | None = None- ) -> None:+ def on_node_run_end(self, node: Node, error: Exception | None, result_event: NodeEvent | None = None) -> None:"""Called when a node finishes execution.@@ -137,29 +129,21 @@ class ObservabilityLayer(GraphEngineLayer):execution_id = node.execution_idif not execution_id:return- node_context = self._node_contexts.get(execution_id)- if not node_context:+ span = self._node_spans.pop(execution_id, None)+ if span is None:return- span = node_context.spanparser = self._get_parser(node)try:parser.parse(node=node, span=span, error=error, result_event=result_event)- span.end()finally:- token = node_context.token- if token is not None:- try:- context_api.detach(token)- except Exception:- logger.warning("Failed to detach OpenTelemetry token: %s", token)- self._node_contexts.pop(execution_id, None)+ span.end()except Exception as e:logger.warning("Failed to end OpenTelemetry span for node %s: %s", node.id, e)@override- def on_event(self, event: GraphEngineEvent) -> None:+ def on_event(self, event: EngineEvent) -> None:"""Record graph-level observability events."""if self._is_disabled:return@@ -169,13 +153,10 @@ class ObservabilityLayer(GraphEngineLayer):@overridedef on_graph_end(self, error: Exception | None) -> None:- """Called when graph execution ends."""- if self._node_contexts:- logger.warning(- "ObservabilityLayer: %d node spans were not properly ended",- len(self._node_contexts),- )- self._node_contexts.clear()+ """Close suspended spans when this engine attempt pauses or terminates."""+ for execution_id in tuple(self._node_spans):+ if (span := self._node_spans.pop(execution_id, None)) is not None:+ span.end()def _record_abort_reason(self, *, reason: str) -> None:span = get_current_span()diff --git a/api/core/model_manager.py b/api/core/model_manager.pyindex d43c15a382f..3f9121bd853 100644--- a/api/core/model_manager.py+++ b/api/core/model_manager.py@@ -409,15 +409,15 @@ class ModelInstance:request_metadata: Mapping[str, object] | None = None,) -> Iterable[bytes]:"""- Invoke large language tts model+ Invoke a text-to-speech model.:param content_text: text content to be translated:param voice: model timbre- :return: text for given audio file+ :return: audio byte stream"""if not isinstance(self.model_type_instance, TTSModel):raise Exception("Model type instance is not TTSModel")- return self._round_robin_invoke(+ chunks = self._round_robin_invoke(self.model_type_instance.invoke,model=self.model_name,credentials=self.credentials,@@ -425,6 +425,7 @@ class ModelInstance:voice=voice,request_metadata=self._resolve_request_metadata(request_metadata),)+ return (chunk.data for chunk in chunks)def _round_robin_invoke(self, function: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R:"""diff --git a/api/core/plugin/backwards_invocation/app.py b/api/core/plugin/backwards_invocation/app.pyindex 2bcf85f579c..7a12c5a0be6 100644--- a/api/core/plugin/backwards_invocation/app.py+++ b/api/core/plugin/backwards_invocation/app.py@@ -1,6 +1,6 @@import uuidfrom collections.abc import Generator, Mapping-from typing import Any, Protocol, cast+from typing import TYPE_CHECKING, Any, Protocol, castfrom sqlalchemy import selectfrom sqlalchemy.orm import Session@@ -27,6 +27,9 @@ from models.model import ()from models.workflow import Workflow+if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+class AppScopedEndUserProvisioner(Protocol):def get_or_create_end_user(@@ -78,6 +81,7 @@ class PluginAppBackwardsInvocation(BaseBackwardsInvocation):inputs: Mapping,files: list[dict],session: Session,+ execution_driver: "WorkflowRunDriver",end_users: AppScopedEndUserProvisioner,) -> Generator[Mapping | str] | Mapping:"""@@ -100,12 +104,14 @@ class PluginAppBackwardsInvocation(BaseBackwardsInvocation):if not query:raise ValueError("missing query")- return cls.invoke_chat_app(app, user, conversation_id, query, stream, inputs, files, session)+ return cls.invoke_chat_app(+ app, user, conversation_id, query, stream, inputs, files, session, execution_driver+ )case AppMode.WORKFLOW:workflow = cls._get_workflow(app)if not workflow:raise ValueError("unexpected app type")- return cls.invoke_workflow_app(app, workflow, user, stream, inputs, files)+ return cls.invoke_workflow_app(app, workflow, user, stream, inputs, files, execution_driver)case AppMode.COMPLETION:return cls.invoke_completion_app(app, user, stream, inputs, files, session)case _:@@ -122,6 +128,7 @@ class PluginAppBackwardsInvocation(BaseBackwardsInvocation):inputs: Mapping,files: list[dict],session: Session,+ execution_driver: "WorkflowRunDriver",) -> Generator[Mapping | str] | Mapping:"""invoke chat app@@ -137,7 +144,7 @@ class PluginAppBackwardsInvocation(BaseBackwardsInvocation):state_owner_user_id=workflow.created_by,)- return AdvancedChatAppGenerator().generate(+ return AdvancedChatAppGenerator(execution_driver=execution_driver).generate(app_model=app,workflow=workflow,user=user,@@ -154,7 +161,7 @@ class PluginAppBackwardsInvocation(BaseBackwardsInvocation):session=session,)case AppMode.AGENT_CHAT:- return AgentChatAppGenerator().generate(+ return AgentChatAppGenerator(execution_driver=execution_driver).generate(app_model=app,user=user,args={@@ -193,6 +200,7 @@ class PluginAppBackwardsInvocation(BaseBackwardsInvocation):stream: bool,inputs: Mapping,files: list[dict],+ execution_driver: "WorkflowRunDriver",) -> Generator[Mapping | str] | Mapping:"""invoke workflow app@@ -202,7 +210,7 @@ class PluginAppBackwardsInvocation(BaseBackwardsInvocation):state_owner_user_id=workflow.created_by,)- return WorkflowAppGenerator().generate(+ return WorkflowAppGenerator(execution_driver=execution_driver).generate(app_model=app,workflow=workflow,user=user,diff --git a/api/core/plugin/backwards_invocation/tool.py b/api/core/plugin/backwards_invocation/tool.pyindex f6b24583153..b6f6e2c6257 100644--- a/api/core/plugin/backwards_invocation/tool.py+++ b/api/core/plugin/backwards_invocation/tool.py@@ -1,5 +1,5 @@from collections.abc import Generator-from typing import Any+from typing import TYPE_CHECKING, Anyfrom sqlalchemy.orm import Session@@ -10,6 +10,9 @@ from core.tools.tool_engine import ToolEnginefrom core.tools.tool_manager import ToolManagerfrom core.tools.utils.message_transformer import ToolFileMessageTransformer+if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+class PluginToolBackwardsInvocation(BaseBackwardsInvocation):"""@@ -27,6 +30,8 @@ class PluginToolBackwardsInvocation(BaseBackwardsInvocation):tool_name: str,tool_parameters: dict[str, Any],credential_id: str | None = None,+ *,+ execution_driver: "WorkflowRunDriver",) -> Generator[ToolInvokeMessage]:"""invoke tool@@ -41,6 +46,7 @@ class PluginToolBackwardsInvocation(BaseBackwardsInvocation):tool_parameters,user_id=user_id,credential_id=credential_id,+ execution_driver=execution_driver,)response = ToolEngine.generic_invoke(session, tool_runtime, tool_parameters, user_id, DifyWorkflowCallbackHandler(), workflow_call_depth=1diff --git a/api/core/plugin/impl/model_runtime.py b/api/core/plugin/impl/model_runtime.pyindex 4446670b8ef..15f876b82d8 100644--- a/api/core/plugin/impl/model_runtime.py+++ b/api/core/plugin/impl/model_runtime.py@@ -32,7 +32,7 @@ from graphon.model_runtime.entities.rerank_entities import MultimodalRerankInputfrom graphon.model_runtime.entities.text_embedding_entities import EmbeddingInputType, EmbeddingResultfrom graphon.model_runtime.model_providers.base.large_language_model import normalize_non_stream_runtime_resultfrom graphon.model_runtime.protocols.runtime import ModelRuntime-from graphon.model_runtime.protocols.tts_runtime import TTSModelVoice+from graphon.model_runtime.protocols.tts_runtime import TTSChunk, TTSModelVoicefrom models.provider_ids import ModelProviderIDlogger = logging.getLogger(__name__)@@ -638,10 +638,10 @@ class PluginModelRuntime(ModelRuntime):content_text: str,voice: str,request_metadata: Mapping[str, object] | None = None,- ) -> Iterable[bytes]:+ ) -> Iterable[TTSChunk]:del request_metadataplugin_id, provider_name = self._split_provider(provider)- return self.client.invoke_tts(+ chunks = self.client.invoke_tts(tenant_id=self.tenant_id,user_id=self.user_id,plugin_id=plugin_id,@@ -651,6 +651,8 @@ class PluginModelRuntime(ModelRuntime):content_text=content_text,voice=voice,)+ # ponytail: the current Dify client exposes bytes; preserve MIME metadata when its TTS response model does.+ return (TTSChunk(data=chunk, mime_type=None) for chunk in chunks)@overridedef get_tts_model_voices(diff --git a/api/core/repositories/celery_workflow_execution_repository.py b/api/core/repositories/celery_workflow_execution_repository.pyindex bc72742beb7..edbf2288abd 100644--- a/api/core/repositories/celery_workflow_execution_repository.py+++ b/api/core/repositories/celery_workflow_execution_repository.py@@ -12,6 +12,7 @@ from sqlalchemy.engine import Enginefrom sqlalchemy.orm import sessionmakerfrom core.repositories.factory import WorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepositoryfrom graphon.entities import WorkflowExecutionfrom models import Account, CreatorUserRole, EndUserfrom models.enums import WorkflowRunTriggeredFrom@@ -86,6 +87,14 @@ class CeleryWorkflowExecutionRepository(WorkflowExecutionRepository):# Determine user role based on user typeself._creator_user_role = CreatorUserRole.ACCOUNT if isinstance(user, Account) else CreatorUserRole.END_USER+ self._sql_repository = SQLAlchemyWorkflowExecutionRepository(+ session_factory=session_factory,+ tenant_id=tenant_id,+ user=user,+ app_id=app_id,+ triggered_from=triggered_from,+ )+logger.info("Initialized CeleryWorkflowExecutionRepository for tenant %s, app %s, triggered_from %s",self._tenant_id,@@ -93,6 +102,10 @@ class CeleryWorkflowExecutionRepository(WorkflowExecutionRepository):self._triggered_from,)+ @override+ def save_synchronously(self, execution: WorkflowExecution) -> None:+ self._sql_repository.save(execution)+@overridedef save(self, execution: WorkflowExecution):"""diff --git a/api/core/repositories/celery_workflow_node_execution_repository.py b/api/core/repositories/celery_workflow_node_execution_repository.pyindex c393f6afd51..e29c191a776 100644--- a/api/core/repositories/celery_workflow_node_execution_repository.py+++ b/api/core/repositories/celery_workflow_node_execution_repository.py@@ -162,6 +162,11 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):# For now, we'll re-raise the exceptionraise+ @override+ def save_execution_data(self, execution: WorkflowNodeExecution) -> None:+ """`save` already queues the complete inputs, process data, and outputs."""+ return None+@overridedef save_synchronously(self, execution: WorkflowNodeExecution) -> None:"""Create the Agent v2 caller row before runtime participant allocation."""@@ -178,6 +183,8 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):self,workflow_execution_id: str,order_config: OrderConfig | None = None,+ *,+ include_paused: bool = False,) -> Sequence[WorkflowNodeExecution]:"""Retrieve workflow node executions from cache after loading persisted history once.@@ -189,6 +196,11 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):Returns:A sequence of WorkflowNodeExecution instances"""+ if include_paused:+ # Resume hydration must not widen the ordinary runtime cache.+ return self._sql_repository.get_by_workflow_execution(+ workflow_execution_id, order_config, include_paused=True+ )try:if workflow_execution_id not in self._database_loaded_workflow_executions:try:diff --git a/api/core/repositories/factory.py b/api/core/repositories/factory.pyindex 3e8b5277402..ff0fd6bf2f3 100644--- a/api/core/repositories/factory.py+++ b/api/core/repositories/factory.py@@ -31,6 +31,10 @@ class OrderConfig:class WorkflowExecutionRepository(Protocol):def save(self, execution: WorkflowExecution): ...+ def save_synchronously(self, execution: WorkflowExecution) -> None:+ """Commit the SQL control record before publishing a lifecycle transition."""+ ...+class WorkflowNodeExecutionRepository(Protocol):def save(self, execution: WorkflowNodeExecution): ...@@ -43,6 +47,8 @@ class WorkflowNodeExecutionRepository(Protocol):self,workflow_execution_id: str,order_config: OrderConfig | None = None,+ *,+ include_paused: bool = False,) -> Sequence[WorkflowNodeExecution]: ...diff --git a/api/core/repositories/human_input_repository.py b/api/core/repositories/human_input_repository.pyindex 4d35c05e0f9..34630d1af1c 100644--- a/api/core/repositories/human_input_repository.py+++ b/api/core/repositories/human_input_repository.py@@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequencefrom datetime import datetimefrom typing import Any, Protocol, override-from sqlalchemy import select+from sqlalchemy import select, updatefrom sqlalchemy.orm import Session, selectinloadfrom core.db.session_factory import session_factory@@ -35,6 +35,7 @@ from models.human_input import (RecipientType,StandaloneWebAppRecipientPayload,)+from repositories.human_input_errors import FormAlreadyHandledError@dataclasses.dataclass(frozen=True)@@ -113,6 +114,10 @@ class HumanInputFormRepository(Protocol):def create_form(self, params: FormCreateParams) -> HumanInputFormEntity: ...+ def mark_timeout(self, node_id: str, *, form_id: str) -> HumanInputFormEntity:+ """Time out an overdue waiting form and return its current state."""+ ...+class _HumanInputFormRecipientEntityImpl(HumanInputFormRecipientEntity):def __init__(self, recipient_model: HumanInputFormRecipient):@@ -547,6 +552,8 @@ class HumanInputFormRepositoryImpl:)if form_id is not None:form_query = form_query.where(HumanInputForm.id == form_id)+ if self._app_id is not None:+ form_query = form_query.where(HumanInputForm.app_id == self._app_id)with session_factory.create_session() as session:form_model: HumanInputForm | None = session.scalars(form_query).first()if form_model is None:@@ -556,6 +563,34 @@ class HumanInputFormRepositoryImpl:recipient_models = session.scalars(recipient_query).all()return _HumanInputFormEntityImpl(form_model=form_model, recipient_models=recipient_models)+ def mark_timeout(self, node_id: str, *, form_id: str) -> HumanInputFormEntity:+ if self._workflow_execution_id is None:+ raise ValueError("workflow_execution_id is required to time out runtime human input forms")++ statement = (+ update(HumanInputForm)+ .where(+ HumanInputForm.id == form_id,+ HumanInputForm.tenant_id == self._tenant_id,+ HumanInputForm.workflow_run_id == self._workflow_execution_id,+ HumanInputForm.node_id == node_id,+ HumanInputForm.status == HumanInputFormStatus.WAITING,+ HumanInputForm.submitted_at.is_(None),+ HumanInputForm.expiration_time <= naive_utc_now(),+ )+ .values(status=HumanInputFormStatus.TIMEOUT)+ )+ if self._app_id is not None:+ statement = statement.where(HumanInputForm.app_id == self._app_id)+ with session_factory.create_session() as session, session.begin():+ session.execute(statement)++ # A submission or global expiration may have won the race to finish the form.+ form = self.get_form(node_id, form_id=form_id)+ if form is None:+ raise FormNotFoundError(f"form not found, id={form_id}")+ return form+class HumanInputFormSubmissionRepository:"""Repository for fetching and submitting human input forms."""@@ -579,6 +614,23 @@ class HumanInputFormSubmissionRepository:return Nonereturn HumanInputFormRecord.from_models(form_model, None)+ def get_by_form_ids(+ self, form_ids: Sequence[str], *, tenant_id: str, app_id: str, workflow_run_id: str+ ) -> dict[str, HumanInputFormRecord]:+ """Materialize a batch of forms without retaining a database session."""+ if not form_ids:+ return {}+ with session_factory.create_session() as session:+ forms = session.scalars(+ select(HumanInputForm).where(+ HumanInputForm.id.in_(form_ids),+ HumanInputForm.tenant_id == tenant_id,+ HumanInputForm.app_id == app_id,+ HumanInputForm.workflow_run_id == workflow_run_id,+ )+ )+ return {form.id: HumanInputFormRecord.from_models(form, None) for form in forms}+def get_by_form_id_and_recipient_type(self,form_id: str,@@ -609,9 +661,11 @@ class HumanInputFormSubmissionRepository:submission_end_user_id: str | None,) -> HumanInputFormRecord:with session_factory.create_session() as session, session.begin():- form_model = session.get(HumanInputForm, form_id)+ form_model = session.get(HumanInputForm, form_id, with_for_update=True)if form_model is None:raise FormNotFoundError(f"form not found, id={form_id}")+ if form_model.status != HumanInputFormStatus.WAITING:+ raise FormAlreadyHandledError(form_model.status)recipient_model = session.get(HumanInputFormRecipient, recipient_id) if recipient_id else Nonediff --git a/api/core/repositories/sqlalchemy_workflow_execution_repository.py b/api/core/repositories/sqlalchemy_workflow_execution_repository.pyindex 6be44a69394..699526597a6 100644--- a/api/core/repositories/sqlalchemy_workflow_execution_repository.py+++ b/api/core/repositories/sqlalchemy_workflow_execution_repository.py@@ -200,10 +200,16 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):# Create a new database sessionwith self._session_factory() as session:- existing_model = session.get(WorkflowRun, db_model.id)+ existing_model = session.get(WorkflowRun, db_model.id, with_for_update=True)if existing_model:if existing_model.tenant_id != self._tenant_id:raise ValueError("Unauthorized access to workflow run")+ # A delayed persistence task must not resurrect a durably stopped run.+ if (+ existing_model.stop_requested_at is not None+ and existing_model.status == WorkflowExecutionStatus.STOPPED+ ):+ return# Preserve the original start time for pause/resume flows.db_model.created_at = existing_model.created_at@@ -214,3 +220,7 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):# Update the in-memory cache for faster subsequent lookupsself._execution_cache[db_model.id] = db_model++ @override+ def save_synchronously(self, execution: WorkflowExecution) -> None:+ self.save(execution)diff --git a/api/core/repositories/sqlalchemy_workflow_node_execution_repository.py b/api/core/repositories/sqlalchemy_workflow_node_execution_repository.pyindex 8d3c449857c..570bbbe10f2 100644--- a/api/core/repositories/sqlalchemy_workflow_node_execution_repository.py+++ b/api/core/repositories/sqlalchemy_workflow_node_execution_repository.py@@ -35,6 +35,7 @@ from models import (from models.enums import ExecutionOffLoadTypefrom models.model import UploadFilefrom models.workflow import WorkflowNodeExecutionOffload+from repositories.workflow_run_control import apply_workflow_stop_to_node, get_stopped_workflow_runfrom services.file_service import FileServicefrom services.variable_truncator import VariableTruncator@@ -390,6 +391,17 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)db_model: The database model to persist"""with self._session_factory() as session:+ if db_model.status in (+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ ):+ apply_workflow_stop_to_node(+ db_model,+ get_stopped_workflow_run(+ session, tenant_id=self._tenant_id, workflow_run_id=db_model.workflow_run_id, lock=True+ ),+ )# Check if record already existsexisting = session.get(WorkflowNodeExecutionModel, db_model.id)@@ -479,6 +491,17 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)db_model.offload_data = offload_datawith self._session_factory() as session, session.begin():+ if db_model.status in (+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ ):+ apply_workflow_stop_to_node(+ db_model,+ get_stopped_workflow_run(+ session, tenant_id=self._tenant_id, workflow_run_id=db_model.workflow_run_id, lock=True+ ),+ )session.merge(db_model)session.flush()@@ -486,7 +509,8 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)self,workflow_run_id: str,order_config: OrderConfig | None = None,- triggered_from: WorkflowNodeExecutionTriggeredFrom = WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ *,+ include_paused: bool = False,) -> Sequence[WorkflowNodeExecutionModel]:"""Retrieve all WorkflowNodeExecution database models for a specific workflow run.@@ -512,10 +536,13 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)stmt = stmt.where(WorkflowNodeExecutionModel.workflow_run_id == workflow_run_id,WorkflowNodeExecutionModel.tenant_id == self._tenant_id,- WorkflowNodeExecutionModel.triggered_from == triggered_from,- WorkflowNodeExecutionModel.status != WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionModel.triggered_from+ == (self._triggered_from or WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN),)+ if not include_paused:+ stmt = stmt.where(WorkflowNodeExecutionModel.status != WorkflowNodeExecutionStatus.PAUSED)+if self._app_id:stmt = stmt.where(WorkflowNodeExecutionModel.app_id == self._app_id)@@ -548,7 +575,8 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)self,workflow_execution_id: str,order_config: OrderConfig | None = None,- triggered_from: WorkflowNodeExecutionTriggeredFrom = WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ *,+ include_paused: bool = False,) -> Sequence[WorkflowNodeExecution]:"""Retrieve all node executions for a workflow execution.@@ -565,7 +593,9 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)Returns:A list of node execution instances"""- db_models = self.get_db_models_by_workflow_run(workflow_execution_id, order_config, triggered_from)+ db_models = self.get_db_models_by_workflow_run(+ workflow_execution_id, order_config, include_paused=include_paused+ )with ThreadPoolExecutor(max_workers=10) as executor:domain_models = executor.map(self._to_domain_model, db_models, timeout=30)diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.pyindex 4352a8679d9..61935b9d231 100644--- a/api/core/tools/tool_manager.py+++ b/api/core/tools/tool_manager.py@@ -63,7 +63,7 @@ from services.tools.mcp_tools_manage_service import MCPToolManageServicefrom services.tools.tools_transform_service import ToolTransformServiceif TYPE_CHECKING:- pass+ from core.app.apps.workflow_app_runner import WorkflowRunDriverlogger = logging.getLogger(__name__)@@ -194,6 +194,7 @@ class ToolManager:invoke_from: InvokeFrom = InvokeFrom.DEBUGGER,tool_invoke_from: ToolInvokeFrom = ToolInvokeFrom.AGENT,credential_id: str | None = None,+ execution_driver: "WorkflowRunDriver | None" = None,) -> BuiltinTool | PluginTool | ApiTool | WorkflowTool | MCPTool:"""get the tool runtime@@ -380,7 +381,7 @@ class ToolManager:if controller_tools is None or len(controller_tools) == 0:raise ToolProviderNotFoundError(f"workflow provider {provider_id} not found")- return controller.get_tools(tenant_id=workflow_provider.tenant_id)[0].fork_tool_runtime(+ workflow_tool = controller.get_tools(tenant_id=workflow_provider.tenant_id)[0].fork_tool_runtime(runtime=ToolRuntime(tenant_id=tenant_id,user_id=user_id,@@ -389,6 +390,8 @@ class ToolManager:tool_invoke_from=tool_invoke_from,))+ workflow_tool.execution_driver = execution_driver+ return workflow_toolcase ToolProviderType.APP:raise NotImplementedError("app provider not implemented")case ToolProviderType.PLUGIN:@@ -423,6 +426,7 @@ class ToolManager:variable_pool: "VariablePool | None" = None,allow_file_parameters: bool = False,use_default_for_missing_form_parameters: bool = False,+ execution_driver: "WorkflowRunDriver | None" = None,) -> Tool:"""get the agent tool runtime@@ -436,6 +440,7 @@ class ToolManager:invoke_from=invoke_from,tool_invoke_from=ToolInvokeFrom.AGENT,credential_id=agent_tool.credential_id,+ execution_driver=execution_driver,)runtime_parameters: dict[str, Any] = {}parameters = tool_entity.get_merged_runtime_parameters()@@ -472,6 +477,7 @@ class ToolManager:user_id: str | None = None,invoke_from: InvokeFrom = InvokeFrom.DEBUGGER,variable_pool: "VariablePool | None" = None,+ execution_driver: "WorkflowRunDriver | None" = None,) -> Tool:"""get the workflow tool runtime@@ -486,6 +492,7 @@ class ToolManager:invoke_from=invoke_from,tool_invoke_from=ToolInvokeFrom.WORKFLOW,credential_id=workflow_tool.credential_id,+ execution_driver=execution_driver,)parameters = tool_runtime.get_merged_runtime_parameters()@@ -517,6 +524,7 @@ class ToolManager:tool_parameters: dict[str, Any],user_id: str | None = None,credential_id: str | None = None,+ execution_driver: "WorkflowRunDriver | None" = None,) -> Tool:"""get tool runtime from plugin@@ -530,6 +538,7 @@ class ToolManager:invoke_from=InvokeFrom.SERVICE_API,tool_invoke_from=ToolInvokeFrom.PLUGIN,credential_id=credential_id,+ execution_driver=execution_driver,)runtime_parameters: dict[str, Any] = {}parameters = tool_entity.get_merged_runtime_parameters()diff --git a/api/core/tools/workflow_as_tool/tool.py b/api/core/tools/workflow_as_tool/tool.pyindex 4c8a5800604..a7ebe79e8fb 100644--- a/api/core/tools/workflow_as_tool/tool.py+++ b/api/core/tools/workflow_as_tool/tool.py@@ -4,7 +4,7 @@ import contextlibimport jsonimport loggingfrom collections.abc import Generator, Mapping, Sequence-from typing import Any, cast, override+from typing import TYPE_CHECKING, Any, cast, overridefrom sqlalchemy import selectfrom sqlalchemy.orm import Session@@ -34,6 +34,9 @@ from models.model import App, EndUserfrom models.utils.file_input_compat import build_file_from_stored_mappingfrom models.workflow import Workflow+if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+logger = logging.getLogger(__name__)_file_access_controller = DatabaseFileAccessController()@@ -56,6 +59,7 @@ class WorkflowTool(Tool):entity: ToolEntity,runtime: ToolRuntime,label: str = "Workflow",+ execution_driver: WorkflowRunDriver | None = None,):self.workflow_app_id = workflow_app_idself.workflow_as_tool_id = workflow_as_tool_id@@ -63,6 +67,7 @@ class WorkflowTool(Tool):self.workflow_entities = workflow_entitiesself.workflow_call_depth = workflow_call_depthself.label = label+ self.execution_driver = execution_driverself._latest_usage = LLMUsage.empty_usage()self._parent_trace_context = Noneself._trace_session_id = None@@ -99,7 +104,9 @@ class WorkflowTool(Tool):from core.app.apps.workflow.app_generator import WorkflowAppGenerator- generator = WorkflowAppGenerator()+ if self.execution_driver is None:+ raise ToolInvokeError("Workflow execution driver is required")+ generator = WorkflowAppGenerator(execution_driver=self.execution_driver)assert self.runtime is not Noneassert self.runtime.invoke_from is not None@@ -226,6 +233,7 @@ class WorkflowTool(Tool):workflow_call_depth=self.workflow_call_depth,version=self.version,label=self.label,+ execution_driver=self.execution_driver,)forked._parent_trace_context = self._parent_trace_context.model_copy() if self._parent_trace_context else Noneforked._trace_session_id = self._trace_session_iddiff --git a/api/core/workflow/human_input_adapter.py b/api/core/workflow/human_input_adapter.pyindex 52021f9f49b..ee99d78246e 100644--- a/api/core/workflow/human_input_adapter.py+++ b/api/core/workflow/human_input_adapter.py@@ -237,6 +237,8 @@ def adapt_node_data_for_graph(node_data: Mapping[str, Any] | BaseModel) -> dict[raise TypeError(f"node data must be a mapping, got {type(node_data).__name__}")node_type = normalized.get("type")+ if node_type == BuiltinNodeTypes.LEGACY_VARIABLE_AGGREGATOR:+ normalized["type"] = BuiltinNodeTypes.VARIABLE_AGGREGATORif node_type == BuiltinNodeTypes.HUMAN_INPUT:return adapt_human_input_node_data_for_graph(normalized)if node_type == BuiltinNodeTypes.TOOL:diff --git a/api/core/workflow/human_input_policy.py b/api/core/workflow/human_input_policy.pyindex 6bc415649ed..8441072dde0 100644--- a/api/core/workflow/human_input_policy.py+++ b/api/core/workflow/human_input_policy.py@@ -11,7 +11,7 @@ from core.workflow.nodes.human_input.pause_reason import (HumanInputRequired,PauseReason,)-from graphon.runtime.graph_runtime_state_protocol import ReadOnlyVariablePool+from graphon.runtime.runtime_state_protocol import ReadOnlyVariablePoolfrom graphon.variables import ArrayStringSegmentfrom models.human_input import ApprovalChannel, RecipientType@@ -121,11 +121,8 @@ def resolve_variable_select_input_options() -> list[FormInputConfig]:"""Resolve variable-backed select options to runtime values."""- # This function replace the SelectInputConfig.option_source.value- # field with acutial runtime values when option_source.type is VARIABLE.- #- # This is a dirty hacks. However it does reduces the logic leaked to callers of- # the api.+ # Convert variable-backed options into a self-contained constant source so+ # downstream callers do not need the runtime variable pool or selector.resolved_inputs: list[FormInputConfig] = []if variable_pool is None:@@ -148,8 +145,13 @@ def resolve_variable_select_input_options(if not isinstance(option_values, ArrayStringSegment):raise TypeError(f"expected ArrayStringSegment, got {type(option_values)}")- updated_option_source = option_source.model_copy(update={"value": option_values.value})- # Ensure frontend receives concrete select options instead of unresolved selectors.+ updated_option_source = option_source.model_copy(+ update={+ "type": ValueSourceType.CONSTANT,+ "selector": (),+ "value": option_values.value,+ }+ )resolved_inputs.append(form_input.model_copy(update={"option_source": updated_option_source},@@ -171,7 +173,6 @@ def resolve_human_input_pause_reason_inputs(if not isinstance(reason, HumanInputRequired):resolved_reasons.append(reason)continue-resolved_inputs = resolve_variable_select_input_options(reason.inputs,variable_pool=variable_pool,diff --git a/api/core/workflow/llm_node.py b/api/core/workflow/llm_node.pyindex 2bb1aa49040..3448a276346 100644--- a/api/core/workflow/llm_node.py+++ b/api/core/workflow/llm_node.py@@ -1,9 +1,10 @@from collections.abc import Callable, Generator, Sequencefrom typing import Any, override+from graphon.file import Filefrom graphon.model_runtime.entities.llm_entities import LLMStructuredOutputfrom graphon.model_runtime.entities.message_entities import PromptMessage-from graphon.node_events.base import NodeEventBase+from graphon.node_events import NodeEventPayloadfrom graphon.nodes.llm.node import LLMNodefrom graphon.nodes.llm.runtime_protocols import LLMPollingCapableProtocol@@ -30,12 +31,14 @@ class DifyLLMNode(LLMNode):def _invoke_llm_with_polling(self,*,+ file_outputs: list[File],polling_model: LLMPollingCapableProtocol,prompt_messages: Sequence[PromptMessage],stop: Sequence[str] | None,- ) -> Generator[NodeEventBase | LLMStructuredOutput]:+ ) -> Generator[NodeEventPayload | LLMStructuredOutput]:try:yield from super()._invoke_llm_with_polling(+ file_outputs=file_outputs,polling_model=polling_model,prompt_messages=prompt_messages,stop=stop,diff --git a/api/core/workflow/node_factory.py b/api/core/workflow/node_factory.pyindex 7f67415fb05..d7b1aff4aa9 100644--- a/api/core/workflow/node_factory.py+++ b/api/core/workflow/node_factory.py@@ -1,6 +1,7 @@import importlibimport pkgutilfrom collections.abc import Callable, Iterator, Mapping, MutableMapping, Sequence+from copy import copyfrom dataclasses import dataclassfrom functools import lru_cachefrom typing import TYPE_CHECKING, Any, cast, final, override@@ -22,7 +23,7 @@ from core.memory.token_buffer_memory import TokenBufferMemoryfrom core.model_manager import ModelInstancefrom core.prompt.entities.advanced_prompt_entities import MemoryConfigfrom core.trigger.constants import TRIGGER_NODE_TYPES-from core.workflow.human_input_adapter import adapt_node_config_for_graph+from core.workflow.human_input_adapter import adapt_node_config_for_graph, parse_human_input_delivery_methodsfrom core.workflow.llm_environment_variable import (parse_llm_model_selector,resolve_llm_model_config,@@ -58,7 +59,7 @@ from core.workflow.template_rendering import CodeExecutorJinja2TemplateRendererfrom extensions.application_services.data_sources import build_data_source_credentialsfrom graphon.entities.base_node_data import BaseNodeDatafrom graphon.entities.graph_config import NodeConfigDict, NodeConfigDictAdapter-from graphon.enums import BuiltinNodeTypes, NodeType+from graphon.enums import BuiltinNodeTypes, NodeExecutionType, NodeTypefrom graphon.file.file_manager import file_managerfrom graphon.graph.graph import NodeFactoryfrom graphon.model_runtime.memory import PromptMessageMemory@@ -76,8 +77,8 @@ from graphon.variables.segments import ArrayObjectSegment, ObjectSegmentfrom models.model import Conversationif TYPE_CHECKING:- from graphon.entities import GraphInitParams- from graphon.runtime import GraphRuntimeState+ from core.app.apps.workflow_app_runner import WorkflowRunDriver+ from graphon.runtime import InitParams, RuntimeStateLATEST_VERSION = "latest"_START_NODE_TYPES: frozenset[NodeType] = frozenset(@@ -89,7 +90,7 @@ _START_NODE_TYPES: frozenset[NodeType] = frozenset(class DifyGraphInitContext:"""Explicit graph-init values owned by the workflow layer.- Dify is gradually removing direct `GraphInitParams` construction from its+ Dify is gradually removing direct `InitParams` construction from itsproduction call sites. Keep the translation here until `graphon` exposes anequivalent explicit API."""@@ -98,11 +99,12 @@ class DifyGraphInitContext:graph_config: Mapping[str, Any]run_context: Mapping[str, Any]call_depth: int+ execution_driver: "WorkflowRunDriver | None" = None- def to_graph_init_params(self) -> "GraphInitParams":- from graphon.entities import GraphInitParams+ def to_graph_init_params(self) -> "InitParams":+ from graphon.runtime import InitParams- return GraphInitParams(+ return InitParams(workflow_id=self.workflow_id,graph_config=self.graph_config,run_context=self.run_context,@@ -310,22 +312,25 @@ class DifyNodeFactory(NodeFactory):cls,*,graph_init_context: DifyGraphInitContext,- graph_runtime_state: "GraphRuntimeState",+ runtime_state: "RuntimeState",) -> "DifyNodeFactory":"""Bridge Dify's explicit init context into the current `graphon` API."""return cls(- graph_init_params=graph_init_context.to_graph_init_params(),- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_context.to_graph_init_params(),+ runtime_state=runtime_state,+ execution_driver=graph_init_context.execution_driver,)def __init__(self,- graph_init_params: "GraphInitParams",- graph_runtime_state: "GraphRuntimeState",+ init_params: "InitParams",+ runtime_state: "RuntimeState",+ execution_driver: "WorkflowRunDriver | None" = None,) -> None:- self.graph_init_params = graph_init_params- self.graph_runtime_state = graph_runtime_state- self._dify_context = self._resolve_dify_context(graph_init_params.run_context)+ self.init_params = init_params+ self.runtime_state = runtime_state+ self._dify_context = self._resolve_dify_context(init_params.run_context)+ self._execution_driver = execution_driverself._code_executor: CodeExecutorProtocol = DefaultWorkflowCodeExecutor()self._code_limits = CodeNodeLimits(max_string_length=dify_config.CODE_MAX_STRING_LENGTH,@@ -358,12 +363,12 @@ class DifyNodeFactory(NodeFactory):self._human_input_runtime = DifyHumanInputNodeRuntime(self._dify_context,workflow_execution_id_getter=lambda: get_system_text(- self.graph_runtime_state.variable_pool,+ self.runtime_state.variable_pool,SystemVariableKey.WORKFLOW_EXECUTION_ID,),conversation_id_getter=self._conversation_id,)- self._tool_runtime = DifyToolNodeRuntime(self._dify_context)+ self._tool_runtime = DifyToolNodeRuntime(self._dify_context, execution_driver=execution_driver)self._http_request_file_manager = file_managerself._document_extractor_unstructured_api_config = UnstructuredApiConfig(api_url=dify_config.UNSTRUCTURED_API_URL,@@ -385,12 +390,24 @@ class DifyNodeFactory(NodeFactory):self._agent_runtime_support = AgentRuntimeSupport()self._agent_message_transformer = AgentMessageTransformer()- def with_runtime_state(self, graph_runtime_state: "GraphRuntimeState") -> "DifyNodeFactory":+ @override+ def with_runtime_state(self, runtime_state: "RuntimeState") -> "DifyNodeFactory":return DifyNodeFactory(- graph_init_params=self.graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=self.init_params,+ runtime_state=runtime_state,+ execution_driver=self._execution_driver,)+ @override+ def with_graph_config(self, graph_config: Mapping[str, Any]) -> "DifyNodeFactory":+ factory = copy(self)+ factory.init_params = self.init_params.model_copy(update={"graph_config": graph_config})+ return factory++ @property+ def execution_driver(self) -> "WorkflowRunDriver | None":+ return self._execution_driver+@staticmethoddef _resolve_dify_context(run_context: Mapping[str, Any]) -> DifyRunContext:raw_ctx = run_context.get(DIFY_RUN_CONTEXT_KEY)@@ -401,7 +418,7 @@ class DifyNodeFactory(NodeFactory):return DifyRunContext.model_validate(raw_ctx)def _conversation_id(self) -> str | None:- return get_system_text(self.graph_runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)+ return get_system_text(self.runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)@overridedef create_node(self, node_config: dict[str, Any] | NodeConfigDict) -> Node:@@ -414,19 +431,9 @@ class DifyNodeFactory(NodeFactory):(including pydantic ValidationError, which subclasses ValueError),if node type is unknown, or if no implementation exists for the resolved version"""- adapted_node_config = adapt_node_config_for_graph(node_config)- typed_node_config = NodeConfigDictAdapter.validate_python(adapted_node_config)+ adapted_node_config, typed_node_config, node_class, resolved_node_data = self._validate_node_config(node_config)node_id = typed_node_config["id"]node_data = typed_node_config["data"]- node_class = self._resolve_node_class(- node_type=node_data.type,- node_version=str(node_data.version),- node_data=node_data,- )- # Graph configs are initially validated against permissive shared node data.- # Re-validate using the resolved node class so workflow-local node schemas- # stay explicit and constructors receive the concrete typed payload.- resolved_node_data = self._validate_resolved_node_data(node_class, node_data)node_type = node_data.typeif node_type == BuiltinNodeTypes.LLM:resolved_node_data = self._resolve_llm_model_reference(cast(LLMNodeData, resolved_node_data))@@ -503,12 +510,35 @@ class DifyNodeFactory(NodeFactory):node = node_class(node_id=node_id,data=constructor_node_data,- graph_init_params=self.graph_init_params,- graph_runtime_state=self.graph_runtime_state,+ init_params=self.init_params,+ runtime_state=self.runtime_state,**node_init_kwargs,)return node+ @override+ def validate_node(self, node_config: NodeConfigDict) -> NodeExecutionType:+ adapted_node_config, typed_node_config, node_class, _ = self._validate_node_config(node_config)+ if typed_node_config["data"].type == BuiltinNodeTypes.HUMAN_INPUT:+ node_data = DifyHumanInputNodeData.model_validate(adapted_node_config["data"])+ parse_human_input_delivery_methods(node_data)+ return node_class.execution_type++ def _validate_node_config(+ self,+ node_config: Mapping[str, Any],+ ) -> tuple[dict[str, Any], NodeConfigDict, type[Node], BaseNodeData]:+ adapted_node_config = adapt_node_config_for_graph(node_config)+ typed_node_config = NodeConfigDictAdapter.validate_python(adapted_node_config)+ node_data = typed_node_config["data"]+ node_class = self._resolve_node_class(+ node_type=node_data.type,+ node_version=str(node_data.version),+ node_data=node_data,+ )+ resolved_node_data = self._validate_resolved_node_data(node_class, node_data)+ return adapted_node_config, typed_node_config, node_class, resolved_node_data+@staticmethoddef _validate_resolved_node_data(node_class: type[Node], node_data: BaseNodeData) -> BaseNodeData:"""@@ -542,7 +572,7 @@ class DifyNodeFactory(NodeFactory):return node_dataselector = parse_llm_model_selector(model_selector)- variable = self.graph_runtime_state.variable_pool.get(selector)+ variable = self.runtime_state.variable_pool.get(selector)if not isinstance(variable, ObjectSegment):raise ValueError(f"LLM environment variable '{selector[1]}' was not found or is not an LLM variable")@@ -703,7 +733,7 @@ class DifyNodeFactory(NodeFactory):if not context_variable_selector:return False- context_value = self.graph_runtime_state.variable_pool.get(context_variable_selector)+ context_value = self.runtime_state.variable_pool.get(context_variable_selector)if not isinstance(context_value, ArrayObjectSegment):return False@@ -738,7 +768,7 @@ class DifyNodeFactory(NodeFactory):if node_data.memory is None:return None- conversation_id = get_system_text(self.graph_runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)+ conversation_id = get_system_text(self.runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)return fetch_memory(conversation_id=conversation_id,app_id=self._dify_context.app_id,diff --git a/api/core/workflow/node_runtime.py b/api/core/workflow/node_runtime.pyindex c54cd7123d5..eb0eef4742a 100644--- a/api/core/workflow/node_runtime.py+++ b/api/core/workflow/node_runtime.py@@ -117,6 +117,7 @@ class PollingLLMRuntimeProtocol(Protocol):if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriverfrom core.tools.__base.tool import Toolfrom core.tools.entities.tool_entities import ToolInvokeMessage as CoreToolInvokeMessagefrom graphon.nodes.llm.file_saver import LLMFileSaver@@ -528,10 +529,12 @@ class DifyToolNodeRuntime(ToolNodeRuntimeProtocol):self,run_context: Mapping[str, Any] | DifyRunContext,session_maker: sessionmaker[Session] | None = None,+ execution_driver: WorkflowRunDriver | None = None,) -> None:self._run_context = resolve_dify_run_context(run_context)self._file_reference_factory = DifyFileReferenceFactory(self._run_context)self._session_maker = session_maker+ self._execution_driver = execution_driver@propertydef file_reference_factory(self) -> FileReferenceFactoryProtocol:@@ -559,6 +562,7 @@ class DifyToolNodeRuntime(ToolNodeRuntimeProtocol):self._run_context.user_id,self._run_context.invoke_from,variable_pool,+ execution_driver=self._execution_driver,)except ToolNodeError:raise@@ -910,11 +914,11 @@ class DifyHumanInputNodeRuntime:def restore_submitted_data(self,*,- node_data: HumanInputNodeData,+ inputs: Sequence[FormInputConfig],submitted_data: Mapping[str, Any],) -> Mapping[str, Any]:restored_data: dict[str, Any] = dict(submitted_data)- for input_config in node_data.inputs:+ for input_config in inputs:output_variable_name = input_config.output_variable_nameif output_variable_name not in submitted_data:continuediff --git a/api/core/workflow/nodes/agent/agent_node.py b/api/core/workflow/nodes/agent/agent_node.pyindex 0f9ae0480d8..d5036f09f4c 100644--- a/api/core/workflow/nodes/agent/agent_node.py+++ b/api/core/workflow/nodes/agent/agent_node.py@@ -6,9 +6,9 @@ from typing import TYPE_CHECKING, Any, overridefrom core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContextfrom core.workflow.system_variables import SystemVariableKey, get_system_text+from graphon.engine_events import NodeEventfrom graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus-from graphon.graph_events import GraphNodeEventBase-from graphon.node_events import NodeEventBase, NodeRunResult, StreamCompletedEvent+from graphon.node_events import NodeEventPayload, NodeRunResult, StreamCompletedEventfrom graphon.nodes.base.node import Nodefrom graphon.nodes.base.variable_template_parser import VariableTemplateParser@@ -23,8 +23,7 @@ from .runtime_support import AgentRuntimeSupportfrom .strategy_protocols import AgentStrategyPresentationProvider, AgentStrategyResolverif TYPE_CHECKING:- from graphon.entities import GraphInitParams- from graphon.runtime import GraphRuntimeState+ from graphon.runtime import InitParams, RuntimeStateclass AgentNode(Node[AgentNodeData]):@@ -40,8 +39,8 @@ class AgentNode(Node[AgentNodeData]):node_id: str,data: AgentNodeData,*,- graph_init_params: GraphInitParams,- graph_runtime_state: GraphRuntimeState,+ init_params: InitParams,+ runtime_state: RuntimeState,strategy_resolver: AgentStrategyResolver,presentation_provider: AgentStrategyPresentationProvider,runtime_support: AgentRuntimeSupport,@@ -50,8 +49,8 @@ class AgentNode(Node[AgentNodeData]):super().__init__(node_id=node_id,data=data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)self._strategy_resolver = strategy_resolverself._presentation_provider = presentation_provider@@ -77,8 +76,8 @@ class AgentNode(Node[AgentNodeData]):@override@singledispatchmethoddef _dispatch( # pyrefly: ignore[missing-override-decorator]- self, event: NodeEventBase- ) -> GraphNodeEventBase:+ self, event: NodeEventPayload+ ) -> NodeEvent:return super()._dispatch(event)@_dispatch.register@@ -98,7 +97,7 @@ class AgentNode(Node[AgentNodeData]):)@override- def _run(self) -> Generator[NodeEventBase]:+ def _run(self) -> Generator[NodeEventPayload]:from core.plugin.impl.exc import PluginDaemonClientSideErrordify_ctx = DifyRunContext.model_validate(self.require_run_context_value(DIFY_RUN_CONTEXT_KEY))@@ -123,7 +122,7 @@ class AgentNode(Node[AgentNodeData]):parameters = self._runtime_support.build_parameters(agent_parameters=agent_parameters,- variable_pool=self.graph_runtime_state.variable_pool,+ variable_pool=self.runtime_state.variable_pool,node_data=self.node_data,strategy=strategy,tenant_id=dify_ctx.tenant_id,@@ -133,7 +132,7 @@ class AgentNode(Node[AgentNodeData]):)parameters_for_log = self._runtime_support.build_parameters(agent_parameters=agent_parameters,- variable_pool=self.graph_runtime_state.variable_pool,+ variable_pool=self.runtime_state.variable_pool,node_data=self.node_data,strategy=strategy,tenant_id=dify_ctx.tenant_id,@@ -144,7 +143,7 @@ class AgentNode(Node[AgentNodeData]):)credentials = self._runtime_support.build_credentials(parameters=parameters)- conversation_id = get_system_text(self.graph_runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)+ conversation_id = get_system_text(self.runtime_state.variable_pool, SystemVariableKey.CONVERSATION_ID)try:message_stream = strategy.invoke(diff --git a/api/core/workflow/nodes/agent/events.py b/api/core/workflow/nodes/agent/events.pyindex 824decb5199..24820fdbe40 100644--- a/api/core/workflow/nodes/agent/events.py+++ b/api/core/workflow/nodes/agent/events.py@@ -3,11 +3,11 @@ from typing import Anyfrom pydantic import Field-from graphon.graph_events import GraphNodeEventBase-from graphon.node_events import NodeEventBase+from graphon.engine_events import NodeEvent+from graphon.node_events import NodeEventPayload-class AgentLogEvent(NodeEventBase):+class AgentLogEvent(NodeEventPayload):message_id: str = Field(..., description="id")label: str = Field(..., description="label")node_execution_id: str = Field(..., description="node execution id")@@ -19,11 +19,7 @@ class AgentLogEvent(NodeEventBase):node_id: str = Field(..., description="node id")-class GraphAgentNodeEventBase(GraphNodeEventBase):- pass---class NodeRunAgentLogEvent(GraphAgentNodeEventBase):+class NodeRunAgentLogEvent(NodeEvent):message_id: str = Field(..., description="message id")label: str = Field(..., description="label")node_execution_id: str = Field(..., description="node execution id")diff --git a/api/core/workflow/nodes/agent/message_transformer.py b/api/core/workflow/nodes/agent/message_transformer.pyindex f23c9d2e6c8..de64593159f 100644--- a/api/core/workflow/nodes/agent/message_transformer.py+++ b/api/core/workflow/nodes/agent/message_transformer.py@@ -16,7 +16,7 @@ from graphon.file import File, FileTransferMethod, get_file_type_by_mime_typefrom graphon.model_runtime.entities.llm_entities import LLMUsage, LLMUsageMetadatafrom graphon.model_runtime.utils.encoders import jsonable_encoderfrom graphon.node_events import (- NodeEventBase,+ NodeEventPayload,NodeRunResult,StreamChunkEvent,StreamCompletedEvent,@@ -45,7 +45,7 @@ class AgentMessageTransformer:node_type: NodeType,node_id: str,node_execution_id: str,- ) -> Generator[NodeEventBase]:+ ) -> Generator[NodeEventPayload]:from core.plugin.impl.plugin import PluginInstallermessage_stream = ToolFileMessageTransformer.transform_tool_invoke_messages(diff --git a/api/core/workflow/nodes/agent_v2/agent_node.py b/api/core/workflow/nodes/agent_v2/agent_node.pyindex f74d44e5012..eb050a8e789 100644--- a/api/core/workflow/nodes/agent_v2/agent_node.py+++ b/api/core/workflow/nodes/agent_v2/agent_node.py@@ -24,6 +24,7 @@ from clients.agent_backend import ()from core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContextfrom core.workflow.system_variables import SystemVariableKey, get_system_text+from graphon.engine_events import NodeRunPauseRequestedEventfrom graphon.enums import (BuiltinNodeTypes,ErrorStrategy,@@ -31,8 +32,7 @@ from graphon.enums import (WorkflowNodeExecutionMetadataKey,WorkflowNodeExecutionStatus,)-from graphon.graph_events import NodeRunPauseRequestedEvent-from graphon.node_events import NodeEventBase, NodeRunResult, StreamChunkEvent, StreamCompletedEvent+from graphon.node_events import NodeEventPayload, NodeRunResult, StreamChunkEvent, StreamCompletedEventfrom graphon.nodes.base.node import Nodefrom graphon.nodes.base.variable_template_parser import VariableTemplateParserfrom models.agent_config_entities import WorkflowNodeJobConfig, WorkflowOutputRoutes@@ -57,8 +57,7 @@ from .runtime_request_builder import (from .session_store import WorkflowAgentSessionScope, WorkflowAgentWorkspaceStoreif TYPE_CHECKING:- from graphon.entities import GraphInitParams- from graphon.runtime import GraphRuntimeState+ from graphon.runtime import InitParams, RuntimeStatelogger = logging.getLogger(__name__)@@ -88,8 +87,8 @@ class DifyAgentNode(Node[DifyAgentNodeData]):node_id: str,data: DifyAgentNodeData,*,- graph_init_params: GraphInitParams,- graph_runtime_state: GraphRuntimeState,+ init_params: InitParams,+ runtime_state: RuntimeState,binding_resolver: WorkflowAgentBindingResolver,runtime_request_builder: WorkflowAgentRuntimeRequestBuilder,agent_backend_client: AgentBackendRunClient,@@ -102,10 +101,9 @@ class DifyAgentNode(Node[DifyAgentNodeData]):super().__init__(node_id=node_id,data=data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)- # The base node parses the factory's serialized payload into node_data.if self.node_data.agent_output_routes.enabled and self.error_strategy == ErrorStrategy.DEFAULT_VALUE:raise ValueError("Output routes do not support the node-level default-value error strategy.")if self.node_data.agent_output_routes.enabled:@@ -129,7 +127,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):event.extras["agent_node"] = {"version": "2", "agent_node_kind": self.node_data.agent_node_kind}@override- def _run(self) -> Generator[NodeEventBase | NodeRunPauseRequestedEvent]:+ def _run(self) -> Generator[NodeEventPayload | NodeRunPauseRequestedEvent]:inputs: dict[str, Any] = {}process_data: dict[str, Any] = {}metadata: dict[str, Any] = {@@ -156,16 +154,16 @@ class DifyAgentNode(Node[DifyAgentNodeData]):inputs: dict[str, Any],process_data: dict[str, Any],metadata: dict[str, Any],- ) -> Generator[NodeEventBase | NodeRunPauseRequestedEvent]:+ ) -> Generator[NodeEventPayload | NodeRunPauseRequestedEvent]:dify_ctx = DifyRunContext.model_validate(self.require_run_context_value(DIFY_RUN_CONTEXT_KEY))- workflow_id = self.graph_init_params.workflow_id+ workflow_id = self.init_params.workflow_idworkflow_run_id = get_system_text(- self.graph_runtime_state.variable_pool,+ self.runtime_state.variable_pool,SystemVariableKey.WORKFLOW_EXECUTION_ID,)# Chatflow sessions also carry their owning conversation.conversation_id = get_system_text(- self.graph_runtime_state.variable_pool,+ self.runtime_state.variable_pool,SystemVariableKey.CONVERSATION_ID,)@@ -248,7 +246,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):workflow_run_id=workflow_run_id,node_id=self._node_id,node_execution_id=self.execution_id,- variable_pool=self.graph_runtime_state.variable_pool,+ variable_pool=self.runtime_state.variable_pool,binding=bundle.binding,agent=bundle.agent,snapshot=bundle.snapshot,@@ -577,11 +575,8 @@ class DifyAgentNode(Node[DifyAgentNodeData]):return cancellation, Nonedef _is_graph_aborted(self) -> bool:- """Let Agent SSE consumption observe GraphEngine's cooperative abort state."""- try:- return self.graph_runtime_state.graph_execution.aborted- except (AttributeError, RuntimeError):- return False+ """Let Agent SSE consumption observe Engine's cooperative abort state."""+ return self.runtime_state.graph_execution.aborteddef _stream_stop_reason(self) -> str:return "workflow_graph_aborted" if self._is_graph_aborted() else "event_stream_failed"diff --git a/api/core/workflow/nodes/agent_v2/workspace_retirement_layer.py b/api/core/workflow/nodes/agent_v2/workspace_retirement_layer.pyindex 7c44897fcb7..0bb793c0aa3 100644--- a/api/core/workflow/nodes/agent_v2/workspace_retirement_layer.py+++ b/api/core/workflow/nodes/agent_v2/workspace_retirement_layer.py@@ -8,9 +8,9 @@ from typing import overridefrom core.app.entities.app_invoke_entities import DifyRunContextfrom core.workflow.nodes.agent_v2.session_store import WorkflowAgentWorkspaceStorefrom core.workflow.system_variables import SystemVariableKey, get_system_text-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import (- GraphEngineEvent,+from graphon.engine.layer import Layer+from graphon.engine_events import (+ EngineEvent,GraphRunAbortedEvent,GraphRunFailedEvent,GraphRunPartialSucceededEvent,@@ -21,7 +21,7 @@ from tasks.collect_agent_resources_task import enqueue_agent_resource_collectionlogger = logging.getLogger(__name__)-class WorkflowAgentWorkspaceRetirementLayer(GraphEngineLayer):+class WorkflowAgentWorkspaceRetirementLayer(Layer):"""Synchronously retire run Workspaces, then enqueue physical collection."""_TERMINAL_EVENTS = (@@ -40,15 +40,11 @@ class WorkflowAgentWorkspaceRetirementLayer(GraphEngineLayer):self._dify_run_context = dify_run_context@override- def on_graph_start(self) -> None:- return-- @override- def on_event(self, event: GraphEngineEvent) -> None:+ def on_event(self, event: EngineEvent) -> None:if not isinstance(event, self._TERMINAL_EVENTS):returnworkflow_run_id = get_system_text(- self.graph_runtime_state.variable_pool,+ self.runtime_state.variable_pool,SystemVariableKey.WORKFLOW_EXECUTION_ID,)if not workflow_run_id:@@ -75,10 +71,6 @@ class WorkflowAgentWorkspaceRetirementLayer(GraphEngineLayer):workspace_ids=workspace_ids,)- @override- def on_graph_end(self, error: Exception | None) -> None:- return-def build_workflow_agent_workspace_retirement_layer(*, dify_run_context: DifyRunContextdiff --git a/api/core/workflow/nodes/datasource/datasource_node.py b/api/core/workflow/nodes/datasource/datasource_node.pyindex f90c06537aa..c4d6b023250 100644--- a/api/core/workflow/nodes/datasource/datasource_node.py+++ b/api/core/workflow/nodes/datasource/datasource_node.py@@ -21,8 +21,7 @@ from .entities import DatasourceNodeData, DatasourceParameter, OnlineDriveDownlofrom .exc import DatasourceNodeErrorif TYPE_CHECKING:- from graphon.entities import GraphInitParams- from graphon.runtime import GraphRuntimeState+ from graphon.runtime import InitParams, RuntimeStateclass DatasourceNode(Node[DatasourceNodeData]):@@ -38,15 +37,15 @@ class DatasourceNode(Node[DatasourceNodeData]):node_id: str,data: DatasourceNodeData,*,- graph_init_params: "GraphInitParams",- graph_runtime_state: "GraphRuntimeState",+ init_params: "InitParams",+ runtime_state: "RuntimeState",datasource_credentials: Callable[[str, str, str, str | None], dict[str, Any]],) -> None:super().__init__(node_id=node_id,data=data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)self.datasource_manager = DatasourceManagerself._datasource_credentials = datasource_credentials@@ -63,7 +62,7 @@ class DatasourceNode(Node[DatasourceNodeData]):"""dify_ctx = DifyRunContext.model_validate(self.require_run_context_value(DIFY_RUN_CONTEXT_KEY))node_data = self.node_data- variable_pool = self.graph_runtime_state.variable_pool+ variable_pool = self.runtime_state.variable_pooldatasource_type_segment = get_system_segment(variable_pool, SystemVariableKey.DATASOURCE_TYPE)if not datasource_type_segment:raise DatasourceNodeError("Datasource type is not set")diff --git a/api/core/workflow/nodes/human_input/boundary.py b/api/core/workflow/nodes/human_input/boundary.pyindex b15bdf9f74f..92e5669d69b 100644--- a/api/core/workflow/nodes/human_input/boundary.py+++ b/api/core/workflow/nodes/human_input/boundary.py@@ -1,29 +1,24 @@from __future__ import annotations-from collections.abc import Iterable, Sequence+from collections.abc import Iterable-from core.repositories.human_input_repository import HumanInputFormSubmissionRepository+from core.repositories.human_input_repository import HumanInputFormRecord, HumanInputFormSubmissionRepositoryfrom core.workflow.human_input_policy import resolve_variable_select_input_optionsfrom core.workflow.system_variables import SystemVariableKey, get_system_text-from graphon.entities.pause_reason import HitlRequired, SchedulingPause-from graphon.enums import BuiltinNodeTypes-from graphon.filters import GraphEventFilterContext-from graphon.graph_events import (- GraphEngineEvent,+from graphon.engine.filter import EngineEventFilterContext+from graphon.engine_events import (+ EngineEvent,NodeRunHumanInputFormFilledEvent,NodeRunHumanInputFormTimeoutEvent,NodeRunStartedEvent,NodeRunSucceededEvent,)-from graphon.runtime.graph_runtime_state_protocol import ReadOnlyVariablePool+from graphon.entities.pause_reason import HitlRequired+from graphon.enums import BuiltinNodeTypes+from graphon.runtime.runtime_state_protocol import ReadOnlyVariablePoolfrom .constants import OUTPUT_FIELD_ACTION_ID, OUTPUT_FIELD_ACTION_VALUE, OUTPUT_FIELD_RENDERED_CONTENT, TIMEOUT_HANDLE-from .pause_reason import HumanInputRequired, PauseReason-from .session_binding import default_session_binding---class HumanInputPauseReasonResolutionError(LookupError):- """Raised when a graph pause reason cannot be resolved into Dify-owned form state."""+from .pause_reason import HumanInputRequiredclass HumanInputFormEventFilter:@@ -38,11 +33,11 @@ class HumanInputFormEventFilter:def filter_id(self) -> str:return "dify-human-input-form-events"- def initialize(self, context: GraphEventFilterContext) -> None:+ def initialize(self, context: EngineEventFilterContext) -> None:self._node_titles.clear()self._app_id = get_system_text(context.runtime_state.variable_pool, SystemVariableKey.APP_ID)- def on_event(self, event: GraphEngineEvent) -> Iterable[GraphEngineEvent]:+ def on_event(self, event: EngineEvent) -> Iterable[EngineEvent]:if isinstance(event, NodeRunStartedEvent) and event.node_type == BuiltinNodeTypes.HUMAN_INPUT:self._node_titles[event.id] = event.node_titleelif isinstance(event, NodeRunSucceededEvent) and event.node_type == BuiltinNodeTypes.HUMAN_INPUT:@@ -50,7 +45,7 @@ class HumanInputFormEventFilter:yield event- def flush(self) -> Iterable[GraphEngineEvent]:+ def flush(self) -> Iterable[EngineEvent]:return ()def _completion_event(@@ -72,8 +67,8 @@ class HumanInputFormEventFilter:node_type=event.node_type,node_title=node_title,node_version=event.node_version,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ container_id=event.container_id,+ node_run_result=result,expiration_time=form.expiration_time,)@@ -83,8 +78,8 @@ class HumanInputFormEventFilter:node_type=event.node_type,node_title=node_title,node_version=event.node_version,- in_iteration_id=event.in_iteration_id,- in_loop_id=event.in_loop_id,+ container_id=event.container_id,+ node_run_result=result,rendered_content=result.outputs[OUTPUT_FIELD_RENDERED_CONTENT].text,action_id=result.outputs[OUTPUT_FIELD_ACTION_ID].text,action_text=result.outputs[OUTPUT_FIELD_ACTION_VALUE].text,@@ -92,47 +87,17 @@ class HumanInputFormEventFilter:)-def enrich_graph_pause_reasons(- *,- reasons: Sequence[HitlRequired | PauseReason],- form_repository: HumanInputFormSubmissionRepository,- variable_pool: ReadOnlyVariablePool | None,-) -> list[PauseReason]:- enriched: list[PauseReason] = []- for reason in reasons:- if isinstance(reason, HitlRequired):- enriched_reason = _enrich_hitl_required(- reason=reason,- form_repository=form_repository,- variable_pool=variable_pool,- )- if enriched_reason is not None:- enriched.append(enriched_reason)- continue- if isinstance(reason, HumanInputRequired | SchedulingPause):- enriched.append(reason)- return enriched---def _enrich_hitl_required(- *,- reason: HitlRequired,- form_repository: HumanInputFormSubmissionRepository,- variable_pool: ReadOnlyVariablePool | None,+def build_human_input_pause_reason(+ *, reason: HitlRequired, record: HumanInputFormRecord, variable_pool: ReadOnlyVariablePool | None) -> HumanInputRequired:- form_id = default_session_binding.resolve_form_id_from_session_id(session_id=reason.session_id)- record = form_repository.get_by_form_id(form_id)- if record is None:- raise HumanInputPauseReasonResolutionError(- f"missing human input form while enriching pause reason: form_id={form_id}, session_id={reason.session_id}"- )-+ """Format a materialized form without reading its repository."""+ definition = record.definitionreturn HumanInputRequired(form_id=record.form_id,form_content=record.rendered_content,- inputs=resolve_variable_select_input_options(record.definition.inputs, variable_pool=variable_pool),- actions=list(record.definition.user_actions),+ inputs=resolve_variable_select_input_options(definition.inputs, variable_pool=variable_pool),+ actions=list(definition.user_actions),node_id=reason.node_id,- node_title=reason.node_title or record.definition.node_title or record.node_id,- resolved_default_values=dict(record.definition.default_values),+ node_title=reason.node_title or definition.node_title or record.node_id,+ resolved_default_values=dict(definition.default_values),)diff --git a/api/core/workflow/nodes/human_input/callback.py b/api/core/workflow/nodes/human_input/callback.pyindex 17b26f3cde6..820857c8e78 100644--- a/api/core/workflow/nodes/human_input/callback.py+++ b/api/core/workflow/nodes/human_input/callback.py@@ -9,9 +9,10 @@ from typing import Anyfrom configs import dify_configfrom core.repositories.human_input_repository import FormCreateParams, HumanInputFormEntity, HumanInputFormRepositoryfrom core.workflow.human_input_adapter import DeliveryChannelConfig+from core.workflow.human_input_policy import resolve_variable_select_input_optionsfrom core.workflow.node_runtime import DifyFileReferenceFactoryfrom graphon.nodes.human_input.entities import Completed, Expired, HITLContext, HITLDecision, PauseRequested-from graphon.runtime.graph_runtime_state_protocol import ReadOnlyVariablePool+from graphon.runtime.runtime_state_protocol import ReadOnlyVariablePoolfrom graphon.variables.factory import build_segmentfrom graphon.variables.segments import Segmentfrom graphon.variables.template_resolution import convert_template@@ -100,6 +101,14 @@ class DifyHITLCallback:return PauseRequested(session_id=self._session_binding.issue_session_id_for_form(form_id=created.id))status = self._normalize_status(form.status)+ if status == HumanInputFormStatus.WAITING.value and not form.submitted:+ if self._is_past_global_deadline(form.created_at):+ msg = f"cannot resume waiting human input form after global timeout, form_id={form.id}"+ raise AssertionError(msg)+ if self._is_past_node_deadline(form.expiration_time):+ form = self._form_repository.mark_timeout(ctx.node_id, form_id=form.id)+ status = self._normalize_status(form.status)+if status == HumanInputFormStatus.TIMEOUT.value:return Expired(selected_handle=TIMEOUT_HANDLE,@@ -115,18 +124,6 @@ class DifyHITLCallback:raise AssertionError(msg)if not form.submitted:- if status == HumanInputFormStatus.WAITING.value and self._is_past_global_deadline(form.created_at):- msg = f"cannot resume waiting human input form after global timeout, form_id={form.id}"- raise AssertionError(msg)- if self._is_past_node_deadline(form.expiration_time):- return Expired(- selected_handle=TIMEOUT_HANDLE,- outputs=self._build_special_outputs(- action_id="",- action_value="",- rendered_content=form.rendered_content,- ),- )return PauseRequested(session_id=self._session_binding.issue_session_id_for_form(form_id=form.id))selected_action_id = form.selected_action_id@@ -156,11 +153,19 @@ class DifyHITLCallback:)def _create_form(self, ctx: HITLContext, *, form_id: str | None = None) -> HumanInputFormEntity:+ form_config = self._node_data.model_copy(+ update={+ "inputs": resolve_variable_select_input_options(+ self._node_data.inputs,+ variable_pool=ctx.variable_pool,+ )+ }+ )params = FormCreateParams(workflow_execution_id=self._workflow_execution_id or ctx.workflow_execution_id,conversation_id=self._conversation_id,node_id=ctx.node_id,- form_config=self._node_data,+ form_config=form_config,rendered_content=render_form_content_before_submission(self._node_data,variable_pool=ctx.variable_pool,diff --git a/api/core/workflow/nodes/human_input/entities.py b/api/core/workflow/nodes/human_input/entities.pyindex 8324a46bf4f..115093bd71b 100644--- a/api/core/workflow/nodes/human_input/entities.py+++ b/api/core/workflow/nodes/human_input/entities.py@@ -16,8 +16,7 @@ from pydantic import BaseModel, Field, NonNegativeInt, field_validator, model_vafrom graphon.entities.base_node_data import BaseNodeDatafrom graphon.enums import BuiltinNodeTypes, NodeTypefrom graphon.file.enums import FileTransferMethod, FileType-from graphon.nodes.base.variable_template_parser import VariableTemplateParser-from graphon.runtime.graph_runtime_state_protocol import ReadOnlyVariablePool+from graphon.runtime.runtime_state_protocol import ReadOnlyVariablePoolfrom graphon.variables.consts import SELECTORS_LENGTHfrom graphon.variables.segments import Segment@@ -292,35 +291,6 @@ class HumanInputNodeData(BaseNodeData):def outputs_field_names(self) -> Sequence[str]:return [match.group("field_name") for match in _OUTPUT_VARIABLE_PATTERN.finditer(self.form_content)]- def extract_variable_selector_to_variable_mapping(- self,- node_id: str,- ) -> Mapping[str, Sequence[str]]:- variable_mappings: dict[str, Sequence[str]] = {}-- def _add_variable_selectors(selectors: Sequence[Sequence[str]]) -> None:- for selector in selectors:- if len(selector) < SELECTORS_LENGTH:- continue- qualified_variable_mapping_key = f"{node_id}.#{'.'.join(selector[:SELECTORS_LENGTH])}#"- variable_mappings[qualified_variable_mapping_key] = list(- selector[:SELECTORS_LENGTH],- )-- form_template_parser = VariableTemplateParser(template=self.form_content)- _add_variable_selectors(- [selector.value_selector for selector in form_template_parser.extract_variable_selectors()]- )-- for form_input in self.inputs:- selectors = form_input.extract_variable_selectors()- for selector in selectors:- value_key = ".".join(selector)- qualified_variable_mapping_key = f"{node_id}.#{value_key}#"- variable_mappings[qualified_variable_mapping_key] = selector-- return variable_mappings-def find_action_text(self, action_id: str) -> str:"""Resolve action display text by id."""for action in self.user_actions:diff --git a/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py b/api/core/workflow/nodes/knowledge_index/knowledge_index_node.pyindex 944a6bb9109..0e16b6f0734 100644--- a/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py+++ b/api/core/workflow/nodes/knowledge_index/knowledge_index_node.py@@ -23,8 +23,7 @@ from .exc import ()if TYPE_CHECKING:- from graphon.entities import GraphInitParams- from graphon.runtime import GraphRuntimeState+ from graphon.runtime import InitParams, RuntimeStatelogger = logging.getLogger(__name__)_INVOKE_FROM_DEBUGGER = "debugger"@@ -39,14 +38,14 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):node_id: str,data: KnowledgeIndexNodeData,*,- graph_init_params: "GraphInitParams",- graph_runtime_state: "GraphRuntimeState",+ init_params: "InitParams",+ runtime_state: "RuntimeState",) -> None:super().__init__(node_id=node_id,data=data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)self.index_processor = IndexProcessor()self.summary_index_service = SummaryIndex()@@ -54,7 +53,7 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):@overridedef _run(self) -> NodeRunResult:node_data = self.node_data- variable_pool = self.graph_runtime_state.variable_pool+ variable_pool = self.runtime_state.variable_pool# get dataset id as stringdataset_id_segment = get_system_segment(variable_pool, SystemVariableKey.DATASET_ID)diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.pyindex a3547fbcf76..ae84c7856d3 100644--- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py+++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py@@ -17,7 +17,6 @@ from core.db.session_factory import session_factoryfrom core.rag.data_post_processor.data_post_processor import RerankingModelDict, WeightsDictfrom core.rag.retrieval.dataset_retrieval import DatasetRetrievalfrom core.workflow.file_reference import parse_file_reference-from graphon.entities import GraphInitParamsfrom graphon.enums import (BuiltinNodeTypes,WorkflowNodeExecutionMetadataKey,@@ -27,6 +26,7 @@ from graphon.model_runtime.entities.llm_entities import LLMUsagefrom graphon.model_runtime.utils.encoders import jsonable_encoderfrom graphon.node_events import NodeRunResultfrom graphon.nodes.base.node import Node+from graphon.runtime import InitParamsfrom graphon.variables import (ArrayFileSegment,FileSegment,@@ -48,7 +48,7 @@ from .retrieval import KnowledgeRetrievalRequest, Sourceif TYPE_CHECKING:from graphon.file import File- from graphon.runtime import GraphRuntimeState+ from graphon.runtime import RuntimeStatelogger = logging.getLogger(__name__)@@ -77,15 +77,15 @@ class KnowledgeRetrievalNode(Node[KnowledgeRetrievalNodeData]):node_id: str,data: KnowledgeRetrievalNodeData,*,- graph_init_params: "GraphInitParams",- graph_runtime_state: "GraphRuntimeState",+ init_params: "InitParams",+ runtime_state: "RuntimeState",session_maker=None,) -> None:super().__init__(node_id=node_id,data=data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)# LLM file outputs, used for MultiModal outputs.self._file_outputs = []@@ -112,7 +112,7 @@ class KnowledgeRetrievalNode(Node[KnowledgeRetrievalNodeData]):variables: dict[str, Any] = {}# extract variablesif self._node_data.query_variable_selector:- variable = self.graph_runtime_state.variable_pool.get(self._node_data.query_variable_selector)+ variable = self.runtime_state.variable_pool.get(self._node_data.query_variable_selector)if not isinstance(variable, StringSegment):return NodeRunResult(status=WorkflowNodeExecutionStatus.FAILED,@@ -123,7 +123,7 @@ class KnowledgeRetrievalNode(Node[KnowledgeRetrievalNodeData]):variables["query"] = queryif self._node_data.query_attachment_selector:- variable = self.graph_runtime_state.variable_pool.get(self._node_data.query_attachment_selector)+ variable = self.runtime_state.variable_pool.get(self._node_data.query_attachment_selector)if not isinstance(variable, ArrayFileSegment) and not isinstance(variable, FileSegment):return NodeRunResult(status=WorkflowNodeExecutionStatus.FAILED,@@ -309,7 +309,7 @@ class KnowledgeRetrievalNode(Node[KnowledgeRetrievalNodeData]):conditions=None,)- variable_pool = self.graph_runtime_state.variable_pool+ variable_pool = self.runtime_state.variable_poolresolved_conditions: list[Condition] = []for cond in conditions.conditions or []:value = cond.valuediff --git a/api/core/workflow/nodes/trigger_plugin/trigger_event_node.py b/api/core/workflow/nodes/trigger_plugin/trigger_event_node.pyindex a3641c34f32..571b5fbf9a8 100644--- a/api/core/workflow/nodes/trigger_plugin/trigger_event_node.py+++ b/api/core/workflow/nodes/trigger_plugin/trigger_event_node.py@@ -56,8 +56,8 @@ class TriggerEventNode(Node[TriggerEventNodeData]):"plugin_unique_identifier": self.node_data.plugin_unique_identifier,},}- node_inputs = dict(self.graph_runtime_state.variable_pool.get_by_prefix(self.id))- system_inputs = self.graph_runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID)+ node_inputs = dict(self.runtime_state.variable_pool.get_by_prefix(self.id))+ system_inputs = self.runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID)for variable_name, value in system_inputs.items():node_inputs[f"{SYSTEM_VARIABLE_NODE_ID}.{variable_name}"] = valuediff --git a/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.py b/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.pyindex d2cdf96d157..7add30f4be7 100644--- a/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.py+++ b/api/core/workflow/nodes/trigger_schedule/trigger_schedule_node.py@@ -34,8 +34,8 @@ class TriggerScheduleNode(Node[TriggerScheduleNodeData]):@overridedef _run(self) -> NodeRunResult:- node_inputs = dict(self.graph_runtime_state.variable_pool.get_by_prefix(self.id))- system_inputs = self.graph_runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID)+ node_inputs = dict(self.runtime_state.variable_pool.get_by_prefix(self.id))+ system_inputs = self.runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID)for variable_name, value in system_inputs.items():node_inputs[f"{SYSTEM_VARIABLE_NODE_ID}.{variable_name}"] = valuediff --git a/api/core/workflow/nodes/trigger_webhook/node.py b/api/core/workflow/nodes/trigger_webhook/node.pyindex 64180275707..b4b1c956b73 100644--- a/api/core/workflow/nodes/trigger_webhook/node.py+++ b/api/core/workflow/nodes/trigger_webhook/node.py@@ -64,11 +64,11 @@ class TriggerWebhookNode(Node[WebhookData]):happens in the trigger controller."""# Get webhook data from variable pool (injected by Celery task)- webhook_inputs = dict(self.graph_runtime_state.variable_pool.get_by_prefix(self.id))+ webhook_inputs = dict(self.runtime_state.variable_pool.get_by_prefix(self.id))# Extract webhook-specific outputs based on node configurationoutputs = self._extract_configured_outputs(webhook_inputs)- system_inputs = self.graph_runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID)+ system_inputs = self.runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID)for variable_name, value in system_inputs.items():outputs[f"{SYSTEM_VARIABLE_NODE_ID}.{variable_name}"] = valuediff --git a/api/core/workflow/variable_pool_initializer.py b/api/core/workflow/variable_pool_initializer.pyindex 5ec31176d2e..04d34eb1cb6 100644--- a/api/core/workflow/variable_pool_initializer.py+++ b/api/core/workflow/variable_pool_initializer.py@@ -2,10 +2,10 @@ from collections.abc import Mapping, Sequencefrom typing import Anyfrom graphon.runtime import VariablePool-from graphon.variables.variables import Variable+from graphon.variables.variables import VariableBase-def add_variables_to_pool(variable_pool: VariablePool, variables: Sequence[Variable]) -> None:+def add_variables_to_pool(variable_pool: VariablePool, variables: Sequence[VariableBase]) -> None:for variable in variables:variable_pool.add(variable.selector, variable)diff --git a/api/core/workflow/workflow_entry.py b/api/core/workflow/workflow_entry.pyindex f919a28aaf9..b510f86136b 100644--- a/api/core/workflow/workflow_entry.py+++ b/api/core/workflow/workflow_entry.py@@ -1,11 +1,12 @@import loggingimport timefrom collections.abc import Generator, Mapping, Sequence-from typing import Any, TypedDict+from contextlib import ExitStack+from contextvars import copy_context+from typing import TYPE_CHECKING, Any, TypedDictfrom uuid import uuid4from configs import dify_config-from context import capture_current_contextfrom core.app.apps.exc import GenerateTaskStoppedErrorfrom core.app.entities.app_invoke_entities import (InvokeFrom,@@ -13,6 +14,8 @@ from core.app.entities.app_invoke_entities import (build_dify_run_context,)from core.app.file_access import DatabaseFileAccessController+from core.app.layers.execution_context_layer import ExecutionContextLayer+from core.app.workflow.file_runtime import create_dify_workflow_file_runtimefrom core.app.workflow.layers.observability import ObservabilityLayerfrom core.credit_usage import CreditUsageAppTypefrom core.repositories.human_input_repository import HumanInputFormSubmissionRepository@@ -33,45 +36,50 @@ from core.workflow.variable_pool_initializer import add_node_inputs_to_pool, addfrom core.workflow.variable_prefixes import ENVIRONMENT_VARIABLE_NODE_IDfrom extensions.otel.runtime import is_instrument_flag_enabledfrom factories import file_factory+from graphon.engine import Engine+from graphon.engine.command import CommandChannel, InMemoryChannel+from graphon.engine.filter import EngineEventFilterContext, ResponseStreamFilter, filter_engine_events+from graphon.engine.layer import ExecutionLimitsLayer, Layer+from graphon.engine_events import EngineEvent, GraphRunFailedEvent, NodeEvent, is_node_result_eventfrom graphon.entities.graph_config import NodeConfigDictAdapterfrom graphon.errors import WorkflowNodeRunFailedErrorfrom graphon.file import File-from graphon.filters import GraphEventFilterContext, ResponseStreamFilter, filter_graph_events+from graphon.file.runtime import use_workflow_file_runtimefrom graphon.graph import Graph-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import CommandChannel, InMemoryChannel-from graphon.graph_engine.layers import DebugLoggingLayer, ExecutionLimitsLayer, GraphEngineLayer-from graphon.graph_events import GraphEngineEvent, GraphNodeEventBase, GraphRunFailedEvent, is_node_result_eventfrom graphon.nodes import BuiltinNodeTypesfrom graphon.nodes.base.node import Nodefrom graphon.nodes.container_effects import ContainerAwaitRequest-from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool+from graphon.runtime import ReadOnlyRuntimeStateWrapper, RuntimeState, VariablePoolfrom graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader, load_into_variable_poolfrom models.workflow import Workflow+from services.workflow_node_variables import add_node_input_to_variable_pool, load_additional_node_variables++if TYPE_CHECKING:+ from core.app.apps.workflow_app_runner import WorkflowRunDriverlogger = logging.getLogger(__name__)_file_access_controller = DatabaseFileAccessController()def iter_dify_graph_engine_events(- engine: GraphEngine,+ engine: Engine,response_stream_filter: ResponseStreamFilter | None = None,-) -> Generator[GraphEngineEvent]:+) -> Generator[EngineEvent]:"""- Apply Dify's response streaming compatibility filter to GraphEngine events.+ Apply Dify's response streaming compatibility filter to Engine events.- Graphon v0.5.0 emits raw variable stream chunks and requires callers to opt+ Graphon emits raw variable stream chunks and requires callers to optinto the legacy response-ordered stream behavior that Dify exposes to itsworkflow runners and tests.``response_stream_filter``, when supplied, must be the same instance a- caller intends to persist on pause (see ``PauseStatePersistenceLayer``) so+ caller intends to persist on pause (see ``WorkflowRunAgg``) sothe filter's ``paths_map`` reflects everything the engine has actuallystreamed for this run."""- yield from filter_graph_events(+ yield from filter_engine_events(engine.run(),- context=GraphEventFilterContext.from_engine(engine),+ context=EngineEventFilterContext.from_engine(engine),filters=[HumanInputFormEventFilter(form_repository=HumanInputFormSubmissionRepository()),response_stream_filter or ResponseStreamFilter(),@@ -102,38 +110,22 @@ class SingleNodeGraphDict(TypedDict):class WorkflowEntry:def __init__(self,- tenant_id: str,- app_id: str,- workflow_id: str,graph_config: Mapping[str, Any],graph: Graph,- user_id: str,- user_from: UserFrom,- invoke_from: InvokeFrom,call_depth: int,- variable_pool: VariablePool,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,command_channel: CommandChannel | None = None,response_stream_filter: ResponseStreamFilter | None = None,) -> None:"""Init workflow entry- :param tenant_id: tenant id- :param app_id: app id- :param workflow_id: workflow id- :param workflow_type: workflow type:param graph_config: workflow graph config:param graph: workflow graph- :param user_id: user id- :param user_from: user from- :param invoke_from: invoke from:param call_depth: call depth- :param variable_pool: variable pool:param graph_runtime_state: pre-created graph runtime state:param command_channel: command channel for external control (optional, defaults to InMemoryChannel):param response_stream_filter: pre-restored filter for resumed runs (optional, defaults to a freshResponseStreamFilter for runs with no prior pause)- :param thread_pool_id: thread pool id"""# check call depthworkflow_call_max_depth = dify_config.WORKFLOW_CALL_MAX_DEPTH@@ -145,52 +137,36 @@ class WorkflowEntry:command_channel = InMemoryChannel()self.command_channel = command_channel+ self.graph_config = graph_configself._response_stream_filter = response_stream_filter or ResponseStreamFilter()- execution_context = capture_current_context()- # ponytail: Graphon snapshots omit process-local context; use a public rebind API when Graphon exposes one.- graph_runtime_state._execution_context = execution_context- self.graph_engine = GraphEngine(- workflow_id=workflow_id,+ file_runtime = create_dify_workflow_file_runtime()+ execution_context_layer = ExecutionContextLayer()+ limits_layer = ExecutionLimitsLayer(+ max_steps=dify_config.WORKFLOW_MAX_EXECUTION_STEPS, max_time=dify_config.WORKFLOW_MAX_EXECUTION_TIME+ )+ self.graph_engine = Engine(graph=graph,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,command_channel=command_channel,- config=GraphEngineConfig(- min_workers=dify_config.GRAPH_ENGINE_MIN_WORKERS,- max_workers=dify_config.GRAPH_ENGINE_MAX_WORKERS,- scale_up_threshold=dify_config.GRAPH_ENGINE_SCALE_UP_THRESHOLD,- scale_down_idle_time=dify_config.GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME,- ),+ workers=dify_config.GRAPH_ENGINE_MAX_WORKERS,+ file_runtime=file_runtime,)- # Add debug logging layer when in debug mode- if dify_config.DEBUG:- logger.info("Debug mode enabled - adding DebugLoggingLayer to GraphEngine")- debug_layer = DebugLoggingLayer(- level="DEBUG",- include_inputs=True,- include_outputs=True,- include_process_data=False, # Process data can be very verbose- logger_name=f"GraphEngine.Debug.{workflow_id[:8]}", # Use workflow ID prefix for unique logger- )- self.graph_engine.layer(debug_layer)-+ self.graph_engine.add_layer(execution_context_layer)# Add execution limits layer- limits_layer = ExecutionLimitsLayer(- max_steps=dify_config.WORKFLOW_MAX_EXECUTION_STEPS, max_time=dify_config.WORKFLOW_MAX_EXECUTION_TIME- )- self.graph_engine.layer(limits_layer)+ self.graph_engine.add_layer(limits_layer)# Add observability layer when OTel is enabledif dify_config.ENABLE_OTEL or is_instrument_flag_enabled():- self.graph_engine.layer(ObservabilityLayer())+ self.graph_engine.add_layer(ObservabilityLayer())- def run(self) -> Generator[GraphEngineEvent]:- graph_engine = self.graph_engine+ @property+ def response_stream_filter(self) -> ResponseStreamFilter:+ return self._response_stream_filter+ def run(self) -> Generator[EngineEvent]:try:- # Preserve Dify's response-stream semantics on top of Graphon 0.5.0.- generator = iter_dify_graph_engine_events(graph_engine, self._response_stream_filter)- yield from generator+ yield from iter_dify_graph_engine_events(self.graph_engine, self._response_stream_filter)except GenerateTaskStoppedError:passexcept Exception as e:@@ -208,7 +184,8 @@ class WorkflowEntry:user_inputs: Mapping[str, Any],variable_pool: VariablePool,variable_loader: VariableLoader = DUMMY_VARIABLE_LOADER,- ) -> tuple[Node, Generator[GraphNodeEventBase | ContainerAwaitRequest]]:+ execution_driver: "WorkflowRunDriver | None",+ ) -> tuple[Node, Generator[NodeEvent | ContainerAwaitRequest]]:"""Single step run workflow node:param workflow: Workflow instance@@ -245,11 +222,12 @@ class WorkflowEntry:graph_config=workflow.graph_dict,run_context=run_context,call_depth=0,+ execution_driver=execution_driver,)- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(variable_pool=variable_pool,start_at=time.perf_counter(),- execution_context=capture_current_context(),+ workflow_id=workflow.id,)if is_start_node_type(node_type):@@ -286,6 +264,14 @@ class WorkflowEntry:variable_mapping=variable_mapping,user_inputs=user_inputs,)+ variable_mapping = load_additional_node_variables(+ node_config=node_config,+ graph_config=workflow.graph_dict,+ variable_mapping=variable_mapping,+ variable_loader=variable_loader,+ variable_pool=variable_pool,+ user_inputs=user_inputs,+ )if node_type != BuiltinNodeTypes.DATASOURCE:cls.mapping_user_inputs_to_variable_pool(variable_mapping=variable_mapping,@@ -297,12 +283,12 @@ class WorkflowEntry:# init workflow run statenode_factory = DifyNodeFactory.from_graph_init_context(graph_init_context=graph_init_context,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)node = node_factory.create_node(node_config)try:- generator = cls._run_node_with_layers(node, tenant_id=workflow.tenant_id)+ generator = cls._run_node_with_layers(node)except Exception as e:logger.exception("error while running node, workflow_id=%s, node_id=%s, node_type=%s, node_version=%s",@@ -363,7 +349,7 @@ class WorkflowEntry:@classmethoddef run_free_node(cls, node_data: dict[str, Any], node_id: str, tenant_id: str, user_id: str, user_inputs: dict[str, Any]- ) -> tuple[Node, Generator[GraphNodeEventBase | ContainerAwaitRequest]]:+ ) -> tuple[Node, Generator[NodeEvent | ContainerAwaitRequest]]:"""Run free node@@ -406,17 +392,17 @@ class WorkflowEntry:run_context=run_context,call_depth=0,)- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(variable_pool=variable_pool,start_at=time.perf_counter(),- execution_context=capture_current_context(),+ workflow_id="",)# init workflow run statenode_config = NodeConfigDictAdapter.validate_python({"id": node_id, "data": node_data})node_factory = DifyNodeFactory.from_graph_init_context(graph_init_context=graph_init_context,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)node = node_factory.create_node(node_config)@@ -442,7 +428,7 @@ class WorkflowEntry:tenant_id=tenant_id,)- generator = cls._run_node_with_layers(node, tenant_id=tenant_id)+ generator = cls._run_node_with_layers(node)return node, generatorexcept Exception as e:@@ -497,7 +483,8 @@ class WorkflowEntry:# WARNING(QuantumGhost): The semantics of this method are not clearly defined,# and multiple parts of the codebase depend on its current behavior.# Modify with caution.- for node_variable, variable_selector in variable_mapping.items():+ # Apply whole objects before nested overrides, independent of editor field order.+ for node_variable, variable_selector in sorted(variable_mapping.items(), key=lambda item: len(item[1])):# fetch node id and variable key from node_variablenode_variable_list = node_variable.split(".")if len(node_variable_list) < 1:@@ -516,13 +503,11 @@ class WorkflowEntry:# fetch variable node id from variable selectorvariable_node_id = variable_selector[0]- variable_key_list = variable_selector[1:]- variable_key_list = list(variable_key_list)# get input value- input_value = user_inputs.get(node_variable)- if not input_value:- input_value = user_inputs.get(node_variable_key)+ input_value = (+ user_inputs[node_variable] if node_variable in user_inputs else user_inputs.get(node_variable_key)+ )if input_value is None:continue@@ -545,36 +530,32 @@ class WorkflowEntry:# append variable and value to variable poolif variable_node_id != ENVIRONMENT_VARIABLE_NODE_ID:- # In single run, the input_value is set as the LLM's structured output value within the variable_pool.- if len(variable_key_list) == 2 and variable_key_list[0] == "structured_output":- input_value = {variable_key_list[1]: input_value}- variable_key_list = variable_key_list[0:1]-- # Support for a single node to reference multiple structured_output variables- current_variable = variable_pool.get([variable_node_id] + variable_key_list)- if current_variable and isinstance(current_variable.value, dict):- input_value = current_variable.value | input_value-- variable_pool.add([variable_node_id] + variable_key_list, input_value)+ add_node_input_to_variable_pool(variable_pool, variable_selector, input_value)@staticmethod- def _run_node_with_layers(node: Node, *, tenant_id: str) -> Generator[GraphNodeEventBase | ContainerAwaitRequest]:+ def _run_node_with_layers(node: Node) -> Generator[NodeEvent | ContainerAwaitRequest]:"""- Run a standalone node with the same quota and observability hooks as GraphEngine.+ Run a standalone node with the execution-context and observability layers used by workflow runs."""- layers: Sequence[GraphEngineLayer] = (ObservabilityLayer(),)+ file_runtime = create_dify_workflow_file_runtime()command_channel = InMemoryChannel()- runtime_state = ReadOnlyGraphRuntimeStateWrapper(node.graph_runtime_state)- for layer in layers:- layer.initialize(runtime_state, command_channel)- layer.on_graph_start()+ runtime_state = ReadOnlyRuntimeStateWrapper(node.runtime_state)+ with use_workflow_file_runtime(file_runtime):+ layers: Sequence[Layer] = (ExecutionContextLayer(), ObservabilityLayer())+ for layer in layers:+ layer.initialize(runtime_state, command_channel)+ layer.on_graph_start()+ # Keep Flask and tracing scopes inside the snippet across suspension,+ # without exposing them to callers interleaving multiple streams.+ snippet_context = copy_context()node.bind_execution_id(str(uuid4()))def _gen():error: Exception | None = None- result_event: GraphNodeEventBase | None = None+ result_event: NodeEvent | None = Nonelayers_finished = False+ node_contexts = ExitStack()def finish_layers() -> None:nonlocal layers_finished@@ -583,14 +564,17 @@ class WorkflowEntry:layers_finished = Truefor layer in layers:layer.on_node_run_end(node, error, result_event)+ node_contexts.close()for layer in layers:layer.on_graph_end(error)try:+ for layer in layers:+ node_contexts.enter_context(layer.node_run_context(node))for layer in layers:layer.on_node_run_start(node)for event in node.run():- if isinstance(event, GraphNodeEventBase) and is_node_result_event(event):+ if isinstance(event, NodeEvent) and is_node_result_event(event):result_event = eventfinish_layers()yield event@@ -600,4 +584,26 @@ class WorkflowEntry:finally:finish_layers()- return _gen()+ def scoped_events():+ events = _gen()+ error: BaseException | None = None+ try:+ while True:+ try:+ event = (+ snippet_context.run(next, events)+ if error is None+ else snippet_context.run(events.throw, error)+ )+ except StopIteration:+ return+ try:+ yield event+ except BaseException as exc:+ error = exc+ else:+ error = None+ finally:+ snippet_context.run(events.close)++ return scoped_events()diff --git a/api/dify_app.py b/api/dify_app.pyindex bbe3f33787a..bb8492e598c 100644--- a/api/dify_app.py+++ b/api/dify_app.py@@ -1,8 +1,12 @@from __future__ import annotations-from typing import TYPE_CHECKING+from contextlib import AbstractContextManager, nullcontext+from typing import TYPE_CHECKING, Any, overridefrom flask import Flask+from flask.ctx import AppContext++from graphon.file.runtime import use_workflow_file_runtimeif TYPE_CHECKING:from extensions.ext_login import DifyLoginManager@@ -12,3 +16,31 @@ class DifyApp(Flask):"""Flask application type with Dify-specific extension attributes."""login_manager: DifyLoginManager++ @override+ def app_context(self) -> AppContext:+ return DifyAppContext(self)+++class DifyAppContext(AppContext):+ """Keep the file adapter available through Flask teardown, including failures."""++ def __init__(self, app: DifyApp) -> None:+ super().__init__(app)+ self._file_scopes: list[AbstractContextManager[None]] = []++ @override+ def push(self) -> None:+ super().push()+ runtime = self.app.extensions.get("workflow_file_runtime")+ scope = use_workflow_file_runtime(runtime) if runtime is not None else nullcontext()+ scope.__enter__()+ self._file_scopes.append(scope)++ @override+ def pop(self, *args: Any, **kwargs: Any) -> None:+ # Forward omitted arguments unchanged so Flask can infer the active exception.+ try:+ super().pop(*args, **kwargs)+ finally:+ self._file_scopes.pop().__exit__(None, None, None)diff --git a/api/enums/__init__.py b/api/enums/__init__.pyindex fdcc5e94795..d53d9c1aefa 100644--- a/api/enums/__init__.py+++ b/api/enums/__init__.py@@ -1,6 +1,18 @@from enum import StrEnum, auto+class WorkflowKind(StrEnum):+ STANDARD = "standard"+ SNIPPET = "snippet"++ @classmethod+ def value_of(cls, value: str) -> "WorkflowKind":+ for kind in cls:+ if kind.value == value:+ return kind+ raise ValueError(f"invalid workflow kind value {value}")++class CloudPlan(StrEnum):"""Enum representing user plan types in the cloud platform.diff --git a/api/extensions/application_services/knowledge.py b/api/extensions/application_services/knowledge.pyindex ee377d87ae4..79227cd4c33 100644--- a/api/extensions/application_services/knowledge.py+++ b/api/extensions/application_services/knowledge.py@@ -52,6 +52,7 @@ from services.knowledge.segments.application import DatasetSegmentApplicationSerfrom services.knowledge.segments.indexing import SegmentIndexingGatewayfrom services.rbac.members import MemberServicefrom services.tag_application_service import TagTargetQuery+from services.workflow_run_agg import WorkflowRunAggfrom tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_taskfrom tasks.delete_segment_from_index_task import delete_segment_from_index_taskfrom tasks.disable_segments_from_index_task import disable_segments_from_index_task@@ -147,7 +148,9 @@ def build_knowledge_services(operations=SQLAlchemyDocumentOperations(session_factory=database_client),metadata_schema=DocumentService.DOCUMENT_METADATA_SCHEMA,),- pipeline_generator=PipelineGenerator(documents=documents, datasource_providers=providers),+ pipeline_generator=PipelineGenerator(+ execution_driver=WorkflowRunAgg.run, documents=documents, datasource_providers=providers+ ),document_sync=DocumentSyncApplicationService(dataset_access=dataset_access,documents=documents,diff --git a/api/extensions/ext_storage.py b/api/extensions/ext_storage.pyindex a59e7118d51..82f500f3c1c 100644--- a/api/extensions/ext_storage.py+++ b/api/extensions/ext_storage.py@@ -141,6 +141,6 @@ storage = Storage()def init_app(app: DifyApp):storage.init_app(app)- from core.app.workflow.file_runtime import bind_dify_workflow_file_runtime+ from core.app.workflow import file_runtime- bind_dify_workflow_file_runtime()+ file_runtime.init_app(app)diff --git a/api/extensions/logstore/repositories/logstore_api_workflow_node_execution_repository.py b/api/extensions/logstore/repositories/logstore_api_workflow_node_execution_repository.pyindex 846429f8c02..3371bccb5a2 100644--- a/api/extensions/logstore/repositories/logstore_api_workflow_node_execution_repository.py+++ b/api/extensions/logstore/repositories/logstore_api_workflow_node_execution_repository.py@@ -21,6 +21,7 @@ from libs.datetime_utils import ensure_naive_utc, naive_utc_nowfrom models.enums import CreatorUserRolefrom models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFromfrom repositories.api_workflow_node_execution_repository import DifyAPIWorkflowNodeExecutionRepository+from repositories.workflow_run_control import apply_workflow_stop_to_node, get_stopped_workflow_runlogger = logging.getLogger(__name__)@@ -102,7 +103,7 @@ def _dict_to_workflow_node_execution_model(data: dict[str, Any]) -> WorkflowNodefinished_at = data.get("finished_at")match finished_at:- case None:+ case None | "":...case str():model.finished_at = ensure_naive_utc(datetime.fromisoformat(finished_at))@@ -131,6 +132,17 @@ class LogstoreAPIWorkflowNodeExecutionRepository(DifyAPIWorkflowNodeExecutionRep"""logger.debug("LogstoreAPIWorkflowNodeExecutionRepository.__init__: initializing")self.logstore_client = AliyunLogStore()+ self._session_maker = session_maker++ def _apply_stopped_run(+ self, executions: Sequence[WorkflowNodeExecutionModel], tenant_id: str, workflow_run_id: str+ ) -> None:+ if self._session_maker is None:+ return+ with self._session_maker() as session:+ stopped = get_stopped_workflow_run(session, tenant_id=tenant_id, workflow_run_id=workflow_run_id)+ for execution in executions:+ apply_workflow_stop_to_node(execution, stopped)@overridedef load_full_process_data(self, execution: WorkflowNodeExecutionModel) -> Mapping[str, Any] | None:@@ -334,6 +346,7 @@ class LogstoreAPIWorkflowNodeExecutionRepository(DifyAPIWorkflowNodeExecutionRepif model and model.id: # Ensure model is validmodels.append(model)+ self._apply_stopped_run(models, tenant_id, workflow_run_id)models = [model for model in models if model.status != WorkflowNodeExecutionStatus.PAUSED]# Sort by index DESC for trace visualization@@ -411,10 +424,13 @@ class LogstoreAPIWorkflowNodeExecutionRepository(DifyAPIWorkflowNodeExecutionRep# For PG mode, result is already the latest version# For SDK mode, if multiple results, select the one with max log_versionif self.logstore_client.supports_pg_protocol or len(results) == 1:- return _dict_to_workflow_node_execution_model(results[0])+ execution = _dict_to_workflow_node_execution_model(results[0])else:max_result = max(results, key=lambda x: int(x.get("log_version", 0)))- return _dict_to_workflow_node_execution_model(max_result)+ execution = _dict_to_workflow_node_execution_model(max_result)+ if execution.workflow_run_id is not None:+ self._apply_stopped_run([execution], execution.tenant_id, execution.workflow_run_id)+ return executionexcept Exception:logger.exception("Failed to get execution by ID from LogStore: execution_id=%s", execution_id)diff --git a/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py b/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.pyindex 59f4836f7f7..baa6402c9c4 100644--- a/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py+++ b/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py@@ -16,21 +16,25 @@ Key Features:import loggingimport osimport time-from collections.abc import Iterable, Mapping, Sequence+from collections.abc import Callable, Iterable, Mapping, Sequencefrom datetime import UTC, datetimefrom typing import Any, cast, overridefrom sqlalchemy.orm import sessionmaker+from core.workflow.nodes.human_input.pause_reason import PauseReason as DifyPauseReasonfrom extensions.logstore.aliyun_logstore import AliyunLogStorefrom extensions.logstore.repositories import safe_float, safe_intfrom extensions.logstore.sql_escape import escape_identifier, escape_logstore_query_value, escape_sql_string+from graphon.entities.pause_reason import PauseReason as GraphonPauseReasonfrom graphon.enums import WorkflowExecutionStatusfrom libs.datetime_utils import ensure_naive_utc, naive_utc_nowfrom libs.infinite_scroll_pagination import InfiniteScrollPaginationfrom models.enums import CreatorUserRole, WorkflowRunTriggeredFromfrom models.workflow import WorkflowRun, WorkflowTypefrom repositories.api_workflow_run_repository import APIWorkflowRunRepository, AppWorkflowRunStats+from repositories.entities.workflow_pause import WorkflowPauseEntity+from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepositoryfrom repositories.types import (AverageInteractionStats,DailyRunsStats,@@ -152,21 +156,84 @@ class LogstoreAPIWorkflowRunRepository(APIWorkflowRunRepository):- Proper time range filtering for LogStore queries"""- def __init__(self, session_maker: sessionmaker | None = None):+ def __init__(self, session_maker: sessionmaker):"""Initialize the repository with LogStore client.Args:- session_maker: SQLAlchemy sessionmaker (unused, for compatibility with factory pattern)+ session_maker: SQLAlchemy sessionmaker for transactional pause and task-control data"""logger.debug("LogstoreAPIWorkflowRunRepository.__init__: initializing")self.logstore_client = AliyunLogStore()+ self._control_repository = DifyAPISQLAlchemyWorkflowRunRepository(session_maker)# Control flag for dual-read (fallback to PostgreSQL when LogStore returns no results)# Set to True to enable fallback for safe migration from PostgreSQL to LogStore# Set to False for new deployments without legacy data in PostgreSQLself._enable_dual_read = os.environ.get("LOGSTORE_DUAL_READ_ENABLED", "true").lower() == "true"+ # Pause state and task ownership are transactional control data, independent+ # of the append-only backend used for workflow history.+ @override+ def create_workflow_pause(+ self,+ workflow_run_id: str,+ state_owner_user_id: str,+ state: str,+ pause_reasons: Sequence[GraphonPauseReason | DifyPauseReason],+ ) -> WorkflowPauseEntity:+ return self._control_repository.create_workflow_pause(+ workflow_run_id, state_owner_user_id, state, pause_reasons+ )++ @override+ def bind_workflow_task(self, *, tenant_id: str, app_id: str, workflow_run_id: str, task_id: str) -> None:+ self._control_repository.bind_workflow_task(+ tenant_id=tenant_id, app_id=app_id, workflow_run_id=workflow_run_id, task_id=task_id+ )++ @override+ def backfill_workflow_pause_task_id(+ self, *, tenant_id: str, app_id: str, workflow_run_id: str, pause_id: str, task_id: str+ ) -> bool:+ return self._control_repository.backfill_workflow_pause_task_id(+ tenant_id=tenant_id, app_id=app_id, workflow_run_id=workflow_run_id, pause_id=pause_id, task_id=task_id+ )++ @override+ def stop_paused_workflow_task(+ self, *, tenant_id: str, app_id: str, task_id: str, owner: tuple[CreatorUserRole, str] | None+ ) -> WorkflowRun | None:+ return self._control_repository.stop_paused_workflow_task(+ tenant_id=tenant_id, app_id=app_id, task_id=task_id, owner=owner+ )++ @override+ def get_workflow_pause(self, workflow_run_id: str) -> WorkflowPauseEntity | None:+ return self._control_repository.get_workflow_pause(workflow_run_id)++ @override+ def resume_workflow_pause(+ self,+ workflow_run_id: str,+ pause_entity: WorkflowPauseEntity,+ *,+ before_resume: Callable[[], None] | None = None,+ ) -> WorkflowPauseEntity:+ return self._control_repository.resume_workflow_pause(+ workflow_run_id, pause_entity, before_resume=before_resume+ )++ @override+ def delete_workflow_pause(self, pause_entity: WorkflowPauseEntity) -> None:+ self._control_repository.delete_workflow_pause(pause_entity)++ @override+ def prune_pauses(+ self, expiration: datetime, resumption_expiration: datetime, limit: int | None = None+ ) -> Sequence[str]:+ return self._control_repository.prune_pauses(expiration, resumption_expiration, limit)+@overridedef get_paginated_workflow_runs(self,@@ -278,6 +345,10 @@ class LogstoreAPIWorkflowRunRepository(APIWorkflowRunRepository):"""logger.debug("get_workflow_run_by_id: tenant_id=%s, app_id=%s, run_id=%s", tenant_id, app_id, run_id)+ control = self._control_repository.get_workflow_run_by_id(tenant_id, app_id, run_id)+ if control is not None and control.task_id is not None:+ return control+try:# Escape parameters to prevent SQL injectionescaped_run_id = escape_identifier(run_id)@@ -383,6 +454,10 @@ class LogstoreAPIWorkflowRunRepository(APIWorkflowRunRepository):"""logger.debug("get_workflow_run_by_id_without_tenant: run_id=%s", run_id)+ control = self._control_repository.get_workflow_run_by_id_without_tenant(run_id)+ if control is not None and control.task_id is not None:+ return control+try:# Escape parameter to prevent SQL injectionescaped_run_id = escape_identifier(run_id)diff --git a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.pyindex c19eb8aa4fb..0ef886953f0 100644--- a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py+++ b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py@@ -11,6 +11,7 @@ from core.repositories.factory import WorkflowExecutionRepositoryfrom core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepositoryfrom extensions.logstore.aliyun_logstore import AliyunLogStorefrom graphon.entities import WorkflowExecution+from graphon.enums import WorkflowExecutionStatusfrom graphon.workflow_type_encoder import WorkflowRuntimeTypeConverterfrom models import (Account,@@ -18,6 +19,8 @@ from models import (EndUser,)from models.enums import WorkflowRunTriggeredFrom+from repositories.entities.workflow_execution import WorkflowExecution as TimedWorkflowExecution+from repositories.workflow_run_control import get_stopped_workflow_runlogger = logging.getLogger(__name__)@@ -72,13 +75,16 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):)self._enable_dual_write = dify_config.LOGSTORE_DUAL_WRITE_ENABLED+ self._session_factory = (+ sessionmaker(bind=session_factory) if isinstance(session_factory, Engine) else session_factory+ )# Control flag for whether to write the `graph` field to LogStore.# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field;# otherwise write an empty {} instead. Defaults to writing the `graph` field.self._enable_put_graph_field = dify_config.LOGSTORE_ENABLE_PUT_GRAPH_FIELD- def _to_logstore_model(self, domain_model: WorkflowExecution) -> list[tuple[str, str]]:+ def _to_logstore_model(self, domain_model: WorkflowExecution, *, log_version: int) -> list[tuple[str, str]]:"""Convert a domain model to a logstore model (List[Tuple[str, str]]).@@ -102,15 +108,14 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):if not self._creator_user_role:raise ValueError("created_by_role is required in repository constructor")- # Generate log_version as nanosecond timestamp for record versioning- log_version = str(time.time_ns())+ version = str(log_version)# Use WorkflowRuntimeTypeConverter to handle complex types (Segment, File, etc.)json_converter = WorkflowRuntimeTypeConverter()logstore_model = [("id", domain_model.id_),- ("log_version", log_version), # Add log_version field for append-only writes+ ("log_version", version), # Add log_version field for append-only writes("tenant_id", self._tenant_id),("app_id", self._app_id or ""),("workflow_id", domain_model.workflow_id),@@ -157,6 +162,13 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):return logstore_model+ @override+ def save_synchronously(self, execution: WorkflowExecution) -> None:+ # Cancellation and pause/resume need a durable SQL owner even when+ # optional SQL history dual-write is disabled. Never swallow this write.+ self.sql_repository.save(execution)+ self.save(execution)+@overridedef save(self, execution: WorkflowExecution) -> None:"""@@ -174,8 +186,24 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):logger.debug("save: id=%s, workflow_id=%s, status=%s", execution.id_, execution.workflow_id, execution.status.value)+ # Capture ordering before reading control state. A concurrent stop's+ # terminal record sorts after this snapshot even if its upload finishes first.+ version = time.time_ns()+ with self._session_factory() as session:+ stopped = get_stopped_workflow_run(session, tenant_id=self._tenant_id, workflow_run_id=execution.id_)+ if stopped is not None:+ execution = TimedWorkflowExecution.model_validate(+ {+ **execution.model_dump(),+ "status": WorkflowExecutionStatus.STOPPED,+ "error_message": stopped.error,+ "outputs": stopped.outputs_dict,+ "finished_at": stopped.finished_at,+ "accumulated_elapsed_time": stopped.elapsed_time,+ }+ )try:- logstore_model = self._to_logstore_model(execution)+ logstore_model = self._to_logstore_model(execution, log_version=version)self.logstore_client.put_log(AliyunLogStore.workflow_execution_logstore, logstore_model)logger.debug("Saved workflow execution to logstore: id=%s", execution.id_)diff --git a/api/extensions/logstore/repositories/logstore_workflow_node_execution_repository.py b/api/extensions/logstore/repositories/logstore_workflow_node_execution_repository.pyindex c75177750b3..6bffc31f615 100644--- a/api/extensions/logstore/repositories/logstore_workflow_node_execution_repository.py+++ b/api/extensions/logstore/repositories/logstore_workflow_node_execution_repository.py@@ -32,6 +32,7 @@ from models import (EndUser,WorkflowNodeExecutionTriggeredFrom,)+from repositories.workflow_run_control import apply_workflow_stop_to_node, get_stopped_workflow_runlogger = logging.getLogger(__name__)@@ -139,6 +140,9 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):# Extract user contextself._triggered_from = triggered_fromself._creator_user_id = user.id+ self._session_factory = (+ sessionmaker(bind=session_factory) if isinstance(session_factory, Engine) else session_factory+ )# Determine user role based on user typeself._creator_user_role = CreatorUserRole.ACCOUNT if isinstance(user, Account) else CreatorUserRole.END_USER@@ -156,7 +160,7 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):# and tests share the same validated source.self._enable_dual_write = dify_config.LOGSTORE_DUAL_WRITE_ENABLED- def _to_logstore_model(self, domain_model: WorkflowNodeExecution) -> Sequence[tuple[str, str]]:+ def _to_logstore_model(self, domain_model: WorkflowNodeExecution, *, log_version: int) -> Sequence[tuple[str, str]]:logger.debug("_to_logstore_model: id=%s, node_id=%s, status=%s",domain_model.id,@@ -170,14 +174,13 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):if not self._creator_user_role:raise ValueError("created_by_role is required in repository constructor")- # Generate log_version as nanosecond timestamp for record versioning- log_version = str(time.time_ns())+ version = str(log_version)json_converter = WorkflowRuntimeTypeConverter()logstore_model = [("id", domain_model.id),- ("log_version", log_version), # Add log_version field for append-only writes+ ("log_version", version), # Add log_version field for append-only writes("tenant_id", self._tenant_id),("app_id", self._app_id or ""),("workflow_id", domain_model.workflow_id),@@ -250,8 +253,9 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):execution.node_execution_id,execution.status.value,)+ execution, version = self._prepare_controlled_write(execution)try:- logstore_model = self._to_logstore_model(execution)+ logstore_model = self._to_logstore_model(execution, log_version=version)self.logstore_client.put_log(AliyunLogStore.workflow_node_execution_logstore, logstore_model)logger.debug(@@ -277,6 +281,25 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):logger.exception("Failed to dual-write node execution to SQL database: id=%s", execution.id)# Don't raise - LogStore write succeeded, SQL is just a backup+ def _prepare_controlled_write(self, execution: WorkflowNodeExecution) -> tuple[WorkflowNodeExecution, int]:+ # Assign ordering before checking SQL so an in-flight pause cannot sort+ # after a stop that commits while the LogStore upload is pending.+ version = time.time_ns()+ if execution.status in (+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ ):+ execution = execution.model_copy()+ with self._session_factory() as session:+ apply_workflow_stop_to_node(+ execution,+ get_stopped_workflow_run(+ session, tenant_id=self._tenant_id, workflow_run_id=execution.workflow_execution_id+ ),+ )+ return execution, version+@overridedef save_synchronously(self, execution: WorkflowNodeExecution) -> None:"""Create the SQL caller row required by Agent v2 participant ownership."""@@ -323,9 +346,12 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):self,workflow_execution_id: str,order_config: OrderConfig | None = None,+ *,+ include_paused: bool = False,) -> Sequence[WorkflowNodeExecution]:"""Retrieve all node executions for a workflow execution.+ Paused nodes are always included; include_paused accepts the shared resume-read contract.Uses LogStore SQL query with window function to get the latest version of each node execution.This ensures we only get the most recent version of each node execution record.Args:@@ -407,6 +433,12 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):logger.warning("Failed to convert row to WorkflowNodeExecution: %s, row=%s", e, row)continue+ with self._session_factory() as session:+ stopped = get_stopped_workflow_run(+ session, tenant_id=self._tenant_id, workflow_run_id=workflow_execution_id+ )+ for execution in executions:+ apply_workflow_stop_to_node(execution, stopped)return executionsexcept Exception:diff --git a/api/extensions/otel/decorators/handlers/workflow_app_runner_handler.py b/api/extensions/otel/decorators/handlers/workflow_app_runner_handler.pyindex 77d8e435d09..3df35aece2d 100644--- a/api/extensions/otel/decorators/handlers/workflow_app_runner_handler.py+++ b/api/extensions/otel/decorators/handlers/workflow_app_runner_handler.py@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)class WorkflowAppRunnerHandler(SpanHandler):- """Span handler for ``WorkflowAppRunner.run``."""+ """Trace the workflow driver's full execution, including graph preparation."""@overridedef wrapper[**P, R](@@ -27,7 +27,7 @@ class WorkflowAppRunnerHandler(SpanHandler):if not arguments:return wrapped(*args, **kwargs)- runner = arguments.get("self")+ runner = arguments.get("runner")if runner is None or not hasattr(runner, "application_generate_entity"):return wrapped(*args, **kwargs)@@ -52,7 +52,7 @@ class WorkflowAppRunnerHandler(SpanHandler):span_name = self._build_span_name(wrapped)except Exception as exc:- logger.warning("Failed to prepare span attributes for WorkflowAppRunner.run: %s", exc, exc_info=True)+ logger.warning("Failed to prepare workflow execution span attributes: %s", exc, exc_info=True)return wrapped(*args, **kwargs)with tracer.start_as_current_span(span_name, kind=SpanKind.INTERNAL, attributes=attributes) as span:diff --git a/api/extensions/otel/parser/base.py b/api/extensions/otel/parser/base.pyindex fc10f183c99..d517fc290ac 100644--- a/api/extensions/otel/parser/base.py+++ b/api/extensions/otel/parser/base.py@@ -17,9 +17,9 @@ from pydantic import BaseModelfrom configs import dify_configfrom enums import DeploymentEditionfrom extensions.otel.semconv.gen_ai import ChainAttributes, GenAIAttributes+from graphon.engine_events import NodeEventfrom graphon.enums import BuiltinNodeTypesfrom graphon.file import File-from graphon.graph_events import GraphNodeEventBasefrom graphon.nodes.base.node import Nodefrom graphon.variables import Segment@@ -90,7 +90,7 @@ class NodeOTelParser(Protocol):"""Parser interface for node-specific OpenTelemetry enrichment."""def parse(- self, *, node: Node, span: "Span", error: Exception | None, result_event: GraphNodeEventBase | None = None+ self, *, node: Node, span: "Span", error: Exception | None, result_event: NodeEvent | None = None) -> None: ...@@ -98,7 +98,7 @@ class DefaultNodeOTelParser:"""Fallback parser used when no node-specific parser is registered."""def parse(- self, *, node: Node, span: "Span", error: Exception | None, result_event: GraphNodeEventBase | None = None+ self, *, node: Node, span: "Span", error: Exception | None, result_event: NodeEvent | None = None) -> None:span.set_attribute("node.id", node.id)if node.execution_id:diff --git a/api/extensions/otel/parser/llm.py b/api/extensions/otel/parser/llm.pyindex ec3c78a12d4..35e62de4ec1 100644--- a/api/extensions/otel/parser/llm.py+++ b/api/extensions/otel/parser/llm.py@@ -10,7 +10,7 @@ from opentelemetry.trace import Spanfrom extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumpsfrom extensions.otel.semconv.gen_ai import LLMAttributes-from graphon.graph_events import GraphNodeEventBase+from graphon.engine_events import NodeEventfrom graphon.nodes.base.node import Nodelogger = logging.getLogger(__name__)@@ -99,7 +99,7 @@ class LLMNodeOTelParser:self._delegate = DefaultNodeOTelParser()def parse(- self, *, node: Node, span: "Span", error: Exception | None, result_event: GraphNodeEventBase | None = None+ self, *, node: Node, span: "Span", error: Exception | None, result_event: NodeEvent | None = None) -> None:self._delegate.parse(node=node, span=span, error=error, result_event=result_event)diff --git a/api/extensions/otel/parser/retrieval.py b/api/extensions/otel/parser/retrieval.pyindex 56672d1fd45..27655c4719d 100644--- a/api/extensions/otel/parser/retrieval.py+++ b/api/extensions/otel/parser/retrieval.py@@ -10,7 +10,7 @@ from opentelemetry.trace import Spanfrom extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumpsfrom extensions.otel.semconv.gen_ai import RetrieverAttributes-from graphon.graph_events import GraphNodeEventBase+from graphon.engine_events import NodeEventfrom graphon.nodes.base.node import Nodefrom graphon.variables import Segment@@ -72,7 +72,7 @@ class RetrievalNodeOTelParser:self._delegate = DefaultNodeOTelParser()def parse(- self, *, node: Node, span: "Span", error: Exception | None, result_event: GraphNodeEventBase | None = None+ self, *, node: Node, span: "Span", error: Exception | None, result_event: NodeEvent | None = None) -> None:self._delegate.parse(node=node, span=span, error=error, result_event=result_event)diff --git a/api/extensions/otel/parser/tool.py b/api/extensions/otel/parser/tool.pyindex 75ddbba4480..4045db1a654 100644--- a/api/extensions/otel/parser/tool.py+++ b/api/extensions/otel/parser/tool.py@@ -6,8 +6,8 @@ from opentelemetry.trace import Spanfrom extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumpsfrom extensions.otel.semconv.gen_ai import ToolAttributes+from graphon.engine_events import NodeEventfrom graphon.enums import WorkflowNodeExecutionMetadataKey-from graphon.graph_events import GraphNodeEventBasefrom graphon.nodes.base.node import Nodefrom graphon.nodes.tool.entities import ToolNodeData@@ -19,7 +19,7 @@ class ToolNodeOTelParser:self._delegate = DefaultNodeOTelParser()def parse(- self, *, node: Node, span: "Span", error: Exception | None, result_event: GraphNodeEventBase | None = None+ self, *, node: Node, span: "Span", error: Exception | None, result_event: NodeEvent | None = None) -> None:self._delegate.parse(node=node, span=span, error=error, result_event=result_event)diff --git a/api/libs/contextvars.py b/api/libs/contextvars.pynew file mode 100644index 00000000000..2c391420347--- /dev/null+++ b/api/libs/contextvars.py@@ -0,0 +1,13 @@+from collections.abc import Generator+from contextlib import ExitStack, contextmanager+from contextvars import Context+++@contextmanager+def use_contextvars(context_vars: Context | None) -> Generator[None]:+ """Apply captured bindings and restore the caller's values after cleanup."""+ with ExitStack() as stack:+ if context_vars is not None:+ for var, value in context_vars.items():+ stack.callback(var.reset, var.set(value))+ yielddiff --git a/api/libs/flask_utils.py b/api/libs/flask_utils.pyindex 280d9198882..1cafee204e7 100644--- a/api/libs/flask_utils.py+++ b/api/libs/flask_utils.py@@ -1,10 +1,12 @@import contextvars-from collections.abc import Generator # Changed from Iterator+from collections.abc import Generatorfrom contextlib import contextmanagerfrom typing import TYPE_CHECKINGfrom flask import Flask, g+from libs.contextvars import use_contextvars+if TYPE_CHECKING:from models import Account, EndUser@@ -13,7 +15,7 @@ if TYPE_CHECKING:def preserve_flask_contexts(flask_app: Flask,context_vars: contextvars.Context,-) -> Generator[None]: # Changed from Iterator[None]+) -> Generator[None]:"""A context manager that handles:1. flask-login's UserProxy copy@@ -22,7 +24,7 @@ def preserve_flask_contexts(This context manager ensures that the Flask application context is properly set up,the current user is preserved across context boundaries, and any provided context variables- are set within the new context.+ are set within the new context. Caller bindings are restored after Flask cleanup.Note:This manager aims to allow use current_user cross thread and app context,@@ -42,29 +44,13 @@ def preserve_flask_contexts(# Current user will be preserved if available```"""- # Set context variables if provided- if context_vars:- for var, val in context_vars.items():- var.set(val)-- # Save current user before entering new app context- saved_user = None- # Check for user in g (works in both request context and app context)- if hasattr(g, "_login_user"):- saved_user = g._login_user-- # Enter Flask app context- with flask_app.app_context():- try:- # Restore user in new app context if it was saved+ with use_contextvars(context_vars):+ # Read the captured user before creating a fresh Flask app context.+ saved_user = g.get("_login_user")+ with flask_app.app_context():if saved_user is not None:g._login_user = saved_user-- # Yield control back to the calleryield- finally:- # Any cleanup can be added here if needed- passdef set_login_user(user: "Account | EndUser"):diff --git a/api/migrations/versions/2026_10_10_1200-c6f4a90d8b31_add_workflow_pause_task_id.py b/api/migrations/versions/2026_10_10_1200-c6f4a90d8b31_add_workflow_pause_task_id.pynew file mode 100644index 00000000000..d2222ee231b--- /dev/null+++ b/api/migrations/versions/2026_10_10_1200-c6f4a90d8b31_add_workflow_pause_task_id.py@@ -0,0 +1,67 @@+"""Persist workflow task ownership and cancellation intent.++Revision ID: c6f4a90d8b31+Revises: f1d3c5b7a902+Create Date: 2026-10-10 12:00:00+"""++import sqlalchemy as sa+from alembic import op++from models.types import StringUUID++revision = "c6f4a90d8b31"+down_revision = "f1d3c5b7a902"+branch_labels = None+depends_on = None+++def upgrade():+ postgresql = op.get_bind().dialect.name == "postgresql"+ inspector = sa.inspect(op.get_bind()) if not postgresql and not op.get_context().as_sql else None+ existing_columns = {column["name"] for column in inspector.get_columns("workflow_runs")} if inspector else set()+ # The index build can fail after committing columns but before recording this revision.+ with op.batch_alter_table("workflow_runs") as batch_op:+ if "task_id" not in existing_columns:+ batch_op.add_column(sa.Column("task_id", StringUUID(), nullable=True), if_not_exists=postgresql)+ if "stop_requested_at" not in existing_columns:+ batch_op.add_column(sa.Column("stop_requested_at", sa.DateTime(), nullable=True), if_not_exists=postgresql)++ if postgresql:+ with op.get_context().autocommit_block():+ # An interrupted concurrent build can leave an invalid index behind.+ op.drop_index(+ "workflow_run_task_id_idx", table_name="workflow_runs", postgresql_concurrently=True, if_exists=True+ )+ op.create_index(+ "workflow_run_task_id_idx",+ "workflow_runs",+ ["task_id"],+ postgresql_concurrently=True,+ postgresql_where=sa.text("task_id IS NOT NULL"),+ )+ elif inspector is None or not any(+ index["name"] == "workflow_run_task_id_idx" for index in inspector.get_indexes("workflow_runs")+ ):+ op.create_index("workflow_run_task_id_idx", "workflow_runs", ["task_id"])+++def downgrade():+ postgresql = op.get_bind().dialect.name == "postgresql"+ inspector = sa.inspect(op.get_bind()) if not postgresql and not op.get_context().as_sql else None+ existing_columns = {column["name"] for column in inspector.get_columns("workflow_runs")} if inspector else set()+ if postgresql:+ with op.get_context().autocommit_block():+ op.drop_index(+ "workflow_run_task_id_idx", table_name="workflow_runs", postgresql_concurrently=True, if_exists=True+ )+ elif inspector is None or any(+ index["name"] == "workflow_run_task_id_idx" for index in inspector.get_indexes("workflow_runs")+ ):+ op.drop_index("workflow_run_task_id_idx", table_name="workflow_runs")++ with op.batch_alter_table("workflow_runs") as batch_op:+ if inspector is None or "stop_requested_at" in existing_columns:+ batch_op.drop_column("stop_requested_at")+ if inspector is None or "task_id" in existing_columns:+ batch_op.drop_column("task_id")diff --git a/api/models/workflow.py b/api/models/workflow.pyindex 2e3827504dd..e6d411bf43c 100644--- a/api/models/workflow.py+++ b/api/models/workflow.py@@ -38,6 +38,7 @@ from core.workflow.variable_prefixes import (CONVERSATION_VARIABLE_NODE_ID,SYSTEM_VARIABLE_NODE_ID,)+from enums import WorkflowKindfrom extensions.ext_storage import Storagefrom factories.variable_factory import TypeMismatchError, build_segment_with_typefrom graphon.entities.graph_config import NodeConfigDict, NodeConfigDictAdapter@@ -151,18 +152,6 @@ class WorkflowType(StrEnum):return cls.WORKFLOW if app_mode == AppMode.WORKFLOW else cls.CHAT-class WorkflowKind(StrEnum):- STANDARD = "standard"- SNIPPET = "snippet"-- @classmethod- def value_of(cls, value: str) -> "WorkflowKind":- for kind in cls:- if kind.value == value:- return kind- raise ValueError(f"invalid workflow kind value {value}")--def resolve_workflow_kind(kind: str | WorkflowKind | None) -> WorkflowKind:if kind is None:return WorkflowKind.STANDARD@@ -804,12 +793,18 @@ class WorkflowRun(Base):sa.PrimaryKeyConstraint("id", name="workflow_run_pkey"),sa.Index("workflow_run_triggerd_from_idx", "tenant_id", "app_id", "triggered_from"),sa.Index("workflow_run_created_at_id_idx", "created_at", "id"),+ sa.Index("workflow_run_task_id_idx", "task_id", postgresql_where=sa.text("task_id IS NOT NULL")),)id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()))tenant_id: Mapped[str] = mapped_column(StringUUID)app_id: Mapped[str] = mapped_column(StringUUID)+ # Public cancellation uses a task ID distinct from the workflow run ID.+ # Nullable for historical runs whose task ID is recovered on pause replay.+ task_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)+ stop_requested_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)+workflow_id: Mapped[str] = mapped_column(StringUUID)type: Mapped[WorkflowType] = mapped_column(EnumText(WorkflowType, length=255))triggered_from: Mapped[WorkflowRunTriggeredFrom] = mapped_column(EnumText(WorkflowRunTriggeredFrom, length=255))@@ -2125,7 +2120,7 @@ class WorkflowPause(DefaultFieldsDCMixin, TypeBase):)# state_object_key stores the object key referencing the serialized runtime state- # of the `GraphEngine`. This object captures the complete execution context of the+ # of the `Engine`. This object captures the complete execution context of the# workflow at the moment it was paused, enabling accurate resumption.state_object_key: Mapped[str] = mapped_column(String(length=255), nullable=False)diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.mdindex f2f2d4b02d8..efb345e70ba 100644--- a/api/openapi/markdown/console-openapi.md+++ b/api/openapi/markdown/console-openapi.md@@ -9427,10 +9427,7 @@ Generate structured output rules using LLM| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |### [POST] /snippets/{snippet_id}/workflow-runs/tasks/{task_id}/stop-**Stop a running snippet workflow task**--Uses both the legacy stop flag mechanism and the graph engine-command channel for backward compatibility.+**Stop a running or paused task belonging to this snippet**#### Parametersdiff --git a/api/pyproject.toml b/api/pyproject.tomlindex 1664695c8d9..4b79df8a2dc 100644--- a/api/pyproject.toml+++ b/api/pyproject.toml@@ -40,12 +40,13 @@ dependencies = ["opentelemetry-instrumentation-redis==0.65b0","opentelemetry-instrumentation-sqlalchemy==0.65b0","opentelemetry-propagator-b3>=1.45.0,<2.0.0",+ "pypandoc==1.17","readabilipy==0.3.0","resend>=2.27.0,<3.0.0","zstandard==0.25.0",# Emerging: newer and fast-moving, use compatible pins"fastopenapi[flask]==0.7.0",- "graphon==0.7.0",+ "graphon==0.8.0","httpx-sse==0.4.3","json-repair==0.60.1",]diff --git a/api/repositories/api_workflow_node_execution_repository.py b/api/repositories/api_workflow_node_execution_repository.pyindex b9197dc7efe..9f077548994 100644--- a/api/repositories/api_workflow_node_execution_repository.py+++ b/api/repositories/api_workflow_node_execution_repository.py@@ -16,7 +16,6 @@ from typing import Any, Protocolfrom sqlalchemy.orm import Session-from core.repositories.factory import WorkflowNodeExecutionRepositoryfrom models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionOffload@@ -41,7 +40,7 @@ class WorkflowNodeExecutionSnapshot:loop_id: str | None = None # Loop id from execution metadata, if any.-class DifyAPIWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository, Protocol):+class DifyAPIWorkflowNodeExecutionRepository(Protocol):"""Protocol for service-layer operations on WorkflowNodeExecutionModel.diff --git a/api/repositories/api_workflow_run_repository.py b/api/repositories/api_workflow_run_repository.pyindex 1314dba318e..0d6a4cb3922 100644--- a/api/repositories/api_workflow_run_repository.py+++ b/api/repositories/api_workflow_run_repository.py@@ -45,7 +45,7 @@ from core.workflow.nodes.human_input.pause_reason import PauseReason as DifyPausfrom graphon.entities.pause_reason import PauseReason as GraphonPauseReasonfrom graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom libs.infinite_scroll_pagination import InfiniteScrollPagination-from models.enums import WorkflowRunTriggeredFrom+from models.enums import CreatorUserRole, WorkflowRunTriggeredFromfrom models.workflow import WorkflowAppLog, WorkflowArchiveLog, WorkflowPause, WorkflowPauseReason, WorkflowRunfrom repositories.entities.workflow_pause import WorkflowPauseEntityfrom repositories.types import (@@ -568,6 +568,22 @@ class APIWorkflowRunRepository(Protocol):# while creating pause....+ def bind_workflow_task(self, *, tenant_id: str, app_id: str, workflow_run_id: str, task_id: str) -> None:+ """Bind the public task before its workflow-started event is exposed."""+ ...++ def stop_paused_workflow_task(+ self, *, tenant_id: str, app_id: str, task_id: str, owner: tuple[CreatorUserRole, str] | None+ ) -> WorkflowRun | None:+ """Admit a task by its durable owner, record stop intent, and finish a paused run."""+ ...++ def backfill_workflow_pause_task_id(+ self, *, tenant_id: str, app_id: str, workflow_run_id: str, pause_id: str, task_id: str+ ) -> bool:+ """Index an unchanged legacy pause; return whether its task ID was written."""+ ...+def get_workflow_pause(self, workflow_run_id: str) -> WorkflowPauseEntity | None:"""Retrieve the current pause for a workflow execution.@@ -579,6 +595,8 @@ class APIWorkflowRunRepository(Protocol):self,workflow_run_id: str,pause_entity: WorkflowPauseEntity,+ *,+ before_resume: Callable[[], None] | None = None,) -> WorkflowPauseEntity:"""Resume a paused workflow.diff --git a/api/repositories/human_input_errors.py b/api/repositories/human_input_errors.pynew file mode 100644index 00000000000..7bdb3fd1a16--- /dev/null+++ b/api/repositories/human_input_errors.py@@ -0,0 +1,9 @@+"""Persistence conflicts for human input form submission."""++from core.workflow.nodes.human_input.enums import HumanInputFormStatus+++class FormAlreadyHandledError(Exception):+ def __init__(self, status: HumanInputFormStatus):+ self.status = status+ super().__init__(f"Form is no longer waiting: {status}")diff --git a/api/repositories/sqlalchemy_api_workflow_run_repository.py b/api/repositories/sqlalchemy_api_workflow_run_repository.pyindex e29bbbddc53..85607289134 100644--- a/api/repositories/sqlalchemy_api_workflow_run_repository.py+++ b/api/repositories/sqlalchemy_api_workflow_run_repository.py@@ -29,12 +29,13 @@ from typing import Any, NamedTuple, cast, overrideimport sqlalchemy as safrom pydantic import ValidationError-from sqlalchemy import and_, delete, func, null, or_, select, tuple_+from sqlalchemy import and_, delete, func, null, or_, select, tuple_, updatefrom sqlalchemy.engine import CursorResult-from sqlalchemy.orm import Session, selectinload, sessionmaker+from sqlalchemy.orm import Session, load_only, selectinload, sessionmakerfrom core.workflow.human_input_forms import load_form_tokens_by_form_idfrom core.workflow.nodes.human_input.entities import FormDefinition+from core.workflow.nodes.human_input.enums import HumanInputFormStatusfrom core.workflow.nodes.human_input.pause_reason import (HumanInputRequired,)@@ -51,15 +52,22 @@ from graphon.entities.pause_reason import (from graphon.entities.pause_reason import (PauseReason as GraphonPauseReason,)-from graphon.enums import WorkflowExecutionStatus, WorkflowType+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowTypefrom libs.datetime_utils import naive_utc_nowfrom libs.helper import convert_datetime_to_datefrom libs.infinite_scroll_pagination import InfiniteScrollPaginationfrom libs.time_parser import get_time_thresholdfrom models import Message-from models.enums import WorkflowRunTriggeredFrom+from models.enums import CreatorUserRole, MessageStatus, WorkflowRunTriggeredFromfrom models.human_input import HumanInputForm, HumanInputFormRecipient-from models.workflow import WorkflowAppLog, WorkflowArchiveLog, WorkflowPause, WorkflowPauseReason, WorkflowRun+from models.workflow import (+ WorkflowAppLog,+ WorkflowArchiveLog,+ WorkflowNodeExecutionModel,+ WorkflowPause,+ WorkflowPauseReason,+ WorkflowRun,+)from repositories.api_workflow_run_repository import (APIWorkflowRunRepository,AppWorkflowRunStats,@@ -95,6 +103,18 @@ class _WorkflowRunError(Exception):pass+class WorkflowRunNotPausedError(_WorkflowRunError):+ """A concurrent stop or resume has already consumed the paused run."""+++class WorkflowTaskOwnerMismatchError(_WorkflowRunError):+ """The scoped task belongs to another initiating user."""+++class WorkflowPauseStoppedError(_WorkflowRunError):+ """A requested stop won while the engine was persisting its pause."""++_HEX_SHARD_VALUES = {"0": 0,"1": 1,@@ -1077,88 +1097,102 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):ValueError: If workflow_run_id is invalid or workflow run doesn't existRuntimeError: If workflow is already paused or in invalid state"""- previous_pause_model_query = select(WorkflowPause).where(WorkflowPause.workflow_run_id == workflow_run_id)- with self._session_maker() as session, session.begin():- # Get the workflow run- workflow_run = session.get(WorkflowRun, workflow_run_id)- if workflow_run is None:- raise ValueError(f"WorkflowRun not found: {workflow_run_id}")-- # Check if workflow is in RUNNING status- # TODO(QuantumGhost): It seems that the persistence of `WorkflowRun.status`- # happens before the execution of GraphLayer- if workflow_run.status not in {WorkflowExecutionStatus.RUNNING, WorkflowExecutionStatus.PAUSED}:- raise _WorkflowRunError(- f"Only WorkflowRun with RUNNING or PAUSED status can be paused, "- f"workflow_run_id={workflow_run_id}, current_status={workflow_run.status}"+ # Do not hold a database transaction during object storage upload.+ # The subsequent run lock checks durable stop intent before publishing pause.+ state_obj_key = f"workflow-state-{uuid.uuid4()}.json"+ storage.save(state_obj_key, state.encode())+ try:+ previous_pause_model_query = select(WorkflowPause).where(WorkflowPause.workflow_run_id == workflow_run_id)+ with self._session_maker() as session, session.begin():+ # Get the workflow run+ workflow_run = session.scalar(+ select(WorkflowRun).where(WorkflowRun.id == workflow_run_id).with_for_update())- #- previous_pause = session.scalars(previous_pause_model_query).first()- if previous_pause:- self._delete_pause_model(session, previous_pause)- # we need to flush here to ensure that the old one is actually deleted.- session.flush()-- state_obj_key = f"workflow-state-{uuid.uuid4()}.json"- storage.save(state_obj_key, state.encode())- # Upload the state file-- # Create the pause record- pause_model = WorkflowPause(- workflow_id=workflow_run.workflow_id,- workflow_run_id=workflow_run.id,- state_object_key=state_obj_key,- )- pause_reason_models = []- for reason in pause_reasons:- match reason:- case HitlRequired():- pause_reason_model = WorkflowPauseReason(- pause_id=pause_model.id,- type_=PauseReasonType.HITL_REQUIRED,- form_id=default_session_binding.resolve_form_id_from_session_id(- session_id=reason.session_id- ),- node_id=reason.node_id,- )- case HumanInputRequired():- pause_reason_model = WorkflowPauseReason(- pause_id=pause_model.id,- type_=PauseReasonType.HITL_REQUIRED,- form_id=reason.form_id,- node_id=reason.node_id,- )- case SchedulingPause():- pause_reason_model = WorkflowPauseReason(- pause_id=pause_model.id,- type_=reason.TYPE,- message=reason.message,- )- case _:- raise AssertionError(f"unknown reason type: {type(reason)}")-- pause_reason_models.append(pause_reason_model)-- # Update workflow run status- workflow_run.status = WorkflowExecutionStatus.PAUSED-- # Save everything in a transaction- session.add(pause_model)- session.add(workflow_run)- session.add_all(pause_reason_models)-- logger.info("Created workflow pause %s for workflow run %s", pause_model.id, workflow_run_id)-- # NOTE(QuantumGhost): repository callers on the Dify side should only- # observe enriched Dify pause reasons. The Graphon-native reason is an- # input-only boundary concern while persisting the pause.- hydrated_pause_reasons = self._hydrate_pause_reasons(session, pause_reason_models)-- return _PrivateWorkflowPauseEntity(- pause_model=pause_model,- reason_models=pause_reason_models,- pause_reasons=hydrated_pause_reasons,- )+ if workflow_run is None:+ raise ValueError(f"WorkflowRun not found: {workflow_run_id}")++ if workflow_run.stop_requested_at is not None or workflow_run.status == WorkflowExecutionStatus.STOPPED:+ raise WorkflowPauseStoppedError(f"Workflow stopped while persisting pause: {workflow_run_id}")++ # Check if workflow is in RUNNING status+ # TODO(QuantumGhost): It seems that the persistence of `WorkflowRun.status`+ # happens before the execution of GraphLayer+ if workflow_run.status not in {WorkflowExecutionStatus.RUNNING, WorkflowExecutionStatus.PAUSED}:+ raise _WorkflowRunError(+ f"Only WorkflowRun with RUNNING or PAUSED status can be paused, "+ f"workflow_run_id={workflow_run_id}, current_status={workflow_run.status}"+ )+ #+ previous_pause = session.scalars(previous_pause_model_query).first()+ if previous_pause:+ self._delete_pause_model(session, previous_pause)+ # we need to flush here to ensure that the old one is actually deleted.+ session.flush()++ # Create the pause record+ pause_model = WorkflowPause(+ workflow_id=workflow_run.workflow_id,+ workflow_run_id=workflow_run.id,+ state_object_key=state_obj_key,+ )+ pause_reason_models = []+ for reason in pause_reasons:+ match reason:+ case HitlRequired():+ pause_reason_model = WorkflowPauseReason(+ pause_id=pause_model.id,+ type_=PauseReasonType.HITL_REQUIRED,+ form_id=default_session_binding.resolve_form_id_from_session_id(+ session_id=reason.session_id+ ),+ node_id=reason.node_id,+ )+ case HumanInputRequired():+ pause_reason_model = WorkflowPauseReason(+ pause_id=pause_model.id,+ type_=PauseReasonType.HITL_REQUIRED,+ form_id=reason.form_id,+ node_id=reason.node_id,+ )+ case SchedulingPause():+ pause_reason_model = WorkflowPauseReason(+ pause_id=pause_model.id,+ type_=reason.TYPE,+ message=reason.message,+ )+ case _:+ raise AssertionError(f"unknown reason type: {type(reason)}")++ pause_reason_models.append(pause_reason_model)++ # Update workflow run status+ workflow_run.status = WorkflowExecutionStatus.PAUSED++ # Save everything in a transaction+ session.add(pause_model)+ session.add(workflow_run)+ session.add_all(pause_reason_models)++ logger.info("Created workflow pause %s for workflow run %s", pause_model.id, workflow_run_id)++ # NOTE(QuantumGhost): repository callers on the Dify side should only+ # observe enriched Dify pause reasons. The Graphon-native reason is an+ # input-only boundary concern while persisting the pause.+ hydrated_pause_reasons = self._hydrate_pause_reasons(session, pause_reason_models)++ return _PrivateWorkflowPauseEntity(+ pause_model=pause_model,+ reason_models=pause_reason_models,+ pause_reasons=hydrated_pause_reasons,+ )+ except Exception:+ try:+ storage.delete(state_obj_key)+ except Exception:+ logger.exception(+ "Failed to clean up unpublished workflow pause state, workflow_run_id=%s", workflow_run_id+ )+ raisedef _get_reasons_by_pause_id(self, session: Session, pause_id: str):reason_stmt = select(WorkflowPauseReason).where(WorkflowPauseReason.pause_id == pause_id)@@ -1199,6 +1233,90 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):pause_reasons.append(_to_dify_pause_reason(reason))return pause_reasons+ @override+ def bind_workflow_task(self, *, tenant_id: str, app_id: str, workflow_run_id: str, task_id: str) -> None:+ with self._session_maker() as session, session.begin():+ session.execute(+ update(WorkflowRun)+ .where(+ WorkflowRun.id == workflow_run_id,+ WorkflowRun.tenant_id == tenant_id,+ WorkflowRun.app_id == app_id,+ WorkflowRun.task_id.is_(None),+ )+ .values(task_id=task_id)+ )++ def list_legacy_workflow_pauses(+ self,+ *,+ tenant_id: str,+ app_id: str,+ owner: tuple[CreatorUserRole, str] | None,+ after_id: str | None,+ limit: int,+ ) -> Sequence[tuple[WorkflowRun, WorkflowPauseEntity]]:+ """Page admitted legacy snapshots without loading graphs or pause reasons."""+ # TODO: Remove legacy task recovery 30 days after the last legacy writer is retired.+ stmt = (+ select(WorkflowRun, WorkflowPause)+ .select_from(WorkflowPause)+ .join(WorkflowRun, WorkflowRun.id == WorkflowPause.workflow_run_id)+ .options(+ load_only(+ WorkflowRun.id,+ WorkflowRun.tenant_id,+ WorkflowRun.app_id,+ WorkflowRun.task_id,+ WorkflowRun.status,+ WorkflowRun.created_by,+ WorkflowRun.created_by_role,+ )+ )+ .where(+ WorkflowRun.tenant_id == tenant_id,+ WorkflowRun.app_id == app_id,+ WorkflowRun.task_id.is_(None),+ WorkflowRun.type.in_([WorkflowType.WORKFLOW, WorkflowType.CHAT]),+ WorkflowRun.status == WorkflowExecutionStatus.PAUSED,+ WorkflowPause.resumed_at.is_(None),+ )+ .order_by(WorkflowPause.id)+ .limit(limit)+ )+ if owner is not None:+ stmt = stmt.where(WorkflowRun.created_by_role == owner[0], WorkflowRun.created_by == owner[1])+ if after_id is not None:+ stmt = stmt.where(WorkflowPause.id > after_id)+ with self._session_maker() as session:+ return [+ (run, _PrivateWorkflowPauseEntity(pause_model=pause, reason_models=(), pause_reasons=()))+ for run, pause in session.execute(stmt).tuples()+ ]++ @override+ def backfill_workflow_pause_task_id(+ self, *, tenant_id: str, app_id: str, workflow_run_id: str, pause_id: str, task_id: str+ ) -> bool:+ current_pause = select(WorkflowPause.workflow_run_id).where(+ WorkflowPause.id == pause_id,+ WorkflowPause.workflow_run_id == workflow_run_id,+ WorkflowPause.resumed_at.is_(None),+ )+ with self._session_maker() as session, session.begin():+ result = session.execute(+ update(WorkflowRun)+ .where(+ WorkflowRun.id.in_(current_pause),+ WorkflowRun.tenant_id == tenant_id,+ WorkflowRun.app_id == app_id,+ WorkflowRun.status == WorkflowExecutionStatus.PAUSED,+ WorkflowRun.task_id.is_(None),+ )+ .values(task_id=task_id)+ )+ return cast(CursorResult, result).rowcount == 1+@overridedef get_workflow_pause(self,@@ -1281,11 +1399,110 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):form_tokens=form_tokens,)+ def stop_paused_workflow_task(+ self,+ *,+ tenant_id: str,+ app_id: str,+ task_id: str,+ owner: tuple[CreatorUserRole, str] | None,+ ) -> WorkflowRun | None:+ """Terminate an admitted app's paused task and invalidate its pending forms.++ ``owner=None`` preserves app-scoped Workflow API cancellation. Chat+ callers supply the initiating account or end user. The run lock is also+ taken by resume, so a stopped run cannot be changed back to running.+ """+ stmt = (+ select(WorkflowRun)+ .where(+ WorkflowRun.tenant_id == tenant_id,+ WorkflowRun.app_id == app_id,+ WorkflowRun.task_id == task_id,+ )+ .with_for_update(of=WorkflowRun)+ )+ with self._session_maker(expire_on_commit=False) as session, session.begin():+ workflow_run = session.scalar(stmt)+ if workflow_run is None:+ return None+ if owner is not None and (workflow_run.created_by_role, workflow_run.created_by) != owner:+ raise WorkflowTaskOwnerMismatchError()+ if workflow_run.status == WorkflowExecutionStatus.STOPPED and workflow_run.stop_requested_at is not None:+ # A retry can finish publishing history/SSE if the external+ # history backend failed after the durable stop committed.+ return workflow_run+ if workflow_run.status not in (WorkflowExecutionStatus.RUNNING, WorkflowExecutionStatus.PAUSED):+ return None+ workflow_run.stop_requested_at = naive_utc_now()+ if workflow_run.status != WorkflowExecutionStatus.PAUSED:+ # A resumed attempt may not have recreated its Redis ownership+ # cache yet. Preserve this durable admission so stop can still+ # arm its command channel during that handoff.+ return workflow_run+ pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == workflow_run.id))+ workflow_run.status = WorkflowExecutionStatus.STOPPED+ workflow_run.error = "User requested stop"+ finished_at = naive_utc_now()+ workflow_run.finished_at = finished_at+ session.execute(+ update(Message)+ .where(+ Message.app_id == app_id,+ Message.workflow_run_id == workflow_run.id,+ Message.status == MessageStatus.PAUSED,+ )+ .values(status=MessageStatus.NORMAL)+ )+ # Nested Workflow Tool forms share their root run ID. The admitted+ # run establishes ownership before these dependent rows are changed.+ session.execute(+ update(HumanInputForm)+ .where(+ HumanInputForm.tenant_id == tenant_id,+ HumanInputForm.workflow_run_id == workflow_run.id,+ HumanInputForm.status == HumanInputFormStatus.WAITING,+ )+ .values(status=HumanInputFormStatus.EXPIRED)+ )+ unfinished_nodes = session.scalars(+ select(WorkflowNodeExecutionModel).where(+ WorkflowNodeExecutionModel.tenant_id == tenant_id,+ WorkflowNodeExecutionModel.workflow_run_id == workflow_run.id,+ WorkflowNodeExecutionModel.status.in_(+ [+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ ]+ ),+ )+ )+ for node in unfinished_nodes:+ node.status = WorkflowNodeExecutionStatus.FAILED+ node.error = workflow_run.error+ node.finished_at = finished_at+ node.elapsed_time = max((finished_at - node.created_at).total_seconds(), 0.0)+ state_object_key = pause.state_object_key if pause is not None else None+ if pause is not None:+ session.execute(delete(WorkflowPauseReason).where(WorkflowPauseReason.pause_id == pause.id))+ session.delete(pause)++ # Storage is external I/O; a cleanup failure must not undo cancellation.+ if state_object_key is not None:+ try:+ storage.delete(state_object_key)+ except Exception:+ logger.exception("Failed to delete stopped workflow state, workflow_run_id=%s", workflow_run.id)+ return workflow_run+@overridedef resume_workflow_pause(self,workflow_run_id: str,pause_entity: WorkflowPauseEntity,+ *,+ before_resume: Callable[[], None] | None = None,) -> WorkflowPauseEntity:"""Resume a paused workflow.@@ -1307,14 +1524,19 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):"""with self._session_maker() as session, session.begin():# Get the workflow run with pause- stmt = select(WorkflowRun).options(selectinload(WorkflowRun.pause)).where(WorkflowRun.id == workflow_run_id)+ stmt = (+ select(WorkflowRun)+ .options(selectinload(WorkflowRun.pause))+ .where(WorkflowRun.id == workflow_run_id)+ .with_for_update()+ )workflow_run = session.scalar(stmt)if workflow_run is None:raise ValueError(f"WorkflowRun not found: {workflow_run_id}")- if workflow_run.status != WorkflowExecutionStatus.PAUSED:- raise _WorkflowRunError(+ if workflow_run.status != WorkflowExecutionStatus.PAUSED or workflow_run.stop_requested_at is not None:+ raise WorkflowRunNotPausedError(f"WorkflowRun is not in PAUSED status, workflow_run_id={workflow_run_id}, "f"current_status={workflow_run.status}")@@ -1332,6 +1554,12 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):if pause_model.resumed_at is not None:raise _WorkflowRunError(f"Cannot resume an already resumed pause, pause_id={pause_model.id}")+ # Reset the bounded Redis cancellation channels while holding the+ # run lock: stop must either win this transition or send its signals+ # after the reset. A duplicate resume must never clear a live stop.+ if before_resume is not None:+ before_resume()+pause_reasons = self._get_reasons_by_pause_id(session, pause_model.id)hydrated_pause_reasons = self._hydrate_pause_reasons(session, pause_reasons)diff --git a/api/repositories/workflow_run_control.py b/api/repositories/workflow_run_control.pynew file mode 100644index 00000000000..7aa2a9737df--- /dev/null+++ b/api/repositories/workflow_run_control.py@@ -0,0 +1,41 @@+"""Keep delayed execution history consistent with durable workflow cancellation."""++from sqlalchemy import select+from sqlalchemy.orm import Session++from graphon.entities import WorkflowNodeExecution+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus+from libs.datetime_utils import ensure_naive_utc+from models.workflow import WorkflowNodeExecutionModel, WorkflowRun+++def get_stopped_workflow_run(+ session: Session, *, tenant_id: str, workflow_run_id: str | None, lock: bool = False+) -> WorkflowRun | None:+ if workflow_run_id is None:+ return None+ statement = select(WorkflowRun).where(WorkflowRun.tenant_id == tenant_id, WorkflowRun.id == workflow_run_id)+ if lock:+ statement = statement.with_for_update()+ run = session.scalar(statement)+ if run is not None and run.status == WorkflowExecutionStatus.STOPPED and run.stop_requested_at is not None:+ return run+ return None+++def apply_workflow_stop_to_node(+ execution: WorkflowNodeExecution | WorkflowNodeExecutionModel, stopped_run: WorkflowRun | None+) -> None:+ if stopped_run is None or execution.status not in (+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ ):+ return+ execution.status = WorkflowNodeExecutionStatus.FAILED+ execution.error = stopped_run.error or "User requested stop"+ execution.finished_at = stopped_run.finished_at+ if stopped_run.finished_at is not None:+ execution.elapsed_time = max(+ (stopped_run.finished_at - ensure_naive_utc(execution.created_at)).total_seconds(), 0.0+ )diff --git a/api/services/agent_tool_inner_service.py b/api/services/agent_tool_inner_service.pyindex 633ca893007..dc77f92b2e2 100644--- a/api/services/agent_tool_inner_service.py+++ b/api/services/agent_tool_inner_service.py@@ -36,6 +36,7 @@ from core.tools.utils.message_transformer import ToolFileMessageTransformerfrom models.model import Appfrom services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponsefrom services.errors.agent_tool_inner import AgentToolInnerServiceError+from services.workflow_run_agg import WorkflowRunAggclass AgentToolInnerService:@@ -65,6 +66,7 @@ class AgentToolInnerService:)try:tool_runtime = ToolManager.get_agent_tool_runtime(+ execution_driver=WorkflowRunAgg.run,tenant_id=request.caller.tenant_id,app_id=request.caller.app_id,agent_tool=agent_tool,diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.pyindex d8569f4e522..94b0f108084 100644--- a/api/services/app_generate_service.py+++ b/api/services/app_generate_service.py@@ -24,7 +24,7 @@ from core.db import session_factoryfrom enums import DeploymentEdition, QuotaTypefrom extensions.otel import AppGenerateHandler, trace_spanfrom models.model import Account, App, AppMode, EndUser-from models.workflow import Workflow, WorkflowRun+from models.workflow import Workflowfrom services.errors.app import (QuotaExceededError,TriggerWorkflowServiceModeUnavailableError,@@ -33,6 +33,7 @@ from services.errors.app import ()from services.errors.llm import InvokeRateLimitErrorfrom services.quota_service import QuotaService, unlimited+from services.workflow_run_agg import WorkflowRunAggfrom services.workflow_service import WorkflowServicefrom tasks.app_generate.workflow_execute_task import AppExecutionParams, workflow_based_app_execution_task@@ -204,7 +205,7 @@ class AppGenerateService:case AppMode.AGENT_CHAT:return rate_limit.generate(AgentChatAppGenerator.convert_to_event_stream(- AgentChatAppGenerator().generate(+ AgentChatAppGenerator(execution_driver=WorkflowRunAgg.run).generate(session=session,app_model=app_model,user=user,@@ -266,10 +267,9 @@ class AppGenerateService:workflow_based_app_execution_task.delay(payload_json)on_subscribe = cls._build_streaming_task_on_subscribe(on_subscribe)- generator = AdvancedChatAppGenerator()return rate_limit.generate(- generator.convert_to_event_stream(- generator.retrieve_events(+ AdvancedChatAppGenerator.convert_to_event_stream(+ AdvancedChatAppGenerator.retrieve_events(AppMode.ADVANCED_CHAT,payload.workflow_run_id,on_subscribe=on_subscribe,@@ -284,7 +284,7 @@ class AppGenerateService:session_factory=session_factory.get_session_maker(),state_owner_user_id=workflow.created_by,)- advanced_generator = AdvancedChatAppGenerator()+ advanced_generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)return rate_limit.generate(advanced_generator.convert_to_event_stream(advanced_generator.generate(@@ -341,7 +341,7 @@ class AppGenerateService:)return rate_limit.generate(WorkflowAppGenerator.convert_to_event_stream(- WorkflowAppGenerator().generate(+ WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate(app_model=app_model,workflow=workflow,user=user,@@ -401,7 +401,7 @@ class AppGenerateService:case AppMode.ADVANCED_CHAT:workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session)return AdvancedChatAppGenerator.convert_to_event_stream(- AdvancedChatAppGenerator().single_iteration_generate(+ AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run).single_iteration_generate(app_model=app_model,workflow=workflow,node_id=node_id,@@ -414,7 +414,7 @@ class AppGenerateService:case AppMode.WORKFLOW:workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session)return AdvancedChatAppGenerator.convert_to_event_stream(- WorkflowAppGenerator().single_iteration_generate(+ WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_iteration_generate(app_model=app_model,workflow=workflow,node_id=node_id,@@ -446,7 +446,7 @@ class AppGenerateService:case AppMode.ADVANCED_CHAT:workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session)return AdvancedChatAppGenerator.convert_to_event_stream(- AdvancedChatAppGenerator().single_loop_generate(+ AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run).single_loop_generate(app_model=app_model,workflow=workflow,node_id=node_id,@@ -459,7 +459,7 @@ class AppGenerateService:case AppMode.WORKFLOW:workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session)return AdvancedChatAppGenerator.convert_to_event_stream(- WorkflowAppGenerator().single_loop_generate(+ WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_loop_generate(app_model=app_model,workflow=workflow,node_id=node_id,@@ -548,19 +548,3 @@ class AppGenerateService:raise ValueError("Workflow not published")return workflow-- @classmethod- def get_response_generator(- cls,- app_model: App,- workflow_run: WorkflowRun,- ):- if workflow_run.status.is_ended():- # TODO(QuantumGhost): handled the ended scenario.- pass-- generator = AdvancedChatAppGenerator()-- return generator.convert_to_event_stream(- generator.retrieve_events(AppMode(app_model.mode), workflow_run.id),- )diff --git a/api/services/app_task_service.py b/api/services/app_task_service.pyindex 453ce4d9bc0..3e5ac31f462 100644--- a/api/services/app_task_service.py+++ b/api/services/app_task_service.py@@ -2,15 +2,41 @@This service provides centralized logic for task control operationslike stopping tasks, handling both legacy Redis flag mechanism and-new GraphEngine command channel mechanism.+new Engine command channel mechanism."""+import json+import logging+from uuid import UUID++from sqlalchemy import select+from sqlalchemy.orm import sessionmaker+from core.app.apps.base_app_queue_manager import AppQueueManager-from core.app.apps.execution_coordinator import set_app_task_stop_flag-from core.app.entities.app_invoke_entities import InvokeFrom+from core.app.apps.execution_coordinator import send_abort_command, set_app_task_stop_flag+from core.app.apps.message_based_app_generator import MessageBasedAppGenerator+from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom+from core.app.entities.task_entities import WorkflowFinishStreamResponse+from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext+from core.repositories.factory import DifyCoreRepositoryFactory+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from extensions.ext_database import dbfrom extensions.ext_redis import RedisClientWrapper, redis_client-from graphon.graph_engine.manager import GraphEngineManager+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType+from libs.datetime_utils import to_utc_timestamp+from models import Account, EndUser, WorkflowRun+from models.enums import CreatorUserRolefrom models.model import AppMode+from models.workflow import WorkflowNodeExecutionModel+from repositories.api_workflow_run_repository import APIWorkflowRunRepository+from repositories.entities.workflow_execution import WorkflowExecution+from repositories.entities.workflow_pause import WorkflowPauseEntity+from repositories.sqlalchemy_api_workflow_run_repository import (+ DifyAPISQLAlchemyWorkflowRunRepository,+ WorkflowTaskOwnerMismatchError,+)++logger = logging.getLogger(__name__)class AppTaskControlService:@@ -24,54 +50,313 @@ class AppTaskControlService:task_id: str,invoke_from: InvokeFrom,user_id: str,- app_mode: AppMode,- ) -> None:- """Stop a running task.+ ) -> bool:+ """Stop a running non-workflow task through the Redis stop flag.- This method handles stopping tasks using both mechanisms:- 1. Legacy Redis flag mechanism (for backward compatibility)- 2. New GraphEngine command channel (for workflow-based apps)+ Workflow-based apps stop through ``stop_workflow_task``, which also+ sends the Engine abort command.Args:task_id: The task ID to stopinvoke_from: The source of the invoke (e.g., DEBUGGER, WEB_APP, SERVICE_API)user_id: The user ID requesting the stop- app_mode: The application mode (CHAT, AGENT_CHAT, ADVANCED_CHAT, WORKFLOW, etc.)-- Returns:- None"""- # Legacy mechanism: Set stop flag in Redis- AppQueueManager.set_stop_flag(task_id, invoke_from, user_id, redis=self._redis_client)+ return AppQueueManager.set_stop_flag(task_id, invoke_from, user_id, redis=self._redis_client)- # New mechanism: Send stop command via GraphEngine for workflow-based apps- # This ensures proper workflow status recording in the persistence layer- if app_mode in (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW):- GraphEngineManager(self._redis_client).send_stop_command(task_id)+ def stop_app_task(+ self,+ *,+ tenant_id: str,+ app_id: str,+ task_id: str,+ app_mode: AppMode,+ caller_role: CreatorUserRole,+ caller_id: str,+ ) -> bool:+ """Stop after app admission, restricting all end users to their own tasks."""+ owner = (caller_role, caller_id) if caller_role == CreatorUserRole.END_USER else None+ if app_mode in (AppMode.WORKFLOW, AppMode.ADVANCED_CHAT):+ admitted = self.stop_workflow_task(+ tenant_id=tenant_id, app_id=app_id, task_id=task_id, app_mode=app_mode, owner=owner+ )+ return admitted or owner is None+ if owner is not None:+ return self.stop_task(task_id, InvokeFrom.OPENAPI, caller_id)+ set_app_task_stop_flag(task_id, redis=self._redis_client)+ return Truedef stop_workflow_task_no_user_check(self, *, task_id: str) -> None:"""Stop a workflow after app admission, without consulting the user ownership cache.- Keep the legacy stop flag before the GraphEngine command, including when+ Keep the legacy stop flag before the Engine command, including whenthe latter fails. Callers must authorize access to the workflow app first."""set_app_task_stop_flag(task_id, redis=self._redis_client)- GraphEngineManager(self._redis_client).send_stop_command(task_id)+ send_abort_command(task_id, redis=self._redis_client)++ def stop_workflow_task(+ self,+ *,+ tenant_id: str,+ app_id: str,+ task_id: str,+ app_mode: AppMode,+ owner: tuple[CreatorUserRole, str] | None,+ ) -> bool:+ """Apply durable workflow admission while retaining this service's Redis transport."""+ return AppTaskService.stop_workflow_task(+ tenant_id=tenant_id,+ app_id=app_id,+ task_id=task_id,+ app_mode=app_mode,+ owner=owner,+ redis=self._redis_client,+ )class AppTaskService:"""Compatibility entry point for callers outside ApplicationServices."""+ @staticmethod+ def index_workflow_pause_task(+ *,+ workflow_run: WorkflowRun,+ pause_entity: WorkflowPauseEntity,+ resumption_context: WorkflowResumptionContext,+ workflow_run_repo: APIWorkflowRunRepository,+ ) -> bool:+ """Recover a missing task ID only from the same run's saved invocation."""+ # TODO: Remove legacy task recovery 30 days after the last legacy writer is retired.+ if workflow_run.task_id is not None:+ return False+ generate_entity = resumption_context.get_generate_entity()+ try:+ task_id = str(UUID(generate_entity.task_id))+ except ValueError:+ return False+ run_id = (+ generate_entity.workflow_run_id+ if isinstance(generate_entity, AdvancedChatAppGenerateEntity)+ else generate_entity.workflow_execution_id+ )+ if (+ run_id != workflow_run.id+ or generate_entity.app_config is None+ or generate_entity.app_config.tenant_id != workflow_run.tenant_id+ or generate_entity.app_config.app_id != workflow_run.app_id+ or generate_entity.user_id != workflow_run.created_by+ ):+ return False+ return workflow_run_repo.backfill_workflow_pause_task_id(+ tenant_id=workflow_run.tenant_id,+ app_id=workflow_run.app_id,+ workflow_run_id=workflow_run.id,+ pause_id=pause_entity.id,+ task_id=task_id,+ )++ @staticmethod+ def _recover_legacy_workflow_task_id(+ *,+ tenant_id: str,+ app_id: str,+ task_id: str,+ owner: tuple[CreatorUserRole, str] | None,+ repository: DifyAPISQLAlchemyWorkflowRunRepository,+ ) -> None:+ # TODO: Remove legacy task recovery 30 days after the last legacy writer is retired.+ after_id: str | None = None+ while pauses := repository.list_legacy_workflow_pauses(+ tenant_id=tenant_id, app_id=app_id, owner=owner, after_id=after_id, limit=100+ ):+ for run, pause in pauses:+ try:+ context = WorkflowResumptionContext.loads(pause.get_state().decode())+ saved_task_id = str(UUID(context.get_generate_entity().task_id))+ except Exception:+ logger.exception("Failed to recover legacy workflow task, workflow_run_id=%s", run.id)+ continue+ if saved_task_id == task_id and AppTaskService.index_workflow_pause_task(+ workflow_run=run,+ pause_entity=pause,+ resumption_context=context,+ workflow_run_repo=repository,+ ):+ return+ after_id = pauses[-1][1].id++ @staticmethod+ def stop_workflow_task(+ *,+ tenant_id: str,+ app_id: str,+ task_id: str,+ app_mode: AppMode,+ owner: tuple[CreatorUserRole, str] | None,+ redis: RedisClientWrapper | None = None,+ ) -> bool:+ """Stop a persisted pause, or signal its currently executing attempt.++ A paused task has no engine consuming Redis commands. Its durable+ transition and form invalidation must complete before acknowledging stop.+ Omitted Redis uses the compatibility transport; injected callers supply theirs.+ """+ try:+ task_id = str(UUID(task_id))+ except ValueError as exc:+ raise ValueError("Invalid task ID: expected a UUID") from exc+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sessionmaker(bind=db.engine))+ try:+ run = repository.stop_paused_workflow_task(tenant_id=tenant_id, app_id=app_id, task_id=task_id, owner=owner)+ if run is None:+ AppTaskService._recover_legacy_workflow_task_id(+ tenant_id=tenant_id, app_id=app_id, task_id=task_id, owner=owner, repository=repository+ )+ run = repository.stop_paused_workflow_task(+ tenant_id=tenant_id, app_id=app_id, task_id=task_id, owner=owner+ )+ except WorkflowTaskOwnerMismatchError:+ return False+ if run is not None and run.status == WorkflowExecutionStatus.RUNNING:+ # Resume has claimed the pause, but may not have opened its queue.+ # The repository has already checked the complete app/user owner.+ AppTaskControlService(+ redis_client=redis if redis is not None else redis_client+ ).stop_workflow_task_no_user_check(task_id=task_id)+ return True+ if run is None:+ return False++ AppTaskService._persist_stopped_history(run)++ # The paused engine cannot execute its terminal Workspace layer. Keep+ # run-owned Agent resources on the same retirement path as a live abort.+ from core.workflow.nodes.agent_v2.session_store import WorkflowAgentWorkspaceStore+ from tasks.collect_agent_resources_task import enqueue_agent_resource_collection++ try:+ workspace_ids = WorkflowAgentWorkspaceStore().retire_workflow_run(+ tenant_id=tenant_id, app_id=app_id, workflow_run_id=run.id+ )+ enqueue_agent_resource_collection(tenant_id=tenant_id, workspace_ids=workspace_ids)+ except Exception:+ logger.exception("Failed to retire stopped Workflow Agent Workspaces, workflow_run_id=%s", run.id)++ event = WorkflowFinishStreamResponse(+ task_id=task_id,+ workflow_run_id=run.id,+ data=WorkflowFinishStreamResponse.Data(+ id=run.id,+ workflow_id=run.workflow_id,+ status=run.status,+ outputs=run.outputs_dict,+ error=run.error,+ elapsed_time=run.elapsed_time,+ total_tokens=run.total_tokens,+ total_steps=run.total_steps,+ created_by={},+ created_at=int(to_utc_timestamp(run.created_at)),+ finished_at=int(to_utc_timestamp(run.finished_at)) if run.finished_at is not None else None,+ exceptions_count=run.exceptions_count,+ files=[],+ ),+ )+ topic = MessageBasedAppGenerator.get_response_topic(app_mode, run.id)+ topic.publish(json.dumps(event.model_dump(mode="json"), ensure_ascii=False).encode())+ return True++ @staticmethod+ def _persist_stopped_history(run: WorkflowRun) -> None:+ # The SQL stop transaction is already committed. Persist the terminal+ # history through the configured backend without holding its run lock.+ user = Account(name="", email="") if run.created_by_role == CreatorUserRole.ACCOUNT else EndUser()+ user.id = run.created_by+ repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(+ session_factory=sessionmaker(bind=db.engine),+ tenant_id=run.tenant_id,+ user=user,+ app_id=run.app_id,+ triggered_from=run.triggered_from,+ )+ repository.save_synchronously(+ WorkflowExecution(+ id_=run.id,+ workflow_id=run.workflow_id,+ workflow_type=WorkflowType(run.type),+ workflow_version=run.version,+ graph=run.graph_dict,+ inputs=run.inputs_dict,+ outputs=run.outputs_dict,+ status=WorkflowExecutionStatus.STOPPED,+ error_message=run.error or "User requested stop",+ total_tokens=run.total_tokens,+ total_steps=run.total_steps,+ exceptions_count=run.exceptions_count,+ started_at=run.created_at,+ accumulated_elapsed_time=run.elapsed_time,+ finished_at=run.finished_at,+ )+ )+ factory = sessionmaker(bind=db.engine)+ with factory() as session:+ scopes = session.execute(+ select(WorkflowNodeExecutionModel.app_id, WorkflowNodeExecutionModel.triggered_from)+ .where(+ WorkflowNodeExecutionModel.tenant_id == run.tenant_id,+ WorkflowNodeExecutionModel.workflow_run_id == run.id,+ WorkflowNodeExecutionModel.status == WorkflowNodeExecutionStatus.FAILED,+ WorkflowNodeExecutionModel.finished_at == run.finished_at,+ WorkflowNodeExecutionModel.error == run.error,+ )+ .distinct()+ ).all()+ for app_id, triggered_from in scopes:+ source = SQLAlchemyWorkflowNodeExecutionRepository(+ session_factory=factory,+ tenant_id=run.tenant_id,+ user=user,+ app_id=app_id,+ triggered_from=triggered_from,+ )+ stopped_nodes = [+ execution+ for execution in source.get_by_workflow_execution(run.id, include_paused=True)+ if execution.status == WorkflowNodeExecutionStatus.FAILED+ and execution.finished_at == run.finished_at+ and execution.error == run.error+ ]+ target = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(+ session_factory=factory,+ tenant_id=run.tenant_id,+ user=user,+ app_id=app_id,+ triggered_from=triggered_from,+ )+ for execution in stopped_nodes:+ target.save(execution)+@staticmethoddef stop_task(task_id: str,invoke_from: InvokeFrom,user_id: str,app_mode: AppMode,+ *,+ tenant_id: str,+ app_id: str,) -> None:+ if app_mode in (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW):+ role = CreatorUserRole.ACCOUNT if invoke_from.runs_as_account() else CreatorUserRole.END_USER+ AppTaskService.stop_workflow_task(+ tenant_id=tenant_id,+ app_id=app_id,+ task_id=task_id,+ app_mode=app_mode,+ owner=(role, user_id),+ )+ returnAppTaskControlService(redis_client=redis_client).stop_task(task_id=task_id,invoke_from=invoke_from,user_id=user_id,- app_mode=app_mode,)diff --git a/api/services/human_input_service.py b/api/services/human_input_service.pyindex 416c3a0c5d2..c577a872fca 100644--- a/api/services/human_input_service.py+++ b/api/services/human_input_service.py@@ -21,6 +21,7 @@ from core.workflow.nodes.human_input.entities import (FormDefinition,FormInputConfig,HumanInputSubmissionValidationError,+ ParagraphInputConfig,SelectInputConfig,UserActionConfig,)@@ -29,14 +30,16 @@ from core.workflow.nodes.human_input.entities import ()from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus, ValueSourceTypefrom factories.file_factory import build_from_mapping, build_from_mappings+from graphon.enums import WorkflowExecutionStatusfrom graphon.file import FileUploadConfig-from graphon.runtime import GraphRuntimeState-from graphon.runtime.graph_runtime_state_protocol import ReadOnlyVariablePool+from graphon.runtime import RuntimeState+from graphon.runtime.runtime_state_protocol import ReadOnlyVariablePoolfrom libs.datetime_utils import ensure_naive_utc, naive_utc_nowfrom libs.exception import BaseHTTPExceptionfrom models.human_input import RecipientTypefrom models.model import App, AppModefrom repositories.factory import DifyAPIRepositoryFactory+from repositories.human_input_errors import FormAlreadyHandledErrorfrom tasks.app_generate.workflow_execute_task import resume_app_execution_file_access_controller = DatabaseFileAccessController()@@ -186,11 +189,28 @@ class HumanInputService:return formdef resolve_form_inputs(self, form: Form) -> Sequence[FormInputConfig]:- variable_pool = self._load_variable_pool_for_form(form)- return resolve_variable_select_input_options(- form.get_definition().inputs,- variable_pool=variable_pool,- )+ definition = form.get_definition()+ inputs = list(definition.inputs)+ if any(+ isinstance(form_input, SelectInputConfig) and form_input.option_source.type == ValueSourceType.VARIABLE+ for form_input in inputs+ ):+ inputs = resolve_variable_select_input_options(+ inputs,+ variable_pool=self._load_variable_pool_for_form(form),+ )+ return [+ form_input.model_copy(+ update={+ "option_source": form_input.option_source.model_copy(+ update={"type": ValueSourceType.CONSTANT, "selector": ()}+ )+ }+ )+ if isinstance(form_input, SelectInputConfig) and form_input.option_source.type == ValueSourceType.VARIABLE+ else form_input+ for form_input in inputs+ ]def submit_form_by_token(self,@@ -212,14 +232,19 @@ class HumanInputService:form_data=form_data,)- result = self._form_repository.mark_submitted(- form_id=form.id,- recipient_id=form.recipient_id,- selected_action_id=selected_action_id,- form_data=normalized_form_data,- submission_user_id=submission_user_id,- submission_end_user_id=submission_end_user_id,- )+ try:+ result = self._form_repository.mark_submitted(+ form_id=form.id,+ recipient_id=form.recipient_id,+ selected_action_id=selected_action_id,+ form_data=normalized_form_data,+ submission_user_id=submission_user_id,+ submission_end_user_id=submission_end_user_id,+ )+ except FormAlreadyHandledError as exc:+ if exc.status == HumanInputFormStatus.SUBMITTED:+ raise FormSubmittedError(form.id) from exc+ raise FormExpiredError(form.id) from excif result.form_kind != HumanInputFormKind.RUNTIME:return@@ -249,6 +274,7 @@ class HumanInputService:form_data: Mapping[str, Any],) -> dict[str, JsonValue]:definition = form.get_definition()+ definition = definition.model_copy(update={"inputs": self.resolve_form_inputs(form)})try:return self.validate_and_normalize_submission(tenant_id=form.tenant_id,@@ -265,6 +291,8 @@ class HumanInputService:if workflow_run is None:raise AssertionError(f"WorkflowRun not found, id={workflow_run_id}")+ if workflow_run.status == WorkflowExecutionStatus.STOPPED:+ returnwith self._session_factory(expire_on_commit=False) as session:app_query = select(App).where(App.id == workflow_run.app_id)app = session.execute(app_query).scalar_one_or_none()@@ -298,7 +326,7 @@ class HumanInputService:return Noneresumption_context = WorkflowResumptionContext.loads(pause_entity.get_state().decode())- runtime_state = GraphRuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)+ runtime_state = RuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)return runtime_state.variable_pool@@ -370,6 +398,12 @@ class HumanInputService:form_input: FormInputConfig,value: Any,) -> JsonValue:+ if isinstance(form_input, ParagraphInputConfig):+ if not isinstance(value, str):+ raise HumanInputSubmissionValidationError(+ f"Invalid value for paragraph input '{form_input.output_variable_name}': expected string"+ )+ return valueif isinstance(form_input, SelectInputConfig):return cls._normalize_select_value(form_input=form_input, value=value)if isinstance(form_input, FileInputConfig):diff --git a/api/services/rag_pipeline/rag_pipeline.py b/api/services/rag_pipeline/rag_pipeline.pyindex 1acc676fe75..c91932e6305 100644--- a/api/services/rag_pipeline/rag_pipeline.py+++ b/api/services/rag_pipeline/rag_pipeline.py@@ -45,10 +45,10 @@ from core.workflow.variable_pool_initializer import add_variables_to_poolfrom core.workflow.workflow_entry import WorkflowEntryfrom enterprise.telemetry.draft_trace import enqueue_draft_node_execution_tracefrom extensions.ext_database import db+from graphon.engine_events import NodeEvent, NodeRunFailedEvent, NodeRunSucceededEventfrom graphon.entities import WorkflowNodeExecutionfrom graphon.enums import BuiltinNodeTypes, ErrorStrategy, NodeType, WorkflowNodeExecutionStatusfrom graphon.errors import WorkflowNodeRunFailedError-from graphon.graph_events import GraphNodeEventBase, NodeRunFailedEvent, NodeRunSucceededEventfrom graphon.node_events import NodeRunResultfrom graphon.nodes.base.node import Nodefrom graphon.nodes.container_effects import ContainerAwaitRequest@@ -96,6 +96,7 @@ from services.workflow_node_execution_trace_service import ()from services.workflow_ref_service import WorkflowReffrom services.workflow_restore import apply_published_workflow_snapshot_to_draft+from services.workflow_run_agg import WorkflowRunAgglogger = logging.getLogger(__name__)@@ -600,6 +601,7 @@ class RagPipelineService:workflow_node_execution = self._handle_node_run_result(getter=lambda: WorkflowEntry.single_step_run(+ execution_driver=WorkflowRunAgg.run,workflow=draft_workflow,node_id=node_id,user_inputs=user_inputs,@@ -949,7 +951,7 @@ class RagPipelineService:self,getter: Callable[[],- tuple[Node, Generator[GraphNodeEventBase | ContainerAwaitRequest]],+ tuple[Node, Generator[NodeEvent | ContainerAwaitRequest]],],start_at: float,tenant_id: str,@@ -1048,7 +1050,7 @@ class RagPipelineService:workflow_node_execution.status = WorkflowNodeExecutionStatus.FAILEDworkflow_node_execution.error = error# update document status- variable_pool = node_instance.graph_runtime_state.variable_pool+ variable_pool = node_instance.runtime_state.variable_poolinvoke_from = get_system_segment(variable_pool, SystemVariableKey.INVOKE_FROM)if invoke_from:if invoke_from.value == InvokeFrom.PUBLISHED_PIPELINE:@@ -1400,6 +1402,7 @@ class RagPipelineService:workflow_node_execution = self._handle_node_run_result(getter=lambda: WorkflowEntry.single_step_run(+ execution_driver=WorkflowRunAgg.run,workflow=draft_workflow,node_id=node_id,user_inputs={},diff --git a/api/services/snippet_generate_service.py b/api/services/snippet_generate_service.pyindex 86177f0112f..2b31cbce4bf 100644--- a/api/services/snippet_generate_service.py+++ b/api/services/snippet_generate_service.py@@ -32,11 +32,13 @@ from core.app.file_access import DatabaseFileAccessControllerfrom core.workflow.snippet_start import SNIPPET_VIRTUAL_START_NODE_IDfrom factories import file_factoryfrom graphon.file.models import File+from graphon.graph.scoping import resolve_container_idfrom models import Accountfrom models.model import App, AppMode, EndUserfrom models.snippet import CustomizedSnippetfrom models.workflow import Workflow, WorkflowNodeExecutionModelfrom services.snippet_service import SnippetService+from services.workflow_run_agg import WorkflowRunAggfrom services.workflow_service import WorkflowServicelogger = logging.getLogger(__name__)@@ -167,7 +169,7 @@ class SnippetGenerateService:# Adapt snippet to App-like interface for WorkflowAppGeneratorapp_proxy = cast(App, _SnippetAsApp(snippet))- response = WorkflowAppGenerator().generate(+ response = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate(app_model=app_proxy,workflow=workflow,user=user,@@ -212,7 +214,7 @@ class SnippetGenerateService:app_proxy = cast(App, _SnippetAsApp(snippet))- response = WorkflowAppGenerator().generate(+ response = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate(app_model=app_proxy,workflow=workflow,user=user,@@ -265,8 +267,8 @@ class SnippetGenerateService:"""Build a new graph dict with a virtual Start node prepended.- The virtual Start node is wired to every existing node that has no- incoming edges (i.e. the current root candidates). This guarantees:+ The virtual Start node is wired to top-level nodes with no incoming+ edges. Container children retain their own entry points.:param graph_dict: Original graph configuration.:param input_fields: Snippet input field definitions from@@ -276,13 +278,18 @@ class SnippetGenerateService:nodes: list[dict[str, Any]] = list(graph_dict.get("nodes", []))edges: list[dict[str, Any]] = list(graph_dict.get("edges", []))- # Identify nodes with no incoming edges.+ nodes_by_id = {node["id"]: node for node in nodes}+ # A container's entry node has no incoming edges, but is not a graph root.nodes_with_incoming: set[str] = set()for edge in edges:target = edge.get("target")if isinstance(target, str):nodes_with_incoming.add(target)- root_candidate_ids = [n["id"] for n in nodes if n["id"] not in nodes_with_incoming]+ root_candidate_ids = [+ node["id"]+ for node in nodes+ if node["id"] not in nodes_with_incoming and not resolve_container_id(node, nodes_by_id=nodes_by_id)+ ]# Build Start node ``variables`` from snippet input fields.start_variables: list[dict[str, Any]] = []@@ -404,7 +411,7 @@ class SnippetGenerateService:with session_maker() as session:return WorkflowAppGenerator.convert_to_event_stream(- WorkflowAppGenerator().single_iteration_generate(+ WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_iteration_generate(app_model=app_proxy,workflow=workflow,node_id=node_id,@@ -451,7 +458,7 @@ class SnippetGenerateService:with session_maker() as session:return WorkflowAppGenerator.convert_to_event_stream(- WorkflowAppGenerator().single_loop_generate(+ WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_loop_generate(app_model=app_proxy,workflow=workflow,node_id=node_id,diff --git a/api/services/workflow/inspector_events.py b/api/services/workflow/inspector_events.pyindex c6d3730d119..6678d1829a1 100644--- a/api/services/workflow/inspector_events.py+++ b/api/services/workflow/inspector_events.py@@ -6,7 +6,7 @@ DB polling. This module owns the redis pub/sub channel that connects the twosides:* :func:`publish_node_changed` / :func:`publish_workflow_completed` —- invoked by :class:`core.app.workflow.layers.persistence.WorkflowPersistenceLayer`+ invoked by :class:`services.workflow_persistence.WorkflowPersistenceLayer`at the very end of each handler, after the DB write has alreadysucceeded. Publish failures are swallowed so the engine never trips on aflaky redis connection.diff --git a/api/services/workflow_event_snapshot_service.py b/api/services/workflow_event_snapshot_service.pyindex 0813e43a5d9..5ae08102a9a 100644--- a/api/services/workflow_event_snapshot_service.py+++ b/api/services/workflow_event_snapshot_service.py@@ -43,8 +43,8 @@ from core.workflow.nodes.human_input.pause_reason import ()from graphon.entities import WorkflowStartReasonfrom graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus-from graphon.runtime import GraphRuntimeState-from graphon.runtime.graph_runtime_state_protocol import ReadOnlyVariablePool+from graphon.runtime import RuntimeState+from graphon.runtime.runtime_state_protocol import ReadOnlyVariablePoolfrom graphon.workflow_type_encoder import WorkflowRuntimeTypeConverterfrom libs.broadcast_channel.exc import SubscriptionClosedErrorfrom libs.datetime_utils import to_utc_timestamp@@ -54,6 +54,7 @@ from models.workflow import WorkflowNodeExecutionTriggeredFrom, WorkflowRunfrom repositories.api_workflow_node_execution_repository import WorkflowNodeExecutionSnapshotfrom repositories.entities.workflow_pause import WorkflowPauseEntityfrom repositories.factory import DifyAPIRepositoryFactory+from services.app_task_service import AppTaskServicelogger = logging.getLogger(__name__)@@ -100,6 +101,18 @@ def build_workflow_event_stream(pause_entity = Noneresumption_context = _load_resumption_context(pause_entity)+ if (+ pause_entity is not None+ and resumption_context is not None+ and workflow_run.tenant_id == tenant_id+ and workflow_run.app_id == app_id+ ):+ AppTaskService.index_workflow_pause_task(+ workflow_run=workflow_run,+ pause_entity=pause_entity,+ resumption_context=resumption_context,+ workflow_run_repo=workflow_run_repo,+ )message_context: MessageContext | None = Noneif app_mode == AppMode.ADVANCED_CHAT:if workflow_run.status == WorkflowExecutionStatus.PAUSED:@@ -543,7 +556,7 @@ def _load_variable_pool_from_resumption_context() -> ReadOnlyVariablePool | None:if resumption_context is None:return None- state = GraphRuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)+ state = RuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)return state.variable_pool@@ -597,7 +610,7 @@ def _build_pause_event(outputs: dict[str, Any] = {}variable_pool: ReadOnlyVariablePool | None = Noneif resumption_context is not None:- state = GraphRuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)+ state = RuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)outputs = dict(WorkflowRuntimeTypeConverter().to_json_encodable(state.outputs or {}))variable_pool = state.variable_pooldiff --git a/api/services/workflow_node_variables.py b/api/services/workflow_node_variables.pynew file mode 100644index 00000000000..85371ec1135--- /dev/null+++ b/api/services/workflow_node_variables.py@@ -0,0 +1,155 @@+"""Load editor inputs not exposed by Graphon's node selector extractors."""++from collections.abc import Mapping, Sequence+from typing import Any++from core.workflow.human_input_adapter import (+ EmailDeliveryMethod,+ adapt_node_config_for_graph,+ parse_human_input_delivery_methods,+)+from core.workflow.nodes.human_input.entities import HumanInputNodeData+from core.workflow.nodes.knowledge_index import KNOWLEDGE_INDEX_NODE_TYPE+from core.workflow.nodes.knowledge_index.entities import KnowledgeIndexNodeData+from graphon.entities.graph_config import NodeConfigDict, NodeConfigDictAdapter+from graphon.enums import BuiltinNodeTypes+from graphon.nodes.base.variable_template_parser import VariableTemplateParser+from graphon.nodes.end.entities import EndNodeData+from graphon.nodes.list_operator.entities import FilterOperator, ListOperatorNodeData+from graphon.nodes.variable_aggregator.entities import VariableAggregatorNodeData+from graphon.runtime import VariablePool+from graphon.variable_loader import VariableLoader, load_into_variable_pool+++def get_human_input_form_variable_mapping(node_id: str, node_data: HumanInputNodeData) -> Mapping[str, Sequence[str]]:+ """Keep editor template paths intact alongside form-field dependency selectors."""+ selectors = [+ selector.value_selector+ for selector in VariableTemplateParser(template=node_data.form_content).extract_variable_selectors()+ if len(selector.value_selector) >= 2+ ]+ selectors.extend(+ selector for form_input in node_data.inputs for selector in form_input.extract_variable_selectors()+ )+ return {f"{node_id}.#{'.'.join(selector)}#": selector for selector in selectors}+++def _input_mappings(config: NodeConfigDict) -> Mapping[str, Sequence[str]]:+ node_id = config["id"]+ data = config["data"]+ selectors: list[Sequence[str]] = []+ match data.type:+ case BuiltinNodeTypes.END:+ end = EndNodeData.model_validate(data.model_dump())+ selectors.extend(output.value_selector for output in end.outputs)+ case BuiltinNodeTypes.VARIABLE_AGGREGATOR:+ aggregator = VariableAggregatorNodeData.model_validate(data.model_dump())+ if aggregator.advanced_settings and aggregator.advanced_settings.group_enabled:+ selectors.extend(+ selector for group in aggregator.advanced_settings.groups for selector in group.variables+ )+ else:+ selectors.extend(aggregator.variables)+ case BuiltinNodeTypes.LIST_OPERATOR:+ operator = ListOperatorNodeData.model_validate(data.model_dump())+ selectors.append(operator.variable)+ templates = []+ if operator.filter_by.enabled:+ templates.extend(+ condition.value+ for condition in operator.filter_by.conditions+ if condition.comparison_operator not in (FilterOperator.EMPTY, FilterOperator.NOT_EMPTY)+ and isinstance(condition.value, str)+ )+ if operator.extract_by.enabled:+ templates.append(operator.extract_by.serial)+ selectors.extend(+ selector.value_selector+ for template in templates+ for selector in VariableTemplateParser(template=template).extract_variable_selectors()+ )+ case BuiltinNodeTypes.HUMAN_INPUT:+ human_input = HumanInputNodeData.model_validate(data.model_dump())+ mapping = dict(get_human_input_form_variable_mapping(node_id, human_input))+ for method in parse_human_input_delivery_methods(data):+ if method.enabled and isinstance(method, EmailDeliveryMethod):+ mapping.update(+ {+ f"{node_id}.#{'.'.join(selector.value_selector)}#": selector.value_selector+ for selector in VariableTemplateParser(+ template=method.config.body+ ).extract_variable_selectors()+ if len(selector.value_selector) >= 2+ }+ )+ return mapping+ case _ if data.type == KNOWLEDGE_INDEX_NODE_TYPE:+ knowledge_index = KnowledgeIndexNodeData.model_validate(data.model_dump())+ return {f"{node_id}.query": knowledge_index.index_chunk_variable_selector}+ return {f"{node_id}.#{'.'.join(selector)}#": selector for selector in selectors}+++def load_additional_node_variables(+ *,+ node_config: NodeConfigDict,+ graph_config: Mapping[str, Any],+ variable_mapping: Mapping[str, Sequence[str]],+ variable_loader: VariableLoader,+ variable_pool: VariablePool,+ user_inputs: Mapping[str, Any],+) -> Mapping[str, Sequence[str]]:+ """Supplement debug inputs, leaving unavailable branches and End outputs unset.++ Container callers supply their already scoped debug subtree. References to+ nodes within it are produced at runtime instead of loaded as external inputs.+ """+ is_container = node_config["data"].type in (BuiltinNodeTypes.ITERATION, BuiltinNodeTypes.LOOP)+ configs = (+ [NodeConfigDictAdapter.validate_python(adapt_node_config_for_graph(node)) for node in graph_config["nodes"]]+ if is_container+ else [node_config]+ )+ internal_ids = {config["id"] for config in configs} if is_container else set()+ additional: dict[str, Sequence[str]] = {}+ optional_keys: set[str] = set()+ for config in configs:+ prefix = f"{config['id']}." if config["id"] != node_config["id"] else ""+ for key, selector in _input_mappings(config).items():+ key = prefix + key+ if key in variable_mapping or (selector and selector[0] in internal_ids):+ continue+ additional[key] = selector+ if config["data"].type in (BuiltinNodeTypes.VARIABLE_AGGREGATOR, BuiltinNodeTypes.END):+ optional_keys.add(key)+ if not additional:+ return variable_mapping++ load_into_variable_pool(variable_loader, variable_pool, additional, user_inputs)+ return {+ **variable_mapping,+ **{+ key: selector+ for key, selector in additional.items()+ if key not in optional_keys+ or key in user_inputs+ or key.split(".", 1)[1] in user_inputs+ or variable_pool.get(selector) is not None+ },+ }+++def add_node_input_to_variable_pool(variable_pool: VariablePool, selector: Sequence[str], value: Any) -> None:+ """Merge an explicit nested input into its root variable without losing siblings."""+ root_selector = list(selector[:2])+ if len(selector) > 2:+ current_variable = variable_pool.get(root_selector)+ current_value = current_variable.value if current_variable is not None else None+ root_value = dict(current_value) if isinstance(current_value, dict) else {}+ parent = root_value+ for key in selector[2:-1]:+ child = parent.get(key)+ parent[key] = dict(child) if isinstance(child, dict) else {}+ parent = parent[key]+ parent[selector[-1]] = value+ value = root_value+ variable_pool.add(root_selector, value)diff --git a/api/core/app/workflow/layers/persistence.py b/api/services/workflow_persistence.pysimilarity index 78%rename from api/core/app/workflow/layers/persistence.pyrename to api/services/workflow_persistence.pyindex 8b3119c1387..de40d24ad30 100644--- a/api/core/app/workflow/layers/persistence.py+++ b/api/services/workflow_persistence.py@@ -1,15 +1,10 @@-"""Workflow persistence layer for GraphEngine.+"""Persist workflow execution events on behalf of the upper run orchestrator.-This layer mirrors the former ``WorkflowCycleManager`` responsibilities by-listening to ``GraphEngineEvent`` instances directly and persisting workflow-and node execution state via the injected repositories.--The design keeps domain persistence concerns inside the engine thread, while-allowing presentation layers to remain read-only observers of repository-state.+The aggregate constructs this observer and supplies its repositories, indexes,+and history before the engine starts. Engine events remain storage-agnostic."""-from collections.abc import Mapping+from collections.abc import Mapping, Sequencefrom dataclasses import dataclassfrom datetime import datetimefrom typing import Any, override@@ -24,21 +19,12 @@ from core.workflow.node_execution_process_data import preserve_workflow_agent_bifrom core.workflow.system_variables import SystemVariableKeyfrom core.workflow.variable_prefixes import SYSTEM_VARIABLE_NODE_IDfrom core.workflow.workflow_run_outputs import project_node_outputs_for_workflow_run-from graphon.entities import WorkflowNodeExecution, WorkflowStartReason-from graphon.enums import (- BuiltinNodeTypes,- WorkflowExecutionStatus,- WorkflowNodeExecutionMetadataKey,- WorkflowNodeExecutionStatus,- WorkflowType,-)-from graphon.graph_engine.layers import GraphEngineLayer-from graphon.graph_events import (- GraphEngineEvent,+from graphon.engine.layer import Layer+from graphon.engine_events import (+ EngineEvent,GraphRunAbortedEvent,GraphRunFailedEvent,GraphRunPartialSucceededEvent,- GraphRunPausedEvent,GraphRunStartedEvent,GraphRunSucceededEvent,NodeRunExceptionEvent,@@ -48,6 +34,14 @@ from graphon.graph_events import (NodeRunStartedEvent,NodeRunSucceededEvent,)+from graphon.entities import WorkflowNodeExecution+from graphon.enums import (+ BuiltinNodeTypes,+ WorkflowExecutionStatus,+ WorkflowNodeExecutionMetadataKey,+ WorkflowNodeExecutionStatus,+ WorkflowType,+)from graphon.node_events import NodeRunResultfrom libs.datetime_utils import naive_utc_nowfrom repositories.entities.workflow_execution import WorkflowExecution@@ -69,20 +63,8 @@ class PersistenceWorkflowInfo:graph_data: Mapping[str, Any]-@dataclass(slots=True)-class _NodeRuntimeSnapshot:- """Lightweight cache to keep node metadata across event phases."""-- node_id: str- title: str- predecessor_node_id: str | None- iteration_id: str | None- loop_id: str | None- created_at: datetime---class WorkflowPersistenceLayer(GraphEngineLayer):- """GraphEngine layer that persists workflow and node execution state."""+class WorkflowPersistenceLayer(Layer):+ """Engine layer that persists workflow and node execution state."""def __init__(self,@@ -102,24 +84,30 @@ class WorkflowPersistenceLayer(GraphEngineLayer):self._workflow_execution: WorkflowExecution | None = Noneself._node_execution_cache: dict[str, WorkflowNodeExecution] = {}- self._node_snapshots: dict[str, _NodeRuntimeSnapshot] = {}- self._node_sequence: int = 0+ self._node_run_indices: Mapping[str, int] = {}++ @property+ def workflow_execution(self) -> WorkflowExecution:+ return self._get_workflow_execution()++ def set_node_run_indices(self, indices: Mapping[str, int]) -> None:+ """Consume the upper layer's indices, allocated before engine event delivery."""+ self._node_run_indices = indices++ def set_node_execution_history(self, executions: Sequence[WorkflowNodeExecution]) -> None:+ """Restore materialized history before execution starts."""+ self._node_execution_cache = {+ execution.node_execution_id or execution.id: execution for execution in executions+ }# ------------------------------------------------------------------- # GraphEngineLayer lifecycle+ # Layer lifecycle# ------------------------------------------------------------------@override- def on_graph_start(self) -> None:- self._workflow_execution = None- self._node_execution_cache.clear()- self._node_snapshots.clear()- self._node_sequence = 0-- @override- def on_event(self, event: GraphEngineEvent) -> None:+ def on_event(self, event: EngineEvent) -> None:match event:case GraphRunStartedEvent():- self._handle_graph_run_started(event)+ self._handle_graph_run_started()case GraphRunSucceededEvent():self._handle_graph_run_succeeded(event)case GraphRunPartialSucceededEvent():@@ -128,8 +116,6 @@ class WorkflowPersistenceLayer(GraphEngineLayer):self._handle_graph_run_failed(event)case GraphRunAbortedEvent():self._handle_graph_run_aborted(event)- case GraphRunPausedEvent():- self._handle_graph_run_paused(event)case NodeRunRetryEvent():self._handle_node_retry(event)case NodeRunStartedEvent():@@ -143,14 +129,10 @@ class WorkflowPersistenceLayer(GraphEngineLayer):case NodeRunPauseRequestedEvent():self._handle_node_pause_requested(event)- @override- def on_graph_end(self, error: Exception | None) -> None:- return-# ------------------------------------------------------------------# Graph-level handlers# ------------------------------------------------------------------- def _handle_graph_run_started(self, event: GraphRunStartedEvent | None = None) -> None:+ def _handle_graph_run_started(self) -> None:execution_id = self._get_execution_id()timing = self._application_generate_entity.workflow_execution_timingtiming.start()@@ -165,89 +147,70 @@ class WorkflowPersistenceLayer(GraphEngineLayer):accumulated_elapsed_time=timing.elapsed_time,)- self._workflow_execution_repository.save(workflow_execution)+ self._workflow_execution_repository.save_synchronously(workflow_execution)self._workflow_execution = workflow_execution- if event is not None and event.reason == WorkflowStartReason.RESUMPTION:- node_executions = self._workflow_node_execution_repository.get_by_workflow_execution(execution_id)- self._node_execution_cache = {execution.id: execution for execution in node_executions}- self._node_sequence = max((execution.index for execution in node_executions), default=0)def _handle_graph_run_succeeded(self, event: GraphRunSucceededEvent) -> None:execution = self._get_workflow_execution()execution.outputs = event.outputsexecution.status = WorkflowExecutionStatus.SUCCEEDED- self._populate_completion_statistics(execution)-- self._workflow_execution_repository.save(execution)- self._enqueue_trace_task(execution)- _inspector_publish_workflow_completed(workflow_run_id=execution.id_, status=str(execution.status.value))+ self._finish_workflow_execution(execution)def _handle_graph_run_partial_succeeded(self, event: GraphRunPartialSucceededEvent) -> None:execution = self._get_workflow_execution()execution.outputs = event.outputsexecution.status = WorkflowExecutionStatus.PARTIAL_SUCCEEDEDexecution.exceptions_count = event.exceptions_count- self._populate_completion_statistics(execution)-- self._workflow_execution_repository.save(execution)- self._enqueue_trace_task(execution)- _inspector_publish_workflow_completed(workflow_run_id=execution.id_, status=str(execution.status.value))+ self._finish_workflow_execution(execution)def _handle_graph_run_failed(self, event: GraphRunFailedEvent) -> None:execution = self._get_workflow_execution()execution.status = WorkflowExecutionStatus.FAILEDexecution.error_message = event.errorexecution.exceptions_count = event.exceptions_count- self._populate_completion_statistics(execution)-- self._fail_running_node_executions(error_message=event.error)- self._workflow_execution_repository.save(execution)- self._enqueue_trace_task(execution)- _inspector_publish_workflow_completed(workflow_run_id=execution.id_, status=str(execution.status.value))+ self._finish_workflow_execution(execution)def _handle_graph_run_aborted(self, event: GraphRunAbortedEvent) -> None:execution = self._get_workflow_execution()execution.status = WorkflowExecutionStatus.STOPPEDexecution.error_message = event.reason or "Workflow execution aborted"- self._populate_completion_statistics(execution)+ self._finish_workflow_execution(execution)- self._fail_running_node_executions(error_message=execution.error_message or "")- self._workflow_execution_repository.save(execution)+ def _finish_workflow_execution(self, execution: WorkflowExecution) -> None:+ self._populate_completion_statistics(execution)+ if execution.status in (WorkflowExecutionStatus.FAILED, WorkflowExecutionStatus.STOPPED):+ self._fail_running_node_executions(error_message=execution.error_message or "")+ self._workflow_execution_repository.save_synchronously(execution)self._enqueue_trace_task(execution)_inspector_publish_workflow_completed(workflow_run_id=execution.id_, status=str(execution.status.value))- def _handle_graph_run_paused(self, event: GraphRunPausedEvent) -> None:- execution = self._get_workflow_execution()- execution.status = WorkflowExecutionStatus.PAUSED- execution.outputs = event.outputs- self._populate_completion_statistics(execution, update_finished=False)-- self._workflow_execution_repository.save(execution)-# ------------------------------------------------------------------# Node-level handlers# ------------------------------------------------------------------def _handle_node_started(self, event: NodeRunStartedEvent) -> None:execution = self._get_workflow_execution()+ iteration_id = event.node_run_result.metadata.get(WorkflowNodeExecutionMetadataKey.ITERATION_ID)+ loop_id = event.node_run_result.metadata.get(WorkflowNodeExecutionMetadataKey.LOOP_ID)- metadata = {- WorkflowNodeExecutionMetadataKey.ITERATION_ID: event.in_iteration_id,- WorkflowNodeExecutionMetadataKey.LOOP_ID: event.in_loop_id,+ metadata: dict[WorkflowNodeExecutionMetadataKey, Any] = {+ **event.node_run_result.metadata,+ WorkflowNodeExecutionMetadataKey.ITERATION_ID: iteration_id,+ WorkflowNodeExecutionMetadataKey.LOOP_ID: loop_id,}-+ previous = self._node_execution_cache.get(event.id)domain_execution = WorkflowNodeExecution(- id=event.id,+ id=previous.id if previous is not None else event.id,node_execution_id=event.id,workflow_id=execution.workflow_id,workflow_execution_id=execution.id_,predecessor_node_id=event.predecessor_node_id,- index=self._next_node_sequence(),+ index=self._node_run_indices[event.id],node_id=event.node_id,node_type=event.node_type,title=event.node_title,status=WorkflowNodeExecutionStatus.RUNNING,metadata=metadata,- created_at=event.start_at,+ created_at=previous.created_at if previous is not None else event.start_at,)self._node_execution_cache[event.id] = domain_execution@@ -256,15 +219,6 @@ class WorkflowPersistenceLayer(GraphEngineLayer):else:self._workflow_node_execution_repository.save(domain_execution)- snapshot = _NodeRuntimeSnapshot(- node_id=event.node_id,- title=event.node_title,- predecessor_node_id=event.predecessor_node_id,- iteration_id=event.in_iteration_id,- loop_id=event.in_loop_id,- created_at=event.start_at,- )- self._node_snapshots[event.id] = snapshot_inspector_publish_node_changed(workflow_run_id=execution.id_, node_id=event.node_id, status="running")def _handle_node_retry(self, event: NodeRunRetryEvent) -> None:@@ -367,10 +321,6 @@ class WorkflowPersistenceLayer(GraphEngineLayer):raise ValueError(f"Node execution not found for id={node_execution_id}")return self._node_execution_cache[node_execution_id]- def _next_node_sequence(self) -> int:- self._node_sequence += 1- return self._node_sequence-def _append_retry_history(self, execution: WorkflowNodeExecution, event: NodeRunRetryEvent) -> None:"""Append a validated full attempt before repository truncation or offload."""finished_at = naive_utc_now()@@ -416,11 +366,10 @@ class WorkflowPersistenceLayer(GraphEngineLayer):merged_process_data[RETRY_HISTORY_PROCESS_DATA_KEY] = raw_historyreturn merged_process_data- def _populate_completion_statistics(self, execution: WorkflowExecution, *, update_finished: bool = True) -> None:+ def _populate_completion_statistics(self, execution: WorkflowExecution) -> None:execution.accumulated_elapsed_time = self._application_generate_entity.workflow_execution_timing.finish()- if update_finished:- execution.finished_at = naive_utc_now()- runtime_state = self.graph_runtime_state+ execution.finished_at = naive_utc_now()+ runtime_state = self.runtime_stateexecution.total_tokens = runtime_state.total_tokensexecution.total_steps = runtime_state.node_run_stepsexecution.outputs = execution.outputs or runtime_state.outputs@@ -437,11 +386,9 @@ class WorkflowPersistenceLayer(GraphEngineLayer):finished_at: datetime | None = None,) -> None:actual_finished_at = finished_at or naive_utc_now()- snapshot = self._node_snapshots.get(domain_execution.id)- start_at = snapshot.created_at if snapshot else domain_execution.created_atdomain_execution.status = statusdomain_execution.finished_at = actual_finished_at- domain_execution.elapsed_time = max((actual_finished_at - start_at).total_seconds(), 0.0)+ domain_execution.elapsed_time = max((actual_finished_at - domain_execution.created_at).total_seconds(), 0.0)if error:domain_execution.error = error@@ -457,7 +404,7 @@ class WorkflowPersistenceLayer(GraphEngineLayer):inputs=node_result.inputs,process_data=process_data,outputs=projected_outputs,- metadata=node_result.metadata,+ metadata={**(domain_execution.metadata or {}), **node_result.metadata},)else:domain_execution.process_data = preserve_workflow_agent_binding_id(@@ -468,10 +415,20 @@ class WorkflowPersistenceLayer(GraphEngineLayer):self._workflow_node_execution_repository.save(domain_execution)self._workflow_node_execution_repository.save_execution_data(domain_execution)+ def persist_unfinished_nodes_synchronously(self) -> None:+ """Make paused participants durable before exposing a resumable pause."""+ for execution in self._node_execution_cache.values():+ if execution.status in (+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ ):+ self._workflow_node_execution_repository.save_synchronously(execution)+def _fail_running_node_executions(self, *, error_message: str) -> None:now = naive_utc_now()for execution in self._node_execution_cache.values():- if execution.status == WorkflowNodeExecutionStatus.RUNNING:+ if execution.status in (WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.RETRY):execution.status = WorkflowNodeExecutionStatus.FAILEDexecution.error = error_messageexecution.finished_at = now@@ -506,5 +463,5 @@ class WorkflowPersistenceLayer(GraphEngineLayer):self._trace_manager.add_trace_task(trace_task)def _system_variables(self) -> Mapping[str, Any]:- runtime_state = self.graph_runtime_state+ runtime_state = self.runtime_statereturn runtime_state.variable_pool.get_by_prefix(SYSTEM_VARIABLE_NODE_ID)diff --git a/api/services/workflow_run_agg.py b/api/services/workflow_run_agg.pynew file mode 100644index 00000000000..c62471e5d33--- /dev/null+++ b/api/services/workflow_run_agg.py@@ -0,0 +1,299 @@+"""Coordinate persistence and publication around a prepared workflow execution."""++import logging+from collections.abc import Generator, Sequence+from contextlib import closing+from dataclasses import replace++from sqlalchemy import Engine+from sqlalchemy.orm import sessionmaker++from core.app.apps.workflow_app_runner import PreparedWorkflowRun, WorkflowBasedAppRunner+from core.app.entities.app_invoke_entities import (+ AdvancedChatAppGenerateEntity,+ RagPipelineGenerateEntity,+ WorkflowAppGenerateEntity,+)+from core.app.layers.conversation_variable_persist_layer import ConversationVariablePersistenceLayer+from core.app.layers.pause_state_persist_layer import (+ PauseStateLayerConfig,+ WorkflowResumptionContext,+ _AdvancedChatAppGenerateEntityWrapper,+ _WorkflowGenerateEntityWrapper,+)+from core.db.session_factory import session_factory+from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository+from core.repositories.human_input_repository import HumanInputFormRecord, HumanInputFormSubmissionRepository+from core.workflow.node_runtime import DifyHumanInputNodeRuntime, resolve_dify_run_context+from core.workflow.nodes.agent_v2.workspace_retirement_layer import build_workflow_agent_workspace_retirement_layer+from core.workflow.nodes.human_input.boundary import build_human_input_pause_reason+from core.workflow.nodes.human_input.enums import HumanInputFormStatus+from core.workflow.nodes.human_input.pause_reason import PauseReason+from core.workflow.nodes.human_input.session_binding import default_session_binding+from core.workflow.system_variables import SystemVariableKey, get_system_text+from extensions.otel import WorkflowAppRunnerHandler, trace_span+from graphon.engine.layer import Layer+from graphon.engine_events import (+ EngineEvent,+ GraphRunAbortedEvent,+ GraphRunFailedEvent,+ GraphRunPartialSucceededEvent,+ GraphRunPausedEvent,+ GraphRunStartedEvent,+ GraphRunSucceededEvent,+ NodeRunHumanInputFormFilledEvent,+ NodeRunHumanInputFormTimeoutEvent,+ NodeRunStartedEvent,+)+from graphon.entities.pause_reason import HitlRequired+from graphon.enums import WorkflowExecutionStatus+from graphon.file.runtime import use_workflow_file_runtime+from repositories.factory import DifyAPIRepositoryFactory+from repositories.sqlalchemy_api_workflow_run_repository import WorkflowPauseStoppedError+from services.conversation_variable_updater import ConversationVariableUpdater+from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer+from services.workflow_run_index import WorkflowRunIndex++logger = logging.getLogger(__name__)+++class WorkflowRunAgg:+ """Own one execution attempt's persistence, form refresh, indexes, and durable pause."""++ def __init__(+ self,+ prepared: PreparedWorkflowRun,+ runner: WorkflowBasedAppRunner,+ pause_state_config: PauseStateLayerConfig | None,+ *,+ workflow_execution_repository: WorkflowExecutionRepository,+ workflow_node_execution_repository: WorkflowNodeExecutionRepository,+ ) -> None:+ self._workflow_execution_repository = workflow_execution_repository+ self._prepared = prepared+ self._runner = runner+ self._pause_state_config = pause_state_config+ self._entry = prepared.entry+ self._runtime_state = self._entry.graph_engine.runtime_state+ run_id = get_system_text(self._runtime_state.variable_pool, SystemVariableKey.WORKFLOW_EXECUTION_ID)+ if not run_id:+ raise ValueError("Workflow run ID is required")+ self._run_id = run_id+ self._run_context = resolve_dify_run_context(self._entry.graph_engine.graph.root_node.run_context)+ self._pending_forms = {+ default_session_binding.resolve_form_id_from_session_id(session_id=reason.session_id): reason+ for reason in self._runtime_state.graph_execution.pause_reasons+ if isinstance(reason, HitlRequired)+ }+ self._published_form_ids: set[str] = set()+ self._form_repository = HumanInputFormSubmissionRepository()+ self.pause_reasons: Sequence[PauseReason] = ()+ is_resuming = self._runtime_state.graph_execution.paused+ histories = (+ workflow_node_execution_repository.get_by_workflow_execution(self._run_id, include_paused=True)+ if is_resuming+ else ()+ )+ self._persistence_layer = WorkflowPersistenceLayer(+ application_generate_entity=prepared.generate_entity,+ workflow_info=PersistenceWorkflowInfo(+ workflow_id=self._entry.graph_engine.graph.root_node.workflow_id,+ workflow_type=prepared.workflow_type,+ version=prepared.workflow_version,+ graph_data=self._entry.graph_config,+ ),+ workflow_execution_repository=workflow_execution_repository,+ workflow_node_execution_repository=workflow_node_execution_repository,+ trace_manager=prepared.generate_entity.trace_manager,+ )+ application_layers: list[Layer] = [self._persistence_layer]+ if not isinstance(prepared.generate_entity, RagPipelineGenerateEntity):+ application_layers.append(+ build_workflow_agent_workspace_retirement_layer(dify_run_context=self._run_context)+ )+ if isinstance(prepared.generate_entity, AdvancedChatAppGenerateEntity):+ application_layers.append(+ ConversationVariablePersistenceLayer(ConversationVariableUpdater(session_factory.get_session_maker()))+ )+ self._application_layers = (*application_layers, *prepared.application_layers)+ for layer in self._application_layers:+ self._entry.graph_engine.add_layer(layer)+ self.index = WorkflowRunIndex(histories)+ self._persistence_layer.set_node_run_indices(self.index.indices)+ self._persistence_layer.set_node_execution_history(histories)+ self._entry.graph_engine.add_layer(self.index)++ @staticmethod+ @trace_span(WorkflowAppRunnerHandler)+ def run(+ runner: WorkflowBasedAppRunner,+ pause_state_config: PauseStateLayerConfig | None,+ workflow_execution_repository: WorkflowExecutionRepository,+ workflow_node_execution_repository: WorkflowNodeExecutionRepository,+ ) -> None:+ prepared = runner.prepare()+ if prepared is None:+ return+ aggregate = WorkflowRunAgg(+ prepared,+ runner,+ pause_state_config,+ workflow_execution_repository=workflow_execution_repository,+ workflow_node_execution_repository=workflow_node_execution_repository,+ )+ with closing(aggregate.iter_events()) as events:+ for event in events:+ runner.handle_event(+ prepared.entry,+ event,+ node_run_index=aggregate.index.index_for(event.id) if isinstance(event, NodeRunStartedEvent) else 1,+ node_execution_snapshots=aggregate.index.root_snapshots,+ pause_reasons=aggregate.pause_reasons,+ )++ def iter_events(self) -> Generator[EngineEvent]:+ """Publish pause only after the engine closes and all persistence succeeds."""+ paused_event = None+ try:+ with closing(self._entry.run()) as events:+ for event in events:+ if isinstance(event, GraphRunPausedEvent):+ paused_event = event+ continue+ if isinstance(event, NodeRunHumanInputFormFilledEvent | NodeRunHumanInputFormTimeoutEvent):+ if event.id in self._published_form_ids:+ continue+ self._published_form_ids.add(event.id)+ if isinstance(+ event,+ GraphRunSucceededEvent+ | GraphRunPartialSucceededEvent+ | GraphRunFailedEvent+ | GraphRunAbortedEvent,+ ):+ self._refresh_form_completions()+ if isinstance(event, GraphRunStartedEvent):+ self._api_run_repository().bind_workflow_task(+ tenant_id=self._run_context.tenant_id,+ app_id=self._prepared.generate_entity.app_config.app_id,+ workflow_run_id=self._run_id,+ task_id=self._prepared.generate_entity.task_id,+ )+ yield event+ if isinstance(event, GraphRunStartedEvent):+ self._refresh_form_completions()+ if paused_event is not None:+ self._refresh_form_completions()+ try:+ self._persist_pause(paused_event)+ except WorkflowPauseStoppedError:+ self._api_run_repository().stop_paused_workflow_task(+ tenant_id=self._run_context.tenant_id,+ app_id=self._prepared.generate_entity.app_config.app_id,+ task_id=self._prepared.generate_entity.task_id,+ owner=None,+ )+ aborted = GraphRunAbortedEvent(reason="User requested stop")+ for layer in self._application_layers:+ layer.on_event(aborted)+ yield aborted+ else:+ yield paused_event+ except Exception as error:+ logger.exception("Workflow run orchestration failed")+ failed = GraphRunFailedEvent(+ error=str(error), exceptions_count=self._runtime_state.graph_execution.exceptions_count+ )+ for layer in self._application_layers:+ try:+ layer.on_event(failed)+ except Exception:+ logger.exception("Failed to handle workflow orchestration failure in %s", type(layer).__name__)+ yield failed++ def _load_forms(self, form_ids: Sequence[str]) -> dict[str, HumanInputFormRecord]:+ forms = self._form_repository.get_by_form_ids(+ form_ids,+ tenant_id=self._run_context.tenant_id,+ app_id=self._prepared.generate_entity.app_config.app_id,+ workflow_run_id=self._run_id,+ )+ for form_id in form_ids:+ if form_id not in forms:+ raise ValueError(f"Human input form not found or does not belong to this workflow run: {form_id}")+ return forms++ def _api_run_repository(self):+ config = self._pause_state_config+ factory = session_factory.get_session_maker() if config is None else config.session_factory+ if isinstance(factory, Engine):+ factory = sessionmaker(factory)+ return DifyAPIRepositoryFactory.create_api_workflow_run_repository(factory)++ def _refresh_form_completions(self) -> None:+ pending = {key: reason for key, reason in self._pending_forms.items() if key not in self._published_form_ids}+ if not pending:+ return+ forms = self._load_forms(tuple(pending))+ for form_id, form in forms.items():+ if form.status == HumanInputFormStatus.SUBMITTED:+ restored = DifyHumanInputNodeRuntime(self._run_context).restore_submitted_data(+ inputs=form.definition.inputs, submitted_data=form.submitted_data or {}+ )+ forms[form_id] = replace(form, submitted_data=dict(restored))+ self._runner.publish_human_input_results(self._entry, pending, forms, self._published_form_ids)++ def _persist_pause(self, event: GraphRunPausedEvent) -> None:+ form_ids = tuple(+ default_session_binding.resolve_form_id_from_session_id(session_id=reason.session_id)+ for reason in event.reasons+ if isinstance(reason, HitlRequired)+ )+ forms = self._load_forms(form_ids)+ self.pause_reasons = [+ build_human_input_pause_reason(+ reason=reason,+ record=forms[default_session_binding.resolve_form_id_from_session_id(session_id=reason.session_id)],+ variable_pool=self._runtime_state.variable_pool,+ )+ if isinstance(reason, HitlRequired)+ else reason+ for reason in event.reasons+ ]+ entity = self._prepared.generate_entity+ elapsed_time = entity.workflow_execution_timing.finish()+ config = self._pause_state_config+ snapshot = None+ if config is not None:+ wrapper = (+ _WorkflowGenerateEntityWrapper(entity=entity)+ if isinstance(entity, WorkflowAppGenerateEntity)+ else _AdvancedChatAppGenerateEntityWrapper(entity=entity)+ )+ with use_workflow_file_runtime(self._entry.graph_engine.file_runtime):+ snapshot = WorkflowResumptionContext(+ generate_entity=wrapper,+ elapsed_time=elapsed_time,+ serialized_graph_runtime_state=self._runtime_state.dumps(),+ serialized_response_stream_filter_state=self._entry.response_stream_filter.dumps(),+ ).dumps()+ execution = self._persistence_layer.workflow_execution+ execution.status = WorkflowExecutionStatus.PAUSED+ execution.accumulated_elapsed_time = elapsed_time+ execution.outputs = event.outputs or self._runtime_state.outputs+ execution.total_tokens = self._runtime_state.total_tokens+ execution.total_steps = self._runtime_state.node_run_steps+ execution.exceptions_count = max(+ execution.exceptions_count, self._runtime_state.graph_execution.exceptions_count+ )+ execution.finished_at = None+ self._persistence_layer.persist_unfinished_nodes_synchronously()+ self._workflow_execution_repository.save_synchronously(execution)+ if config is not None and snapshot is not None:+ repository = self._api_run_repository()+ repository.create_workflow_pause(+ workflow_run_id=self._run_id,+ state_owner_user_id=config.state_owner_user_id,+ state=snapshot,+ pause_reasons=self.pause_reasons,+ )diff --git a/api/services/workflow_run_index.py b/api/services/workflow_run_index.pynew file mode 100644index 00000000000..85fbc42aa3b--- /dev/null+++ b/api/services/workflow_run_index.py@@ -0,0 +1,49 @@+"""Allocate each node execution's persisted and published index before it starts."""++from collections.abc import Mapping, Sequence+from threading import Lock+from types import MappingProxyType+from typing import override++from core.app.entities.queue_entities import NodeExecutionSnapshot+from graphon.engine.layer import Layer+from graphon.entities import WorkflowNodeExecution+from graphon.enums import WorkflowNodeExecutionMetadataKey+from graphon.nodes.base.node import Node+++class WorkflowRunIndex(Layer):+ def __init__(self, root_executions: Sequence[WorkflowNodeExecution]) -> None:+ super().__init__()+ self._lock = Lock()+ self._indices: dict[str, int] = {}+ for execution in root_executions:+ self._indices[execution.id] = execution.index+ if execution.node_execution_id:+ self._indices[execution.node_execution_id] = execution.index+ self._next_index = max((execution.index for execution in root_executions), default=0)+ self.root_snapshots = tuple(+ NodeExecutionSnapshot(+ execution_id=execution.node_execution_id or execution.id,+ title=execution.title,+ index=execution.index,+ start_at=execution.created_at,+ iteration_id=str((execution.metadata or {}).get(WorkflowNodeExecutionMetadataKey.ITERATION_ID) or ""),+ loop_id=str((execution.metadata or {}).get(WorkflowNodeExecutionMetadataKey.LOOP_ID) or ""),+ )+ for execution in root_executions+ )++ @property+ def indices(self) -> Mapping[str, int]:+ return MappingProxyType(self._indices)++ def index_for(self, execution_id: str) -> int:+ return self._indices[execution_id]++ @override+ def on_node_run_start(self, node: Node) -> None:+ with self._lock:+ if node.execution_id not in self._indices:+ self._next_index += 1+ self._indices[node.execution_id] = self._next_indexdiff --git a/api/services/workflow_service.py b/api/services/workflow_service.pyindex 71a4c3f3394..74fef8c57c4 100644--- a/api/services/workflow_service.py+++ b/api/services/workflow_service.py@@ -55,6 +55,7 @@ from events.app_event import app_draft_workflow_was_synced, app_published_workflfrom extensions.ext_database import dbfrom extensions.ext_storage import storagefrom factories.file_factory import build_from_mapping, build_from_mappings+from graphon.engine_events import NodeEvent, NodeRunFailedEvent, NodeRunStartedEvent, NodeRunSucceededEventfrom graphon.entities import WorkflowNodeExecutionfrom graphon.entities.graph_config import NodeConfigDictfrom graphon.enums import (@@ -65,7 +66,6 @@ from graphon.enums import ()from graphon.errors import WorkflowNodeRunFailedErrorfrom graphon.file import File-from graphon.graph_events import GraphNodeEventBase, NodeRunFailedEvent, NodeRunSucceededEventfrom graphon.node_events import NodeRunResultfrom graphon.nodes import BuiltinNodeTypesfrom graphon.nodes.base.node import Node@@ -97,7 +97,9 @@ from services.errors.app import (from services.human_input_service import HumanInputServicefrom services.system_feature_service import SystemFeatureServicefrom services.workflow.workflow_converter import WorkflowConverter+from services.workflow_node_variables import get_human_input_form_variable_mappingfrom services.workflow_ref_service import WorkflowRef+from services.workflow_run_agg import WorkflowRunAggfrom services.workflow_version_number_service import allocate_version_numberfrom .errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError@@ -1196,7 +1198,7 @@ class WorkflowService:add_variables_to_pool(variable_pool,build_bootstrap_variables(- system_variables=default_system_variables(),+ system_variables=build_system_variables(),environment_variables=draft_workflow.environment_variables,),)@@ -1215,6 +1217,7 @@ class WorkflowService:enclosing_node_id = Nonerun = WorkflowEntry.single_step_run(+ execution_driver=WorkflowRunAgg.run,workflow=draft_workflow,node_id=node_id,user_inputs=user_inputs,@@ -1223,18 +1226,6 @@ class WorkflowService:variable_loader=variable_loader,)- # run draft workflow node- start_at = time.perf_counter()- node_execution = self._handle_single_step_result(- invoke_node_fn=lambda: run,- start_at=start_at,- node_id=node_id,- )-- # Set workflow_id on the NodeExecution- node_execution.workflow_id = draft_workflow.id-- # Create repository and save the node executionrepository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(session_factory=db.engine,tenant_id=app_model.tenant_id,@@ -1242,6 +1233,46 @@ class WorkflowService:app_id=app_model.id,triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,)+ node, node_events = run+ started_event: NodeRunStartedEvent | None = None++ def persist_node_start() -> Generator[NodeEvent | ContainerAwaitRequest]:+ nonlocal started_event+ for event in node_events:+ if isinstance(event, NodeRunStartedEvent) and event.node_id == node_id:+ started_event = event+ # Agent workspace allocation requires this caller row before the node body runs.+ repository.save_synchronously(+ WorkflowNodeExecution(+ id=event.id,+ node_execution_id=event.id,+ workflow_id=draft_workflow.id,+ index=1,+ predecessor_node_id=event.predecessor_node_id,+ node_id=event.node_id,+ node_type=event.node_type,+ title=event.node_title,+ process_data=event.node_run_result.process_data or None,+ metadata=event.node_run_result.metadata,+ created_at=event.start_at,+ )+ )+ yield event++ start_at = time.perf_counter()+ node_execution = self._handle_single_step_result(+ invoke_node_fn=lambda: (node, persist_node_start()),+ start_at=start_at,+ node_id=node_id,+ )+ node_execution.workflow_id = draft_workflow.id+ if started_event is not None:+ node_execution.id = started_event.id+ node_execution.node_execution_id = started_event.id+ node_execution.created_at = started_event.start_at+ node_execution.predecessor_node_id = started_event.predecessor_node_id+ repository.save_synchronously(node_execution)+ # LogStore needs its terminal append in addition to the synchronous SQL caller row.repository.save(node_execution)workflow_node_execution = self._node_execution_service_repo.get_execution_by_id(node_execution.id)@@ -1600,7 +1631,7 @@ class WorkflowService:human_input_node_data = HumanInputNodeData.model_validate(adapt_human_input_node_data_for_graph(node_config["data"]))- variable_mapping = human_input_node_data.extract_variable_selector_to_variable_mapping(node_config["id"])+ variable_mapping = get_human_input_form_variable_mapping(node_config["id"], human_input_node_data)normalized_user_inputs: dict[str, Any] = dict(manual_inputs)load_into_variable_pool(@@ -1645,7 +1676,7 @@ class WorkflowService:self,invoke_node_fn: Callable[[],- tuple[Node, Generator[GraphNodeEventBase | ContainerAwaitRequest]],+ tuple[Node, Generator[NodeEvent | ContainerAwaitRequest]],],start_at: float,node_id: str,@@ -1685,7 +1716,7 @@ class WorkflowService:self,invoke_node_fn: Callable[[],- tuple[Node, Generator[GraphNodeEventBase | ContainerAwaitRequest]],+ tuple[Node, Generator[NodeEvent | ContainerAwaitRequest]],],) -> tuple[Node, NodeRunResult | None, bool, str | None]:"""diff --git a/api/tasks/app_generate/workflow_execute_task.py b/api/tasks/app_generate/workflow_execute_task.pyindex 2036e340afe..33357466d34 100644--- a/api/tasks/app_generate/workflow_execute_task.py+++ b/api/tasks/app_generate/workflow_execute_task.py@@ -3,6 +3,7 @@ import loggingimport uuidfrom collections.abc import Generator, Mappingfrom enum import StrEnum+from functools import partialfrom typing import Annotated, Anyfrom celery import shared_task@@ -24,10 +25,10 @@ from core.app.entities.task_entities import WorkflowFinishStreamResponse, Workflfrom core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContextfrom core.repositories import DifyCoreRepositoryFactoryfrom extensions.ext_database import db+from graphon.engine.filter import ResponseStreamFilterfrom graphon.entities import WorkflowStartReasonfrom graphon.enums import WorkflowExecutionStatus-from graphon.filters import ResponseStreamFilter-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom libs.datetime_utils import naive_utc_nowfrom libs.flask_utils import set_login_userfrom libs.helper import to_timestamp@@ -36,6 +37,9 @@ from models.enums import CreatorUserRole, WorkflowRunTriggeredFromfrom models.model import App, AppMode, Conversation, EndUser, Messagefrom models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowRunfrom repositories.factory import DifyAPIRepositoryFactory+from repositories.sqlalchemy_api_workflow_run_repository import WorkflowRunNotPausedError+from services.app_task_service import AppTaskService+from services.workflow_run_agg import WorkflowRunAgglogger = logging.getLogger(__name__)@@ -211,7 +215,7 @@ class _AppRunner:):exec_params = self._exec_paramsif exec_params.app_mode == AppMode.ADVANCED_CHAT:- return AdvancedChatAppGenerator().generate(+ return AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run).generate(app_model=app,workflow=workflow,user=user,@@ -223,7 +227,7 @@ class _AppRunner:session=session,)if exec_params.app_mode == AppMode.WORKFLOW:- return WorkflowAppGenerator().generate(+ return WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate(app_model=app,workflow=workflow,user=user,@@ -500,7 +504,7 @@ def _resume_app_execution(payload: dict[str, Any]) -> None:generate_entity = resumption_context.get_generate_entity()- graph_runtime_state = GraphRuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)+ graph_runtime_state = RuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)response_stream_filter = resumption_context.get_response_stream_filter()conversation = None@@ -559,13 +563,42 @@ def _resume_app_execution(payload: dict[str, Any]) -> None:)return+ # TODO: Remove legacy task recovery 30 days after the last legacy writer is retired.+ task_id = workflow_run.task_id+ if task_id is None:+ try:+ task_id = str(uuid.UUID(generate_entity.task_id))+ except ValueError:+ logger.warning("Invalid saved task ID for workflow run %s", workflow_run_id)+ return+ AppTaskService.index_workflow_pause_task(+ workflow_run=workflow_run,+ pause_entity=pause_entity,+ resumption_context=resumption_context,+ workflow_run_repo=workflow_run_repo,+ )+ indexed_run = workflow_run_repo.get_workflow_run_by_id(+ workflow_run.tenant_id, workflow_run.app_id, workflow_run.id+ )+ if indexed_run is None or indexed_run.task_id != task_id:+ logger.warning("Cannot recover saved task ownership for workflow run %s", workflow_run_id)+ return+ workflow_run.task_id = task_id+ generate_entity.task_id = task_id+# The resumed attempt reuses the paused run's task ID, so cancellation# signals armed against that ID before or during the pause would abort it# immediately and report it as stopped by the user. This attempt is starting# deliberately, so drop them before any engine can observe them.- clear_app_task_cancellation_signals(generate_entity.task_id)-- workflow_run_repo.resume_workflow_pause(workflow_run_id, pause_entity)+ try:+ workflow_run_repo.resume_workflow_pause(+ workflow_run_id,+ pause_entity,+ before_resume=partial(clear_app_task_cancellation_signals, generate_entity.task_id),+ )+ except WorkflowRunNotPausedError:+ logger.info("Workflow run %s was stopped or resumed by another request", workflow_run_id)+ returnpause_config = PauseStateLayerConfig(session_factory=session_factory,@@ -617,7 +650,7 @@ def _resume_advanced_chat(conversation: Conversation,message: Message,generate_entity: AdvancedChatAppGenerateEntity,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,response_stream_filter: ResponseStreamFilter,session_factory: sessionmaker,pause_state_config: PauseStateLayerConfig,@@ -647,7 +680,7 @@ def _resume_advanced_chat(triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,)- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)try:response = generator.resume(@@ -685,7 +718,7 @@ def _resume_workflow(workflow: Workflow,user: Account | EndUser,generate_entity: WorkflowAppGenerateEntity,- graph_runtime_state: GraphRuntimeState,+ graph_runtime_state: RuntimeState,response_stream_filter: ResponseStreamFilter,session_factory: sessionmaker,pause_state_config: PauseStateLayerConfig,@@ -716,7 +749,7 @@ def _resume_workflow(triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,)- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)try:response = generator.resume(diff --git a/api/tasks/async_workflow_tasks.py b/api/tasks/async_workflow_tasks.pyindex a3187b148c8..7e0cf0e14b2 100644--- a/api/tasks/async_workflow_tasks.py+++ b/api/tasks/async_workflow_tasks.py@@ -8,6 +8,7 @@ with appropriate retry policies and error handling.import loggingfrom datetime import UTC, datetimefrom typing import Any+from uuid import UUIDfrom celery import shared_taskfrom sqlalchemy import select@@ -23,7 +24,7 @@ from core.app.layers.trigger_post_layer import TriggerPostLayerfrom core.db.session_factory import session_factoryfrom core.repositories import DifyCoreRepositoryFactoryfrom extensions.ext_database import db-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom models.account import Accountfrom models.enums import CreatorUserRole, WorkflowRunTriggeredFrom, WorkflowTriggerStatusfrom models.model import App, EndUser, Tenant@@ -31,12 +32,14 @@ from models.trigger import WorkflowTriggerLogfrom models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowRunfrom repositories.factory import DifyAPIRepositoryFactoryfrom repositories.sqlalchemy_workflow_trigger_log_repository import SQLAlchemyWorkflowTriggerLogRepository+from services.app_task_service import AppTaskServicefrom services.errors.app import WorkflowNotFoundErrorfrom services.workflow.entities import (TriggerData,WorkflowResumeTaskData,WorkflowTaskData,)+from services.workflow_run_agg import WorkflowRunAggfrom tasks.workflow_cfs_scheduler.cfs_scheduler import AsyncWorkflowCFSPlanEntity, AsyncWorkflowCFSPlanSchedulerfrom tasks.workflow_cfs_scheduler.entities import AsyncWorkflowQueue, AsyncWorkflowSystemStrategy@@ -145,7 +148,7 @@ def _execute_workflow_common(user = _get_user(session, trigger_log)# Execute workflow using WorkflowAppGenerator- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)# Adapt trigger inputs and files for the generator.args = _build_generator_args(trigger_data)@@ -224,7 +227,30 @@ def resume_workflow_execution(task_data_dict: dict[str, Any]) -> None:)return- graph_runtime_state = GraphRuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)+ # TODO: Remove legacy task recovery 30 days after the last legacy writer is retired.+ task_id = workflow_run.task_id+ if task_id is None:+ try:+ task_id = str(UUID(generate_entity.task_id))+ except ValueError:+ logger.warning("Invalid saved task ID for workflow run %s", task_data.workflow_run_id)+ return+ AppTaskService.index_workflow_pause_task(+ workflow_run=workflow_run,+ pause_entity=pause_entity,+ resumption_context=resumption_context,+ workflow_run_repo=workflow_run_repo,+ )+ indexed_run = workflow_run_repo.get_workflow_run_by_id(+ workflow_run.tenant_id, workflow_run.app_id, workflow_run.id+ )+ if indexed_run is None or indexed_run.task_id != task_id:+ logger.warning("Cannot recover saved task ownership for workflow run %s", task_data.workflow_run_id)+ return+ workflow_run.task_id = task_id+ generate_entity.task_id = task_id++ graph_runtime_state = RuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)response_stream_filter = resumption_context.get_response_stream_filter()with session_factory() as session:@@ -258,7 +284,7 @@ def resume_workflow_execution(task_data_dict: dict[str, Any]) -> None:state_owner_user_id=workflow.created_by,)- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)start_time = datetime.now(UTC)graph_engine_layers = []trigger_log = _query_trigger_log_info(session_factory, task_data.workflow_run_id)diff --git a/api/tasks/mail_human_input_delivery_task.py b/api/tasks/mail_human_input_delivery_task.pyindex 6748ca3ae8e..e2b7e7e0c46 100644--- a/api/tasks/mail_human_input_delivery_task.py+++ b/api/tasks/mail_human_input_delivery_task.py@@ -13,7 +13,7 @@ from core.app.layers.pause_state_persist_layer import WorkflowResumptionContextfrom core.workflow.human_input_adapter import EmailDeliveryConfig, EmailDeliveryMethodfrom extensions.ext_database import dbfrom extensions.ext_mail import mail-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom models.human_input import (DeliveryMethodType,HumanInputDelivery,@@ -139,7 +139,7 @@ def _load_variable_pool(workflow_run_id: str | None) -> VariablePool | None:logger.exception("Failed to load resumption context for workflow run %s", workflow_run_id)return None- graph_runtime_state = GraphRuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)+ graph_runtime_state = RuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)return graph_runtime_state.variable_pooldiff --git a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.pyindex 1c5716dcec8..43861940769 100644--- a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py+++ b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py@@ -28,6 +28,7 @@ from models.enums import WorkflowRunTriggeredFromfrom models.workflow import Workflow, WorkflowNodeExecutionTriggeredFromfrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom services.file_service import FileService+from services.workflow_run_agg import WorkflowRunAgglogger = logging.getLogger(__name__)@@ -179,6 +180,7 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any],# Since we're already in a thread pool, no need for nested threadingpipeline_generator = PipelineGenerator(+ execution_driver=WorkflowRunAgg.run,documents=SQLAlchemyDocumentRepository(session_factory=session_factory),datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers,)diff --git a/api/tasks/rag_pipeline/rag_pipeline_run_task.py b/api/tasks/rag_pipeline/rag_pipeline_run_task.pyindex d164e4fec97..251c439e9ca 100644--- a/api/tasks/rag_pipeline/rag_pipeline_run_task.py+++ b/api/tasks/rag_pipeline/rag_pipeline_run_task.py@@ -29,6 +29,7 @@ from models.enums import WorkflowRunTriggeredFromfrom models.workflow import Workflow, WorkflowNodeExecutionTriggeredFromfrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom services.file_service import FileService+from services.workflow_run_agg import WorkflowRunAgglogger = logging.getLogger(__name__)@@ -193,6 +194,7 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any],# Since we're already in a thread pool, no need for nested threadingpipeline_generator = PipelineGenerator(+ execution_driver=WorkflowRunAgg.run,documents=SQLAlchemyDocumentRepository(session_factory=session_factory),datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers,)diff --git a/api/tasks/retry_document_indexing_task.py b/api/tasks/retry_document_indexing_task.pyindex 13d4e72e1ea..5a12b74ff1a 100644--- a/api/tasks/retry_document_indexing_task.py+++ b/api/tasks/retry_document_indexing_task.py@@ -21,6 +21,7 @@ from services.feature_service import FeatureServicefrom services.knowledge.indexing.adapters.execution import build_document_indexing_servicefrom services.knowledge.resource_scope import DatasetReffrom services.rag_pipeline.rag_pipeline import RagPipelineService+from services.workflow_run_agg import WorkflowRunAgglogger = logging.getLogger(__name__)@@ -126,6 +127,7 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str], user_document,user,generator=PipelineGenerator(+ execution_driver=WorkflowRunAgg.run,documents=SQLAlchemyDocumentRepository(session_factory=session_factory.get_session_maker()),diff --git a/api/tasks/workflow_execution_tasks.py b/api/tasks/workflow_execution_tasks.pyindex ba3c9fe3df3..c11ee99dcb8 100644--- a/api/tasks/workflow_execution_tasks.py+++ b/api/tasks/workflow_execution_tasks.py@@ -13,6 +13,7 @@ from celery import shared_taskfrom sqlalchemy import selectfrom core.db.session_factory import session_factory+from graphon.enums import WorkflowExecutionStatusfrom graphon.workflow_type_encoder import WorkflowRuntimeTypeConverterfrom models import CreatorUserRole, WorkflowRunfrom models.enums import WorkflowRunTriggeredFrom@@ -51,7 +52,7 @@ def save_workflow_execution_task(execution = WorkflowExecution.model_validate(execution_data)# Check if workflow run already exists- existing_run = session.scalar(select(WorkflowRun).where(WorkflowRun.id == execution.id_))+ existing_run = session.scalar(select(WorkflowRun).where(WorkflowRun.id == execution.id_).with_for_update())if existing_run:# Update existing workflow run@@ -125,6 +126,8 @@ def _update_workflow_run_from_execution(workflow_run: WorkflowRun, execution: Wo"""Update a WorkflowRun database model from a WorkflowExecution domain entity."""+ if workflow_run.stop_requested_at is not None and workflow_run.status == WorkflowExecutionStatus.STOPPED:+ returnjson_converter = WorkflowRuntimeTypeConverter()workflow_run.status = execution.statusworkflow_run.outputs = (diff --git a/api/tasks/workflow_node_execution_tasks.py b/api/tasks/workflow_node_execution_tasks.pyindex 623dd739449..0bec9aa0f79 100644--- a/api/tasks/workflow_node_execution_tasks.py+++ b/api/tasks/workflow_node_execution_tasks.py@@ -17,9 +17,11 @@ from core.workflow.node_execution_process_data import preserve_workflow_agent_bifrom graphon.entities.workflow_node_execution import (WorkflowNodeExecution,)+from graphon.enums import WorkflowNodeExecutionStatusfrom graphon.workflow_type_encoder import WorkflowRuntimeTypeConverterfrom models import CreatorUserRole, WorkflowNodeExecutionModelfrom models.workflow import WorkflowNodeExecutionTriggeredFrom+from repositories.workflow_run_control import apply_workflow_stop_to_node, get_stopped_workflow_runlogger = logging.getLogger(__name__)@@ -53,6 +55,18 @@ def save_workflow_node_execution_task(# Deserialize execution dataexecution = WorkflowNodeExecution.model_validate(execution_data)+ if execution.status in (+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ ):+ apply_workflow_stop_to_node(+ execution,+ get_stopped_workflow_run(+ session, tenant_id=tenant_id, workflow_run_id=execution.workflow_execution_id, lock=True+ ),+ )+# Check if node execution already existsexisting_execution = session.scalar(select(WorkflowNodeExecutionModel).where(@@ -147,6 +161,18 @@ def _update_node_execution_from_domain(node_execution: WorkflowNodeExecutionMode"""Update a WorkflowNodeExecutionModel database model from a WorkflowNodeExecution domain entity."""+ # Cleanup can insert the terminal row before its queued start is delivered.+ if (+ node_execution.status == WorkflowNodeExecutionStatus.FAILED+ and node_execution.finished_at is not None+ and execution.status+ in (+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.RETRY,+ )+ ):+ return# Update serialized datajson_converter = WorkflowRuntimeTypeConverter()node_execution.inputs = json.dumps(json_converter.to_json_encodable(execution.inputs)) if execution.inputs else "{}"diff --git a/api/tests/conftest.py b/api/tests/conftest.pyindex e526685433c..046023a5840 100644--- a/api/tests/conftest.py+++ b/api/tests/conftest.py@@ -1,8 +1,12 @@+from collections.abc import Iterator+import pytest-from core.app.workflow.file_runtime import bind_dify_workflow_file_runtime+from core.app.workflow.file_runtime import create_dify_workflow_file_runtime+from graphon.file.runtime import use_workflow_file_runtime@pytest.fixture(autouse=True)-def _bind_workflow_file_runtime() -> None:- bind_dify_workflow_file_runtime()+def _bind_workflow_file_runtime() -> Iterator[None]:+ with use_workflow_file_runtime(create_dify_workflow_file_runtime()):+ yielddiff --git a/api/tests/integration_tests/workflow/nodes/test_code.py b/api/tests/integration_tests/workflow/nodes/test_code.pyindex 8bbb0957b3c..8bc74f721c3 100644--- a/api/tests/integration_tests/workflow/nodes/test_code.py+++ b/api/tests/integration_tests/workflow/nodes/test_code.py@@ -15,7 +15,7 @@ from graphon.node_events import NodeRunResultfrom graphon.nodes.code.code_node import CodeNodefrom graphon.nodes.code.entities import CodeNodeDatafrom graphon.nodes.code.limits import CodeNodeLimits-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_paramspytest_plugins = ("tests.integration_tests.workflow.nodes.__mock.code_executor",)@@ -44,14 +44,16 @@ def _init_code_node(code_config: dict) -> CodeNode:)variable_pool.add(["code", "args1"], 1)variable_pool.add(["code", "args2"], 2)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())- node_factory = DifyNodeFactory(graph_init_params=init_params, graph_runtime_state=graph_runtime_state)+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )+ node_factory = DifyNodeFactory(init_params=init_params, runtime_state=graph_runtime_state)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")return CodeNode(node_id=str(uuid.uuid4()),data=CodeNodeData.model_validate(code_config["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,code_executor=node_factory._code_executor,code_limits=CodeNodeLimits(max_string_length=dify_config.CODE_MAX_STRING_LENGTH,@@ -90,8 +92,8 @@ def _code_config(code: str, output_type: str, *, variables: bool = True) -> dictdef test_execute_code(setup_code_executor_mock) -> None:code = "def main(args1: int, args2: int):\n return {'result': args1 + args2}"node = _init_code_node(_code_config(code, "number"))- node.graph_runtime_state.variable_pool.add(["1", "args1"], 1)- node.graph_runtime_state.variable_pool.add(["1", "args2"], 2)+ node.runtime_state.variable_pool.add(["1", "args1"], 1)+ node.runtime_state.variable_pool.add(["1", "args2"], 2)result = node._run()@@ -106,8 +108,8 @@ def test_execute_code(setup_code_executor_mock) -> None:def test_execute_code_output_validator(setup_code_executor_mock) -> None:code = "def main(args1: int, args2: int):\n return {'result': args1 + args2}"node = _init_code_node(_code_config(code, "string"))- node.graph_runtime_state.variable_pool.add(["1", "args1"], 1)- node.graph_runtime_state.variable_pool.add(["1", "args2"], 2)+ node.runtime_state.variable_pool.add(["1", "args1"], 1)+ node.runtime_state.variable_pool.add(["1", "args2"], 2)result = node._run()diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.pyindex d580388e5e4..66f464f18c1 100644--- a/api/tests/integration_tests/workflow/nodes/test_http.py+++ b/api/tests/integration_tests/workflow/nodes/test_http.py@@ -15,7 +15,7 @@ from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatusfrom graphon.file.file_manager import file_managerfrom graphon.graph import Graphfrom graphon.nodes.http_request import HttpRequestNode, HttpRequestNodeConfig, HttpRequestNodeData-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_paramspytest_plugins = ("tests.integration_tests.workflow.nodes.__mock.http",)@@ -64,12 +64,14 @@ def init_http_node(config: dict):variable_pool.add(["a", "args1"], 1)variable_pool.add(["a", "args2"], 2)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )# Create node factorynode_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -77,8 +79,8 @@ def init_http_node(config: dict):node = HttpRequestNode(node_id=str(uuid.uuid4()),data=HttpRequestNodeData.model_validate(config["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,http_request_config=HTTP_REQUEST_CONFIG,http_client=ssrf_proxy,tool_file_manager_factory=ToolFileManager,@@ -711,12 +713,14 @@ def test_nested_object_variable_selector(setup_http_mock):variable_pool.add(["a", "args2"], 2)variable_pool.add(["a", "args3"], {"nested": "nested_value"}) # Only for this test- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )# Create node factorynode_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -724,8 +728,8 @@ def test_nested_object_variable_selector(setup_http_mock):node = HttpRequestNode(node_id=str(uuid.uuid4()),data=HttpRequestNodeData.model_validate(graph_config["nodes"][1]["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,http_request_config=HTTP_REQUEST_CONFIG,http_client=ssrf_proxy,tool_file_manager_factory=ToolFileManager,diff --git a/api/tests/test_containers_integration_tests/controllers/web/test_human_input_form.py b/api/tests/test_containers_integration_tests/controllers/web/test_human_input_form.pyindex d80e3fe31ff..afbfbf0434d 100644--- a/api/tests/test_containers_integration_tests/controllers/web/test_human_input_form.py+++ b/api/tests/test_containers_integration_tests/controllers/web/test_human_input_form.py@@ -22,7 +22,7 @@ from core.workflow.nodes.human_input.entities import (from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus, ValueSourceTypefrom core.workflow.nodes.human_input.pause_reason import HumanInputRequiredfrom graphon.enums import WorkflowExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom models.account import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.enums import CreatorUserRole, WorkflowRunTriggeredFromfrom models.human_input import (@@ -108,7 +108,7 @@ def _build_resumption_context(*, app: App, workflow_run: WorkflowRun, options: lcall_depth=0,workflow_execution_id=workflow_run.id,)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)runtime_state.variable_pool.add(("start", "options"), options)return WorkflowResumptionContext(generate_entity=_WorkflowGenerateEntityWrapper(entity=generate_entity),@@ -231,6 +231,6 @@ def test_get_human_input_form_resolves_runtime_select_options(assert response.status_code == 200, response.get_data(as_text=True)body = json.loads(response.get_data(as_text=True))- assert body["inputs"][0]["option_source"]["type"] == "variable"- assert body["inputs"][0]["option_source"]["selector"] == ["start", "options"]+ assert body["inputs"][0]["option_source"]["type"] == "constant"+ assert body["inputs"][0]["option_source"]["selector"] == []assert body["inputs"][0]["option_source"]["value"] == ["approve", "reject"]diff --git a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.pyindex 56da1c948d1..d5d4e7de52b 100644--- a/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py+++ b/api/tests/test_containers_integration_tests/core/app/layers/test_pause_state_persist_layer.py@@ -1,599 +1,277 @@-"""Comprehensive TestContainers-based integration tests for PauseStatePersistenceLayer class.+"""Workflow pause orchestration against real Engine, PostgreSQL, Redis, and storage."""-This test suite covers complete integration scenarios including:-- Real database interactions using containerized PostgreSQL-- Real storage operations using test storage backend-- Complete workflow: event -> state serialization -> database save -> storage save-- Testing with actual WorkflowRunService (not mocked)-- Real Workflow and WorkflowRun instances in database-- Database transactions and rollback behavior-- Actual file upload and retrieval through storage-- Workflow status transitions in database-- Error handling with real database constraints-- Multiple pause events in sequence-- Integration with real ReadOnlyGraphRuntimeState implementations--These tests use TestContainers to spin up real services for integration testing,-providing more reliable and realistic test scenarios than mocks.-"""--import uuid-from time import time-from unittest.mock import Mock+import json+from collections.abc import Mapping+from contextlib import closing+from dataclasses import dataclass+from time import perf_counter+from uuid import uuid4import pytest-from sqlalchemy import Engine, delete, select+from sqlalchemy import Engine, selectfrom sqlalchemy.orm import Session, sessionmakerfrom core.app.app_config.entities import WorkflowUIBasedAppConfig-from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity-from core.app.layers.pause_state_persist_layer import (- PauseStatePersistenceLayer,- WorkflowResumptionContext,-)+from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager+from core.app.apps.workflow_app_runner import PreparedWorkflowRun, WorkflowBasedAppRunner+from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity+from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepositoryfrom core.workflow.system_variables import build_system_variables+from core.workflow.workflow_entry import WorkflowEntryfrom extensions.ext_storage import storage-from graphon.entities.pause_reason import SchedulingPause-from graphon.enums import WorkflowExecutionStatus-from graphon.filters import GraphEventFilterContext, ResponseStreamFilter-from graphon.graph_engine.entities.commands import GraphEngineCommand-from graphon.graph_engine.layers.base import GraphEngineLayerNotInitializedError-from graphon.graph_events import GraphRunPausedEvent+from graphon.engine.command import InMemoryChannel, PauseCommand+from graphon.engine_events import GraphRunFailedEvent, GraphRunPausedEvent, GraphRunStartedEvent, GraphRunSucceededEvent+from graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom graphon.model_runtime.entities.llm_entities import LLMUsage-from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool-from libs.datetime_utils import naive_utc_now+from graphon.nodes.start.entities import StartNodeData+from graphon.runtime import RuntimeState, VariablePoolfrom models import Account-from models import WorkflowPause as WorkflowPauseModel-from models.model import AppMode, UploadFile-from models.workflow import Workflow, WorkflowRun-from repositories.factory import DifyAPIRepositoryFactory+from models.account import Tenant, TenantAccountJoin, TenantAccountRole+from models.enums import WorkflowRunTriggeredFrom+from models.model import AppMode+from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowPause, WorkflowRunfrom repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository-from services.file_service import FileService-from services.workflow_run_service import WorkflowRunService---def _create_initialized_response_stream_filter() -> ResponseStreamFilter:- """Build a `ResponseStreamFilter` that has already run `initialize()`.-- `ResponseStreamFilter.dumps()` raises `RuntimeError` unless the filter has- processed a `GraphEventFilterContext` first. In production this always- happens before any event (including `GraphRunPausedEvent`) reaches- `PauseStatePersistenceLayer.on_event`, so tests that exercise `on_event`- or a subsequent `dumps()` call need a filter in that same state. A- nodeless graph is enough to satisfy the precondition.- """- response_stream_filter = ResponseStreamFilter()- context = GraphEventFilterContext(graph=Mock(nodes={}), runtime_state=Mock())- response_stream_filter.initialize(context)- return response_stream_filter---class _TestCommandChannelImpl:- """Real implementation of CommandChannel for testing."""-- def __init__(self):- self._commands: list[GraphEngineCommand] = []-- def fetch_commands(self) -> list[GraphEngineCommand]:- """Fetch pending commands for this GraphEngine instance."""- return self._commands.copy()-- def send_command(self, command: GraphEngineCommand) -> None:- """Send a command to be processed by this GraphEngine instance."""- self._commands.append(command)---class TestPauseStatePersistenceLayerTestContainers:- """Comprehensive TestContainers-based integration tests for PauseStatePersistenceLayer class."""-- @pytest.fixture- def engine(self, db_session_with_containers: Session):- """Get database engine from TestContainers session."""- bind = db_session_with_containers.get_bind()- assert isinstance(bind, Engine)- return bind-- @pytest.fixture- def file_service(self, engine: Engine):- """Create FileService instance with TestContainers engine."""- return FileService(engine)-- @pytest.fixture- def workflow_run_service(self, engine: Engine, file_service: FileService):- """Create WorkflowRunService instance with TestContainers engine and FileService."""- session_factory = sessionmaker(bind=engine, expire_on_commit=False)- workflow_runs = DifyAPISQLAlchemyWorkflowRunRepository(session_maker=session_factory)- return WorkflowRunService(- workflow_runs=workflow_runs,- node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(- session_maker=session_factory- ),- )-- @pytest.fixture(autouse=True)- def setup_test_data(self, db_session_with_containers: Session, file_service, workflow_run_service):- """Set up test data for each test method using TestContainers."""- # Create test tenant and account- from models.account import AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus-- tenant = Tenant(- name="Test Tenant",- status=TenantStatus.NORMAL,- )- db_session_with_containers.add(tenant)- db_session_with_containers.commit()-- account = Account(- email="[email protected]",- name="Test User",- interface_language="en-US",- status=AccountStatus.ACTIVE,- )- db_session_with_containers.add(account)- db_session_with_containers.commit()-- # Create tenant-account join- tenant_join = TenantAccountJoin(- tenant_id=tenant.id,- account_id=account.id,- role=TenantAccountRole.OWNER,- current=True,- )- db_session_with_containers.add(tenant_join)- db_session_with_containers.commit()-- # Set test data- self.test_tenant_id = tenant.id- self.test_user_id = account.id- self.test_app_id = str(uuid.uuid4())- self.test_workflow_id = str(uuid.uuid4())- self.test_workflow_run_id = str(uuid.uuid4())-- # Create test workflow- self.test_workflow = Workflow(- id=self.test_workflow_id,- tenant_id=self.test_tenant_id,- app_id=self.test_app_id,- type="workflow",- version="draft",- graph='{"nodes": [], "edges": []}',- features='{"file_upload": {"enabled": false}}',- created_by=self.test_user_id,- created_at=naive_utc_now(),- )-- # Create test workflow run- self.test_workflow_run = WorkflowRun(- id=self.test_workflow_run_id,- tenant_id=self.test_tenant_id,- app_id=self.test_app_id,- workflow_id=self.test_workflow_id,- type="workflow",- triggered_from="debugging",- version="draft",- status=WorkflowExecutionStatus.RUNNING,- created_by=self.test_user_id,- created_by_role="account",- created_at=naive_utc_now(),- )+from services.workflow_run_agg import WorkflowRunAgg- # Store session and service instances- self.session = db_session_with_containers- self.file_service = file_service- self.workflow_run_service = workflow_run_service- # Save test data to database- self.session.add(self.test_workflow)- self.session.add(self.test_workflow_run)- self.session.commit()+@dataclass+class WorkflowCase:+ session: Session+ engine: Engine+ account: Account+ workflow: Workflow+ run_id: str- yield-- # Cleanup- self._cleanup_test_data()-- def _cleanup_test_data(self):- """Clean up test data after each test method."""- try:- # Clean up workflow pauses- self.session.execute(delete(WorkflowPauseModel))- # Clean up upload files- self.session.execute(- delete(UploadFile).where(- UploadFile.tenant_id == self.test_tenant_id,- )- )- # Clean up workflow runs- self.session.execute(- delete(WorkflowRun).where(- WorkflowRun.tenant_id == self.test_tenant_id,- WorkflowRun.app_id == self.test_app_id,- )- )- # Clean up workflows- self.session.execute(- delete(Workflow).where(- Workflow.tenant_id == self.test_tenant_id,- Workflow.app_id == self.test_app_id,- )- )- self.session.commit()- except Exception as e:- self.session.rollback()- raise e-- def _create_graph_runtime_state(+ def prepare(self,- outputs: dict[str, object] | None = None,+ *,+ outputs: Mapping[str, object] | None = None,+ variables: Mapping[tuple[str, str], object] | None = None,total_tokens: int = 0,node_run_steps: int = 0,- variables: dict[tuple[str, str], object] | None = None,- workflow_run_id: str | None = None,- ) -> ReadOnlyGraphRuntimeState:- """Create a real GraphRuntimeState for testing."""- start_at = time()-- execution_id = workflow_run_id or getattr(self, "test_workflow_run_id", None) or str(uuid.uuid4())-- # Create variable pool- variable_pool = VariablePool.from_bootstrap(- system_variables=build_system_variables(workflow_execution_id=execution_id)- )- if variables:- for (node_id, var_key), value in variables.items():- variable_pool.add([node_id, var_key], value)-- # Create LLM usage- llm_usage = LLMUsage.empty_usage()- llm_usage.total_tokens = total_tokens-- # Create graph runtime state- graph_runtime_state = GraphRuntimeState(- variable_pool=variable_pool,- start_at=start_at,- llm_usage=llm_usage,- outputs=outputs or {},- node_run_steps=node_run_steps,- )-- return ReadOnlyGraphRuntimeStateWrapper(graph_runtime_state)-- def _create_generate_entity(- self,- workflow_execution_id: str | None = None,- user_id: str | None = None,- workflow_id: str | None = None,- ) -> WorkflowAppGenerateEntity:- execution_id = workflow_execution_id or getattr(self, "test_workflow_run_id", str(uuid.uuid4()))- wf_id = workflow_id or getattr(self, "test_workflow_id", str(uuid.uuid4()))- tenant_id = getattr(self, "test_tenant_id", "tenant-123")- app_id = getattr(self, "test_app_id", "app-123")- app_config = WorkflowUIBasedAppConfig(- tenant_id=str(tenant_id),- app_id=str(app_id),- app_mode=AppMode.WORKFLOW,- workflow_id=str(wf_id),- )- return WorkflowAppGenerateEntity(- task_id=str(uuid.uuid4()),- app_config=app_config,+ pause: bool = True,+ ) -> tuple[PreparedWorkflowRun, WorkflowRunAgg]:+ entity = WorkflowAppGenerateEntity(+ task_id=str(uuid4()),+ app_config=WorkflowUIBasedAppConfig(+ tenant_id=self.workflow.tenant_id,+ app_id=self.workflow.app_id,+ app_mode=AppMode.WORKFLOW,+ workflow_id=self.workflow.id,+ ),inputs={},files=[],- user_id=user_id or getattr(self, "test_user_id", str(uuid.uuid4())),- stream=False,+ user_id=self.account.id,+ stream=True,invoke_from=InvokeFrom.DEBUGGER,- workflow_execution_id=execution_id,- )-- def _create_pause_state_persistence_layer(- self,- workflow_run: WorkflowRun | None = None,- workflow: Workflow | None = None,- state_owner_user_id: str | None = None,- generate_entity: WorkflowAppGenerateEntity | None = None,- ) -> PauseStatePersistenceLayer:- """Create PauseStatePersistenceLayer with real dependencies."""- owner_id = state_owner_user_id- if owner_id is None:- if workflow is not None and workflow.created_by:- owner_id = workflow.created_by- elif workflow_run is not None and workflow_run.created_by:- owner_id = workflow_run.created_by- else:- owner_id = getattr(self, "test_user_id", None)-- assert owner_id is not None- owner_id = str(owner_id)- workflow_execution_id = (- workflow_run.id if workflow_run is not None else getattr(self, "test_workflow_run_id", None)- )- assert workflow_execution_id is not None- workflow_id = workflow.id if workflow is not None else getattr(self, "test_workflow_id", None)- assert workflow_id is not None- entity_user_id = getattr(self, "test_user_id", owner_id)- entity = generate_entity or self._create_generate_entity(- workflow_execution_id=str(workflow_execution_id),- user_id=entity_user_id,- workflow_id=str(workflow_id),+ workflow_execution_id=self.run_id,)-- return PauseStatePersistenceLayer(- session_factory=self.session.get_bind(),- state_owner_user_id=owner_id,- generate_entity=entity,- response_stream_filter=_create_initialized_response_stream_filter(),- )-- def test_complete_pause_flow_with_real_dependencies(self, db_session_with_containers: Session):- """Test complete pause flow: event -> state serialization -> database save -> storage save."""- # Arrange- layer = self._create_pause_state_persistence_layer()-- # Create real graph runtime state with test data- test_outputs = {"result": "test_output", "step": "intermediate"}- test_variables = {- ("node1", "var1"): "string_value",- ("node2", "var2"): {"complex": "object"},- }- graph_runtime_state = self._create_graph_runtime_state(- outputs=test_outputs,- total_tokens=100,- node_run_steps=5,- variables=test_variables,- )-- command_channel = _TestCommandChannelImpl()- layer.initialize(graph_runtime_state, command_channel)-- # Create pause event- event = GraphRunPausedEvent(- reasons=[SchedulingPause(message="test pause")],- outputs={"intermediate": "result"},- )-- # Act- layer.on_event(event)-- # Assert - Verify pause state was saved to database- self.session.refresh(self.test_workflow_run)- workflow_run = self.session.get(WorkflowRun, self.test_workflow_run_id)- assert workflow_run is not None- assert workflow_run.status == WorkflowExecutionStatus.PAUSED-- # Verify pause state exists in database- pause_model = self.session.scalars(- select(WorkflowPauseModel).where(WorkflowPauseModel.workflow_run_id == workflow_run.id)- ).first()- assert pause_model is not None- assert pause_model.workflow_id == self.test_workflow_id- assert pause_model.workflow_run_id == self.test_workflow_run_id- assert pause_model.state_object_key != ""- assert pause_model.resumed_at is None-- storage_content = storage.load(pause_model.state_object_key).decode()- resumption_context = WorkflowResumptionContext.loads(storage_content)- assert resumption_context.version == "1"- assert resumption_context.serialized_graph_runtime_state == graph_runtime_state.dumps()- persisted_entity = resumption_context.get_generate_entity()- assert isinstance(persisted_entity, WorkflowAppGenerateEntity)- assert persisted_entity.workflow_execution_id == self.test_workflow_run_id-- def test_state_persistence_and_retrieval(self, db_session_with_containers: Session):- """Test that pause state can be persisted and retrieved correctly."""- # Arrange- layer = self._create_pause_state_persistence_layer()-- # Create complex test data- complex_outputs = {- "nested": {"key": "value", "number": 42},- "list": [1, 2, 3, {"nested": "item"}],- "boolean": True,- "null_value": None,- }- complex_variables = {- ("node1", "var1"): "string_value",- ("node2", "var2"): {"complex": "object"},- ("node3", "var3"): [1, 2, 3],- }-- graph_runtime_state = self._create_graph_runtime_state(- outputs=complex_outputs,- total_tokens=250,- node_run_steps=10,- variables=complex_variables,- )-- command_channel = _TestCommandChannelImpl()- layer.initialize(graph_runtime_state, command_channel)-- event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")])-- # Act - Save pause state- layer.on_event(event)-- # Assert - Retrieve and verify- pause_entity = self.workflow_run_service._workflow_runs.get_workflow_pause(self.test_workflow_run_id)- assert pause_entity is not None- assert pause_entity.workflow_execution_id == self.test_workflow_run_id- assert pause_entity.get_pause_reasons() == event.reasons-- state_bytes = pause_entity.get_state()- resumption_context = WorkflowResumptionContext.loads(state_bytes.decode())- retrieved_state = GraphRuntimeState.from_snapshot(resumption_context.serialized_graph_runtime_state)-- assert retrieved_state.outputs == complex_outputs- assert retrieved_state.total_tokens == 250- assert retrieved_state.node_run_steps == 10- assert resumption_context.get_generate_entity().workflow_execution_id == self.test_workflow_run_id-- def test_database_transaction_handling(self, db_session_with_containers: Session):- """Test that database transactions are handled correctly."""- # Arrange- layer = self._create_pause_state_persistence_layer()- graph_runtime_state = self._create_graph_runtime_state(- outputs={"test": "transaction"},- total_tokens=50,+ queue = WorkflowAppQueueManager(entity.task_id, self.account.id, entity.invoke_from, AppMode.WORKFLOW)+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id=self.workflow.app_id)+ variable_pool = VariablePool.from_bootstrap(+ system_variables=build_system_variables(+ app_id=self.workflow.app_id, workflow_id=self.workflow.id, workflow_execution_id=self.run_id+ ))-- command_channel = _TestCommandChannelImpl()- layer.initialize(graph_runtime_state, command_channel)-- event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")])-- # Act- layer.on_event(event)-- # Assert - Verify data is committed and accessible in new session- with Session(bind=self.session.get_bind(), expire_on_commit=False) as new_session:- workflow_run = new_session.get(WorkflowRun, self.test_workflow_run_id)- assert workflow_run is not None- assert workflow_run.status == WorkflowExecutionStatus.PAUSED-- pause_model = new_session.scalars(- select(WorkflowPauseModel).where(WorkflowPauseModel.workflow_run_id == workflow_run.id)- ).first()- assert pause_model is not None- assert pause_model.workflow_run_id == self.test_workflow_run_id- assert pause_model.resumed_at is None- assert pause_model.state_object_key != ""-- def test_file_storage_integration(self, db_session_with_containers: Session):- """Test integration with file storage system."""- # Arrange- layer = self._create_pause_state_persistence_layer()-- # Create large state data to test storage- large_outputs = {"data": "x" * 10000} # 10KB of data- graph_runtime_state = self._create_graph_runtime_state(- outputs=large_outputs,- total_tokens=1000,+ for selector, value in (variables or {}).items():+ variable_pool.add(selector, value)+ usage = LLMUsage.empty_usage()+ usage.total_tokens = total_tokens+ runtime = RuntimeState(+ workflow_id=self.workflow.id,+ variable_pool=variable_pool,+ start_at=perf_counter(),+ llm_usage=usage,+ outputs=dict(outputs or {}),+ node_run_steps=node_run_steps,)-- command_channel = _TestCommandChannelImpl()- layer.initialize(graph_runtime_state, command_channel)-- event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")])-- # Act- layer.on_event(event)-- # Assert - Verify file was uploaded to storage- self.session.refresh(self.test_workflow_run)- pause_model = self.session.scalars(- select(WorkflowPauseModel).where(WorkflowPauseModel.workflow_run_id == self.test_workflow_run.id)- ).first()- assert pause_model is not None- assert pause_model.state_object_key != ""-- # Verify content in storage- storage_content = storage.load(pause_model.state_object_key).decode()- resumption_context = WorkflowResumptionContext.loads(storage_content)- assert resumption_context.serialized_graph_runtime_state == graph_runtime_state.dumps()- assert resumption_context.get_generate_entity().workflow_execution_id == self.test_workflow_run_id-- def test_workflow_with_different_creators(self, db_session_with_containers: Session):- """Test pause state with workflows created by different users."""- # Arrange - Create workflow with different creator- different_user_id = str(uuid.uuid4())- different_workflow = Workflow(- id=str(uuid.uuid4()),- tenant_id=self.test_tenant_id,- app_id=self.test_app_id,- type="workflow",- version="draft",- graph='{"nodes": [], "edges": []}',- features='{"file_upload": {"enabled": false}}',- created_by=different_user_id,- created_at=naive_utc_now(),+ graph = runner._init_graph(+ graph_config=self.workflow.graph_dict,+ graph_runtime_state=runtime,+ workflow_id=self.workflow.id,+ tenant_id=self.workflow.tenant_id,+ user_id=self.account.id,+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ call_depth=0,)-- different_workflow_run = WorkflowRun(- id=str(uuid.uuid4()),- tenant_id=self.test_tenant_id,- app_id=self.test_app_id,- workflow_id=different_workflow.id,- type="workflow",- triggered_from="debugging",- version="draft",- status=WorkflowExecutionStatus.RUNNING,- created_by=self.test_user_id, # Run created by different user- created_by_role="account",- created_at=naive_utc_now(),+ runs = SQLAlchemyWorkflowExecutionRepository(+ self.engine,+ tenant_id=self.workflow.tenant_id,+ user=self.account,+ app_id=self.workflow.app_id,+ triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,)-- self.session.add(different_workflow)- self.session.add(different_workflow_run)- self.session.commit()-- layer = self._create_pause_state_persistence_layer(- workflow_run=different_workflow_run,- workflow=different_workflow,+ nodes = SQLAlchemyWorkflowNodeExecutionRepository(+ self.engine,+ tenant_id=self.workflow.tenant_id,+ user=self.account,+ app_id=self.workflow.app_id,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,)-- graph_runtime_state = self._create_graph_runtime_state(- outputs={"creator_test": "different_creator"},- workflow_run_id=different_workflow_run.id,+ commands = InMemoryChannel()+ if pause:+ commands.send_command(PauseCommand(reason="integration pause"))+ entry = WorkflowEntry(+ graph=graph,+ graph_config=self.workflow.graph_dict,+ call_depth=0,+ graph_runtime_state=runtime,+ command_channel=commands,)-- command_channel = _TestCommandChannelImpl()- layer.initialize(graph_runtime_state, command_channel)-- event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")])-- # Act- layer.on_event(event)-- # Assert - Should use workflow creator (not run creator)- self.session.refresh(different_workflow_run)- pause_model = self.session.scalars(- select(WorkflowPauseModel).where(WorkflowPauseModel.workflow_run_id == different_workflow_run.id)- ).first()- assert pause_model is not None-- # Verify the state owner is the workflow creator- pause_entity = self.workflow_run_service._workflow_runs.get_workflow_pause(different_workflow_run.id)- assert pause_entity is not None- resumption_context = WorkflowResumptionContext.loads(pause_entity.get_state().decode())- assert resumption_context.get_generate_entity().workflow_execution_id == different_workflow_run.id-- def test_layer_ignores_non_pause_events(self, db_session_with_containers: Session):- """Test that layer ignores non-pause events."""- # Arrange- layer = self._create_pause_state_persistence_layer()- graph_runtime_state = self._create_graph_runtime_state()-- command_channel = _TestCommandChannelImpl()- layer.initialize(graph_runtime_state, command_channel)-- # Import other event types- from graphon.graph_events import (- GraphRunFailedEvent,- GraphRunStartedEvent,- GraphRunSucceededEvent,+ prepared = PreparedWorkflowRun(entry, WorkflowType.WORKFLOW, self.workflow.version, entity, ())+ aggregate = WorkflowRunAgg(+ prepared,+ runner,+ PauseStateLayerConfig(self.engine, self.workflow.created_by),+ workflow_execution_repository=runs,+ workflow_node_execution_repository=nodes,)-- # Act - Send non-pause events- layer.on_event(GraphRunStartedEvent())- layer.on_event(GraphRunSucceededEvent(outputs={"result": "success"}))- layer.on_event(GraphRunFailedEvent(error="test error", exceptions_count=1))-- # Assert - No pause state should be created- self.session.refresh(self.test_workflow_run)- assert self.test_workflow_run.status == WorkflowExecutionStatus.RUNNING-- pause_states = self.session.scalars(- select(WorkflowPauseModel).where(WorkflowPauseModel.workflow_run_id == self.test_workflow_run_id)- ).all()- assert len(pause_states) == 0-- def test_layer_requires_initialization(self, db_session_with_containers: Session):- """Test that layer requires proper initialization before handling events."""- # Arrange- layer = self._create_pause_state_persistence_layer()- # Don't initialize - graph_runtime_state should be uninitialized-- event = GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")])-- # Act & Assert - Should raise GraphEngineLayerNotInitializedError- with pytest.raises(GraphEngineLayerNotInitializedError):- layer.on_event(event)+ return prepared, aggregate++ def stored_pause(self) -> WorkflowPause:+ self.session.expire_all()+ pause = self.session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == self.run_id))+ assert pause is not None+ return pause+++def workflow_case(db_session_with_containers: Session) -> WorkflowCase:+ session = db_session_with_containers+ engine = session.get_bind()+ assert isinstance(engine, Engine)+ tenant = Tenant(name="Pause integration tenant")+ account = Account(name="Pause integration user", email=f"{uuid4()}@example.com")+ session.add_all([tenant, account])+ session.flush()+ session.add(+ TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole.OWNER, current=True)+ )+ graph: dict[str, object] = {+ "nodes": [{"id": "start", "data": StartNodeData(title="Start").model_dump(mode="json")}],+ "edges": [],+ }+ workflow = Workflow(+ id=str(uuid4()),+ tenant_id=tenant.id,+ app_id=str(uuid4()),+ type="workflow",+ version="draft",+ graph=json.dumps(graph),+ features="{}",+ created_by=account.id,+ )+ session.add(workflow)+ session.commit()+ return WorkflowCase(session, engine, account, workflow, str(uuid4()))+++def test_complete_pause_flow_with_real_dependencies(workflow_case: WorkflowCase) -> None:+ prepared, aggregate = workflow_case.prepare(outputs={"result": "intermediate"}, total_tokens=100, node_run_steps=5)+ with closing(aggregate.iter_events()) as events:+ assert isinstance(next(events), GraphRunStartedEvent)+ workflow_case.session.expire_all()+ run = workflow_case.session.get(WorkflowRun, workflow_case.run_id)+ assert run is not None+ assert run.status == WorkflowExecutionStatus.RUNNING+ assert workflow_case.session.scalar(select(WorkflowPause)) is None+ remaining = list(events)+ assert isinstance(remaining[-1], GraphRunPausedEvent)+ pause = workflow_case.stored_pause()+ workflow_case.session.refresh(run)+ assert run.status == WorkflowExecutionStatus.PAUSED+ assert run.finished_at is None+ assert run.outputs_dict == {"result": "intermediate"}+ assert run.total_tokens == 100+ assert run.total_steps == prepared.entry.graph_engine.runtime_state.node_run_steps+ assert pause.workflow_id == workflow_case.workflow.id+ assert pause.resumed_at is None+ snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())+ assert snapshot.serialized_graph_runtime_state == prepared.entry.graph_engine.runtime_state.dumps()+ assert snapshot.get_response_stream_filter().dumps() == prepared.entry.response_stream_filter.dumps()+ entity = snapshot.get_generate_entity()+ assert isinstance(entity, WorkflowAppGenerateEntity)+ assert entity.workflow_execution_id == workflow_case.run_id+++def test_state_persistence_and_retrieval(workflow_case: WorkflowCase) -> None:+ outputs = {"nested": {"key": "value", "number": 42}, "list": [1, 2, 3, {"nested": "item"}], "null": None}+ variables = {("node", "text"): "value", ("node", "object"): {"complex": "object"}, ("node", "list"): [1, 2, 3]}+ prepared, aggregate = workflow_case.prepare(+ outputs=outputs, variables=variables, total_tokens=250, node_run_steps=10+ )+ events = list(aggregate.iter_events())+ assert isinstance(events[-1], GraphRunPausedEvent)+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sessionmaker(workflow_case.engine))+ pause = repository.get_workflow_pause(workflow_case.run_id)+ assert pause is not None+ assert pause.workflow_execution_id == workflow_case.run_id+ assert pause.get_pause_reasons() == events[-1].reasons+ snapshot = WorkflowResumptionContext.loads(pause.get_state().decode())+ restored = RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state)+ assert restored.outputs == outputs+ assert restored.total_tokens == 250+ assert restored.node_run_steps == prepared.entry.graph_engine.runtime_state.node_run_steps+ for selector, expected in variables.items():+ variable = restored.variable_pool.get(selector)+ assert variable is not None+ assert variable.to_object() == expected+++def test_database_transaction_handling(workflow_case: WorkflowCase) -> None:+ _, aggregate = workflow_case.prepare(outputs={"test": "transaction"})+ assert isinstance(list(aggregate.iter_events())[-1], GraphRunPausedEvent)+ with Session(workflow_case.engine) as session:+ run = session.get(WorkflowRun, workflow_case.run_id)+ assert run is not None+ assert run.status == WorkflowExecutionStatus.PAUSED+ pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run.id))+ assert pause is not None+ assert pause.resumed_at is None+ assert storage.load(pause.state_object_key)+++def test_workflow_with_different_creators(workflow_case: WorkflowCase) -> None:+ creator = Account(name="Workflow creator", email=f"{uuid4()}@example.com")+ workflow_case.session.add(creator)+ workflow_case.session.flush()+ workflow_case.workflow.created_by = creator.id+ workflow_case.session.commit()+ _, aggregate = workflow_case.prepare()+ assert isinstance(list(aggregate.iter_events())[-1], GraphRunPausedEvent)+ pause = workflow_case.stored_pause()+ snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())+ run = workflow_case.session.get(WorkflowRun, workflow_case.run_id)+ assert run is not None+ assert run.created_by == workflow_case.account.id+ assert run.created_by != workflow_case.workflow.created_by+ assert snapshot.get_generate_entity().user_id == workflow_case.account.id+++def test_successful_execution_creates_no_pause(workflow_case: WorkflowCase) -> None:+ _, aggregate = workflow_case.prepare(pause=False)+ events = list(aggregate.iter_events())+ assert isinstance(events[-1], GraphRunSucceededEvent)+ assert not any(isinstance(event, GraphRunPausedEvent) for event in events)+ workflow_case.session.expire_all()+ run = workflow_case.session.get(WorkflowRun, workflow_case.run_id)+ assert run is not None+ assert run.status == WorkflowExecutionStatus.SUCCEEDED+ assert workflow_case.session.scalar(select(WorkflowPause)) is None+++def test_active_execution_cannot_publish_a_durable_pause(workflow_case: WorkflowCase) -> None:+ prepared, aggregate = workflow_case.prepare()+ with prepared.entry.graph_engine.runtime_state.graph_execution.track_execution():+ events = list(aggregate.iter_events())+ assert isinstance(events[-1], GraphRunFailedEvent)+ assert "during active execution" in events[-1].error+ assert not any(isinstance(event, GraphRunPausedEvent) for event in events)+ workflow_case.session.expire_all()+ run = workflow_case.session.get(WorkflowRun, workflow_case.run_id)+ assert run is not None+ assert run.status == WorkflowExecutionStatus.FAILED+ assert workflow_case.session.scalar(select(WorkflowPause)) is Nonediff --git a/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.py b/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.pyindex 27d0b452456..6b7f4b07f9f 100644--- a/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.py+++ b/api/tests/test_containers_integration_tests/core/workflow/test_human_input_resume_node_execution.py@@ -3,11 +3,10 @@ import uuidimport pytestfrom sqlalchemy import delete, select-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmakerfrom core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity-from core.app.workflow.layers import PersistenceWorkflowInfo, WorkflowPersistenceLayerfrom core.repositories.human_input_repository import HumanInputFormRepository, HumanInputFormRepositoryImplfrom core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepositoryfrom core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository@@ -17,16 +16,17 @@ from core.workflow.nodes.human_input.callback import (from core.workflow.nodes.human_input.entities import HumanInputNodeData, UserActionConfigfrom core.workflow.nodes.human_input.enums import HumanInputFormStatusfrom core.workflow.system_variables import build_system_variables+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannelfrom graphon.enums import WorkflowTypefrom graphon.graph import Graph-from graphon.graph_engine import GraphEngine-from graphon.graph_engine.command_channels import InMemoryChannelfrom graphon.nodes.end.end_node import EndNodefrom graphon.nodes.end.entities import EndNodeData+from graphon.nodes.human_input.entities import HumanInputNodeData as GraphonHumanInputNodeDatafrom graphon.nodes.human_input.human_input_node import HumanInputNodefrom graphon.nodes.start.entities import StartNodeDatafrom graphon.nodes.start.start_node import StartNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom libs.datetime_utils import naive_utc_nowfrom models import Accountfrom models.account import AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus@@ -34,10 +34,12 @@ from models.enums import CreatorUserRole, WorkflowRunTriggeredFromfrom models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipientfrom models.model import App, AppMode, IconTypefrom models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun+from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer+from services.workflow_run_index import WorkflowRunIndexfrom tests.workflow_test_utils import build_test_graph_init_params-def _build_runtime_state(workflow_execution_id: str, app_id: str, workflow_id: str, user_id: str) -> GraphRuntimeState:+def _build_runtime_state(workflow_execution_id: str, app_id: str, workflow_id: str, user_id: str) -> RuntimeState:variable_pool = VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id=workflow_execution_id,@@ -48,11 +50,11 @@ def _build_runtime_state(workflow_execution_id: str, app_id: str, workflow_id: suser_inputs={},conversation_variables=[],)- return GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter())def _build_graph(- runtime_state: GraphRuntimeState,+ runtime_state: RuntimeState,tenant_id: str,app_id: str,workflow_id: str,@@ -75,8 +77,8 @@ def _build_graph(start_node = StartNode(node_id="start",data=start_data,- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)human_data = HumanInputNodeData(@@ -93,9 +95,9 @@ def _build_graph()human_node = HumanInputNode(node_id="human",- data=human_data,- graph_init_params=params,- graph_runtime_state=runtime_state,+ data=GraphonHumanInputNodeData(title=human_data.title),+ init_params=params,+ runtime_state=runtime_state,hitl_callback=hitl_callback,)@@ -107,8 +109,8 @@ def _build_graph(end_node = EndNode(node_id="end",data=end_data,- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)return (@@ -237,7 +239,9 @@ class TestHumanInputResumeNodeExecutionIntegration:self.session.execute(delete(Tenant).where(Tenant.id == self.tenant.id))self.session.commit()- def _build_persistence_layer(self, execution_id: str) -> WorkflowPersistenceLayer:+ def _build_run_layers(+ self, execution_id: str, *, resuming: bool+ ) -> tuple[WorkflowRunIndex, WorkflowPersistenceLayer]:generate_entity = _build_generate_entity(tenant_id=self.tenant.id,app_id=self.app.id,@@ -245,21 +249,24 @@ class TestHumanInputResumeNodeExecutionIntegration:workflow_execution_id=execution_id,user_id=self.account.id,)+ session_factory = sessionmaker(bind=self.session.get_bind(), expire_on_commit=False)execution_repo = SQLAlchemyWorkflowExecutionRepository(- session_factory=self.session.get_bind(),+ session_factory=session_factory,tenant_id=self.tenant.id,user=self.account,app_id=self.app.id,triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,)node_execution_repo = SQLAlchemyWorkflowNodeExecutionRepository(- session_factory=self.session.get_bind(),+ session_factory=session_factory,tenant_id=self.tenant.id,user=self.account,app_id=self.app.id,triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,)- return WorkflowPersistenceLayer(+ history = node_execution_repo.get_by_workflow_execution(execution_id, include_paused=True) if resuming else ()+ run_index = WorkflowRunIndex(history)+ persistence = WorkflowPersistenceLayer(application_generate_entity=generate_entity,workflow_info=PersistenceWorkflowInfo(workflow_id=self.workflow.id,@@ -270,15 +277,18 @@ class TestHumanInputResumeNodeExecutionIntegration:workflow_execution_repository=execution_repo,workflow_node_execution_repository=node_execution_repo,)+ persistence.set_node_run_indices(run_index.indices)+ persistence.set_node_execution_history(history)+ return run_index, persistence- def _run_graph(self, graph: Graph, runtime_state: GraphRuntimeState, execution_id: str) -> None:- engine = GraphEngine(- workflow_id=self.workflow.id,+ def _run_graph(self, graph: Graph, runtime_state: RuntimeState, execution_id: str) -> None:+ engine = Engine(graph=graph,- graph_runtime_state=runtime_state,+ runtime_state=runtime_state,command_channel=InMemoryChannel(),)- engine.layer(self._build_persistence_layer(execution_id))+ for layer in self._build_run_layers(execution_id, resuming=runtime_state.graph_execution.paused):+ engine.add_layer(layer)for _ in engine.run():continue@@ -302,9 +312,16 @@ class TestHumanInputResumeNodeExecutionIntegration:form_repository,)self._run_graph(paused_graph, runtime_state, execution_id)+ human_execution_id = self.session.scalar(+ select(WorkflowNodeExecutionModel.id).where(+ WorkflowNodeExecutionModel.workflow_run_id == execution_id,+ WorkflowNodeExecutionModel.node_id == "human",+ )+ )+ assert human_execution_id is not Nonesnapshot = runtime_state.dumps()- resumed_state = GraphRuntimeState.from_snapshot(snapshot)+ resumed_state = RuntimeState.from_snapshot(snapshot)form = self.session.scalar(select(HumanInputForm).where(HumanInputForm.tenant_id == self.tenant.id,@@ -335,6 +352,17 @@ class TestHumanInputResumeNodeExecutionIntegration:)records = self.session.execute(stmt).scalars().all()assert len(records) == 1- assert records[0].status != "paused"+ assert records[0].id == human_execution_id+ assert records[0].status == "succeeded"assert records[0].triggered_from == WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUNassert records[0].created_by_role == CreatorUserRole.ACCOUNT+ executions = self.session.scalars(+ select(WorkflowNodeExecutionModel)+ .where(WorkflowNodeExecutionModel.workflow_run_id == execution_id)+ .order_by(WorkflowNodeExecutionModel.index)+ ).all()+ assert [(execution.node_id, execution.index) for execution in executions] == [+ ("start", 1),+ ("human", 2),+ ("end", 3),+ ]diff --git a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py b/api/tests/test_containers_integration_tests/services/test_app_generate_service.pyindex 1f8825ed3a1..233c8fbbb38 100644--- a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py+++ b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py@@ -99,7 +99,7 @@ class TestAppGenerateService:mock_advanced_chat_generator_instance.generate.return_value = ["advanced_chat_response"]mock_advanced_chat_generator_instance.single_iteration_generate.return_value = ["single_iteration_response"]mock_advanced_chat_generator_instance.single_loop_generate.return_value = ["single_loop_response"]- mock_advanced_chat_generator_instance.retrieve_events.return_value = ["advanced_chat_events"]+ mock_advanced_chat_generator.retrieve_events.return_value = ["advanced_chat_events"]mock_advanced_chat_generator_instance.convert_to_event_stream.return_value = ["advanced_chat_stream"]mock_advanced_chat_generator.convert_to_event_stream.return_value = ["advanced_chat_stream"]@@ -350,10 +350,8 @@ class TestAppGenerateService:assert result == ["test_response"]# Verify advanced chat generator was called- mock_external_service_dependencies["advanced_chat_generator"].return_value.retrieve_events.assert_called_once()- mock_external_service_dependencies[- "advanced_chat_generator"- ].return_value.convert_to_event_stream.assert_called_once()+ mock_external_service_dependencies["advanced_chat_generator"].retrieve_events.assert_called_once()+ mock_external_service_dependencies["advanced_chat_generator"].convert_to_event_stream.assert_called_once()def test_generate_workflow_mode_success(self, db_session_with_containers: Session, mock_external_service_dependenciesdiff --git a/api/tests/test_containers_integration_tests/services/test_workflow_service.py b/api/tests/test_containers_integration_tests/services/test_workflow_service.pyindex b200504f907..4817f445e23 100644--- a/api/tests/test_containers_integration_tests/services/test_workflow_service.py+++ b/api/tests/test_containers_integration_tests/services/test_workflow_service.py@@ -1645,7 +1645,7 @@ class TestWorkflowService:import uuidfrom datetime import datetime- from graphon.graph_events import NodeRunSucceededEvent+ from graphon.engine_events import NodeRunSucceededEventfrom graphon.node_events import NodeRunResultnode = _ExecutedNode(@@ -1714,7 +1714,7 @@ class TestWorkflowService:import uuidfrom datetime import datetime- from graphon.graph_events import NodeRunFailedEvent+ from graphon.engine_events import NodeRunFailedEventfrom graphon.node_events import NodeRunResultnode = _ExecutedNode(@@ -1778,7 +1778,7 @@ class TestWorkflowService:import uuidfrom datetime import datetime- from graphon.graph_events import NodeRunFailedEvent+ from graphon.engine_events import NodeRunFailedEventfrom graphon.node_events import NodeRunResultnode = _ExecutedNode(diff --git a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_event_snapshot_service.py b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_event_snapshot_service.pyindex 4b5a0fb8bc6..86bd5985b5a 100644--- a/api/tests/test_containers_integration_tests/services/workflow/test_workflow_event_snapshot_service.py+++ b/api/tests/test_containers_integration_tests/services/workflow/test_workflow_event_snapshot_service.py@@ -16,7 +16,7 @@ from core.workflow.nodes.human_input.entities import SelectInputConfig, StringLifrom core.workflow.nodes.human_input.enums import HumanInputFormStatus, ValueSourceTypefrom core.workflow.nodes.human_input.pause_reason import HumanInputRequiredfrom graphon.enums import WorkflowExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom models.enums import CreatorUserRolefrom models.human_input import HumanInputFormfrom models.model import AppMode@@ -79,7 +79,7 @@ def _build_resumption_context(workflow_run_id: str) -> WorkflowResumptionContextcall_depth=0,workflow_execution_id=workflow_run_id,)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)runtime_state.variable_pool.add(("start", "options"), ["approve", "reject"])wrapper = _WorkflowGenerateEntityWrapper(entity=generate_entity)return WorkflowResumptionContext(diff --git a/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.py b/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.pyindex 068c29740db..a2600d2605e 100644--- a/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_mail_human_input_delivery_task.py@@ -21,7 +21,7 @@ from core.workflow.human_input_adapter import (from core.workflow.nodes.human_input.entities import HumanInputNodeDatafrom extensions.ext_storage import storagefrom graphon.enums import WorkflowExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom models.account import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRolefrom models.enums import CreatorUserRole, WorkflowRunTriggeredFromfrom models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient@@ -138,7 +138,7 @@ def _create_workflow_pause_state()db_session_with_containers.add(workflow_run)- runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)resumption_context = WorkflowResumptionContext(generate_entity={"type": AppMode.WORKFLOW,diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.pyindex e345c79f368..9a73057d179 100644--- a/api/tests/unit_tests/configs/test_dify_config.py+++ b/api/tests/unit_tests/configs/test_dify_config.py@@ -105,11 +105,11 @@ def test_dify_config() -> None:assert config.AGENT_SHELL_ENABLED is Trueassert config.SENTRY_TRACES_SAMPLE_RATE == 1.0assert config.TEMPLATE_TRANSFORM_MAX_LENGTH == 400_000- assert config.GRAPH_ENGINE_SCALE_UP_THRESHOLD == 0assert config.APP_MAX_EXECUTION_TIME == 3600assert config.WORKFLOW_MAX_EXECUTION_TIME == 3600assert config.OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES == 780assert config.OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS == 3900+ assert config.GRAPH_ENGINE_MAX_WORKERS == 10# annotated field with custom configured valueassert config.HTTP_REQUEST_MAX_READ_TIMEOUT == 300diff --git a/api/tests/unit_tests/controllers/console/explore/test_installed_app_admission.py b/api/tests/unit_tests/controllers/console/explore/test_installed_app_admission.pyindex bc1e3d61717..49a29544119 100644--- a/api/tests/unit_tests/controllers/console/explore/test_installed_app_admission.py+++ b/api/tests/unit_tests/controllers/console/explore/test_installed_app_admission.py@@ -28,10 +28,12 @@ from controllers.console.explore.installed_app_admission import get_installed_apfrom controllers.console.flask_admission import console_account_admissionfrom enums import DeploymentEdition, WebAppAccessModefrom extensions.ext_login import DifyLoginManager, unauthorized_handler+from graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom libs.external_api import ExternalApifrom machinery.context import RequestContext-from models import Account, App, AppMode, InstalledApp, Tenant+from models import Account, App, AppMode, InstalledApp, Tenant, WorkflowRunfrom models.account import AccountStatus+from models.enums import CreatorUserRole, WorkflowRunTriggeredFromfrom repositories.installed_app_repository import SQLAlchemyInstalledAppRepositoryfrom services.app_task_service import AppTaskControlServicefrom services.installed_app_access_service import InstalledAppAccessService, InstalledAppRef@@ -554,6 +556,32 @@ class _StopServices:_TASK_ID = "task-with-non-uuid-id"+_WORKFLOW_TASK_ID = "22222222-2222-4222-8222-222222222222"+++def running_workflow(+ harness: _Harness,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> WorkflowRun:+ run = WorkflowRun(+ id=str(uuid4()),+ tenant_id=harness.target_app.tenant_id,+ app_id=harness.target_app.id,+ workflow_id=str(uuid4()),+ task_id=_WORKFLOW_TASK_ID,+ type=WorkflowType.CHAT,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.RUNNING,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=harness.account.id,+ )+ with sqlite_session_factory.begin() as session:+ session.add(run)+ monkeypatch.setattr(type(app_task_module.db), "engine", property(lambda _: sqlite_session_factory.kw["bind"]))+ return run@pytest.fixture@@ -594,8 +622,8 @@ def stop_redis(return redis-def _stop_url(harness: _Harness, message_kind: str) -> str:- return f"/installed-apps/{harness.installed_app.id}/{message_kind}-messages/{_TASK_ID}/stop"+def _stop_url(harness: _Harness, message_kind: str, task_id: str = _TASK_ID) -> str:+ return f"/installed-apps/{harness.installed_app.id}/{message_kind}-messages/{task_id}/stop"def _set_app_mode(harness: _Harness, session_factory: sessionmaker[Session], mode: AppMode) -> None:@@ -617,24 +645,30 @@ def _set_app_mode(harness: _Harness, session_factory: sessionmaker[Session], moddef test_stop_handlers_preserve_mode_specific_commands_and_response(harness: _Harness,stop_redis: _StopRedis,+ running_workflow: WorkflowRun,sqlite_session_factory: sessionmaker[Session],message_kind: str,mode: AppMode,) -> None:_set_app_mode(harness, sqlite_session_factory, mode)+ task_id = _WORKFLOW_TASK_ID if mode == AppMode.ADVANCED_CHAT else _TASK_ID- response = harness.app.test_client().post(_stop_url(harness, message_kind))+ response = harness.app.test_client().post(_stop_url(harness, message_kind, task_id))_assert_json_response(response, status=200, body={"result": "success"})- assert stop_redis.reads == [f"generate_task_belong:{_TASK_ID}"]- assert stop_redis.values[f"generate_task_stopped:{_TASK_ID}"] == b"1"- assert stop_redis.expirations[f"generate_task_stopped:{_TASK_ID}"] == 600+ assert stop_redis.reads == ([] if mode == AppMode.ADVANCED_CHAT else [f"generate_task_belong:{task_id}"])+ assert stop_redis.values[f"generate_task_stopped:{task_id}"] == b"1"+ assert stop_redis.expirations[f"generate_task_stopped:{task_id}"] == 600if mode == AppMode.ADVANCED_CHAT:- command_key = f"workflow:{_TASK_ID}:commands"+ command_key = f"workflow:{task_id}:commands"assert set(stop_redis.commands) == {command_key}assert [json.loads(command) for command in stop_redis.commands[command_key]] == [- {"command_type": "abort", "payload": None, "reason": "User requested stop"}+ {"command_type": "abort", "reason": "User requested stop"}]+ with sqlite_session_factory() as session:+ run = session.get(WorkflowRun, running_workflow.id)+ assert run is not None+ assert run.stop_requested_at is not Noneelse:assert stop_redis.commands == {}assert harness.state.permission_calls == [(harness.account.id, harness.target_app.id)]@@ -647,33 +681,48 @@ def test_stop_handlers_preserve_mode_specific_commands_and_response(def test_stop_handlers_preserve_mode_specific_behavior_when_task_ownership_does_not_match(harness: _Harness,stop_redis: _StopRedis,+ running_workflow: WorkflowRun,sqlite_session_factory: sessionmaker[Session],message_kind: str,mode: AppMode,ownership: str,) -> None:_set_app_mode(harness, sqlite_session_factory, mode)- owner_key = f"generate_task_belong:{_TASK_ID}"- if ownership == "missing":+ task_id = _WORKFLOW_TASK_ID if mode == AppMode.ADVANCED_CHAT else _TASK_ID+ owner_key = f"generate_task_belong:{task_id}"+ if mode == AppMode.ADVANCED_CHAT:+ stop_redis.values[owner_key] = f"account-{harness.account.id}".encode()+ with sqlite_session_factory.begin() as session:+ run = session.get(WorkflowRun, running_workflow.id)+ assert run is not None+ if ownership == "missing":+ session.delete(run)+ elif ownership == "different-account":+ run.created_by = str(uuid4())+ else:+ run.created_by_role = CreatorUserRole.END_USER+ elif ownership == "missing":stop_redis.values.pop(owner_key)elif ownership == "different-account":stop_redis.values[owner_key] = b"account-someone-else"else:stop_redis.values[owner_key] = f"end-user-{harness.account.id}".encode()- response = harness.app.test_client().post(_stop_url(harness, message_kind))+ response = harness.app.test_client().post(_stop_url(harness, message_kind, task_id))_assert_json_response(response, status=200, body={"result": "success"})- assert stop_redis.reads == [owner_key]- assert f"generate_task_stopped:{_TASK_ID}" not in stop_redis.values+ assert stop_redis.reads == ([] if mode == AppMode.ADVANCED_CHAT else [owner_key])+ assert f"generate_task_stopped:{task_id}" not in stop_redis.values+ assert stop_redis.operations == []+ assert stop_redis.commands == {}if mode == AppMode.ADVANCED_CHAT:- assert stop_redis.operations == ["graph_command"]- assert [json.loads(command) for command in stop_redis.commands[f"workflow:{_TASK_ID}:commands"]] == [- {"command_type": "abort", "payload": None, "reason": "User requested stop"}- ]- else:- assert stop_redis.operations == []- assert stop_redis.commands == {}+ with sqlite_session_factory() as session:+ run = session.get(WorkflowRun, running_workflow.id)+ if ownership == "missing":+ assert run is None+ else:+ assert run is not None+ assert run.stop_requested_at is None@pytest.mark.parametrize(@@ -709,11 +758,13 @@ def test_stop_handlers_reject_wrong_modes_without_sending_commands(def test_stop_handlers_use_admitted_mode_when_app_is_removed(harness: _Harness,stop_redis: _StopRedis,+ running_workflow: WorkflowRun,sqlite_session_factory: sessionmaker[Session],message_kind: str,mode: AppMode,) -> None:_set_app_mode(harness, sqlite_session_factory, mode)+ task_id = _WORKFLOW_TASK_ID if mode == AppMode.ADVANCED_CHAT else _TASK_IDdef remove_app() -> None:with sqlite_session_factory.begin() as session:@@ -723,15 +774,20 @@ def test_stop_handlers_use_admitted_mode_when_app_is_removed(harness.state.permission_action = remove_app- response = harness.app.test_client().post(_stop_url(harness, message_kind))+ response = harness.app.test_client().post(_stop_url(harness, message_kind, task_id))_assert_json_response(response, status=200, body={"result": "success"})assert harness.state.permission_calls == [(harness.account.id, harness.target_app.id)]- assert stop_redis.reads == [f"generate_task_belong:{_TASK_ID}"]- assert stop_redis.values[f"generate_task_stopped:{_TASK_ID}"] == b"1"+ assert stop_redis.reads == ([] if mode == AppMode.ADVANCED_CHAT else [f"generate_task_belong:{task_id}"])+ assert stop_redis.values[f"generate_task_stopped:{task_id}"] == b"1"assert stop_redis.operations == (["legacy_flag", "graph_command"] if mode == AppMode.ADVANCED_CHAT else ["legacy_flag"])+ if mode == AppMode.ADVANCED_CHAT:+ with sqlite_session_factory() as session:+ run = session.get(WorkflowRun, running_workflow.id)+ assert run is not None+ assert run.stop_requested_at is not None@pytest.mark.parametrize("message_kind", ["completion", "chat"])@@ -765,12 +821,14 @@ def test_stop_handlers_enforce_admission_before_sending_commands(def test_stop_handlers_propagate_redis_failure_to_existing_http_error_handler(harness: _Harness,stop_redis: _StopRedis,+ running_workflow: WorkflowRun,sqlite_session_factory: sessionmaker[Session],message_kind: str,mode: AppMode,failure_stage: str,) -> None:_set_app_mode(harness, sqlite_session_factory, mode)+ task_id = _WORKFLOW_TASK_ID if mode == AppMode.ADVANCED_CHAT else _TASK_IDfailure = RedisConnectionError("Redis unavailable")if failure_stage == "read":stop_redis.read_error = failure@@ -782,7 +840,7 @@ def test_stop_handlers_propagate_redis_failure_to_existing_http_error_handler(exceptions.append(exception)with got_request_exception.connected_to(capture_exception):- response = harness.app.test_client().post(_stop_url(harness, message_kind))+ response = harness.app.test_client().post(_stop_url(harness, message_kind, task_id))_assert_json_response(response,@@ -790,14 +848,21 @@ def test_stop_handlers_propagate_redis_failure_to_existing_http_error_handler(body={"code": "unknown", "message": "Internal Server Error", "status": 500},)assert any(exception is failure for exception in exceptions)- assert f"generate_task_stopped:{_TASK_ID}" not in stop_redis.values+ assert f"generate_task_stopped:{task_id}" not in stop_redis.valuesassert stop_redis.commands == {}assert stop_redis.operations == (["legacy_flag"] if failure_stage == "flag" else [])+ if mode == AppMode.ADVANCED_CHAT:+ assert stop_redis.reads == []+ with sqlite_session_factory() as session:+ run = session.get(WorkflowRun, running_workflow.id)+ assert run is not None+ assert run.stop_requested_at is not Nonedef test_chat_stop_preserves_success_and_legacy_flag_when_graph_redis_fails(harness: _Harness,stop_redis: _StopRedis,+ running_workflow: WorkflowRun,sqlite_session_factory: sessionmaker[Session],caplog: pytest.LogCaptureFixture,) -> None:@@ -805,11 +870,16 @@ def test_chat_stop_preserves_success_and_legacy_flag_when_graph_redis_fails(failure = RedisConnectionError("Graph channel unavailable")stop_redis.command_error = failure- response = harness.app.test_client().post(_stop_url(harness, "chat"))+ response = harness.app.test_client().post(_stop_url(harness, "chat", _WORKFLOW_TASK_ID))_assert_json_response(response, status=200, body={"result": "success"})+ assert stop_redis.reads == []assert stop_redis.operations == ["legacy_flag", "graph_command"]- assert stop_redis.values[f"generate_task_stopped:{_TASK_ID}"] == b"1"- assert stop_redis.expirations[f"generate_task_stopped:{_TASK_ID}"] == 600+ assert stop_redis.values[f"generate_task_stopped:{_WORKFLOW_TASK_ID}"] == b"1"+ assert stop_redis.expirations[f"generate_task_stopped:{_WORKFLOW_TASK_ID}"] == 600assert stop_redis.commands == {}assert any(record.exc_info is not None and record.exc_info[1] is failure for record in caplog.records)+ with sqlite_session_factory() as session:+ run = session.get(WorkflowRun, running_workflow.id)+ assert run is not None+ assert run.stop_requested_at is not Nonediff --git a/api/tests/unit_tests/controllers/console/explore/test_trial_generation.py b/api/tests/unit_tests/controllers/console/explore/test_trial_generation.pyindex 7a63f03ba82..3ad37acf1c7 100644--- a/api/tests/unit_tests/controllers/console/explore/test_trial_generation.py+++ b/api/tests/unit_tests/controllers/console/explore/test_trial_generation.py@@ -30,6 +30,7 @@ from graphon.model_runtime.errors.invoke import InvokeErrorfrom libs.external_api import ExternalApifrom models import Account, AccountTrialAppRecord, App, AppMode, Tenant, TrialAppfrom models.account import AccountStatus+from models.enums import CreatorUserRolefrom repositories.trial_app_repository import TrialAppRepositoryfrom services.account_errors import AccountNotFoundErrorfrom services.app_definition_query_service import AppDefinitionUnavailableError@@ -108,14 +109,31 @@ class _Runtime:@dataclassclass _Tasks:calls: list[str] = field(default_factory=list)- chat_calls: list[tuple[str, str, str]] = field(default_factory=list)+ chat_calls: list[tuple[str, str]] = field(default_factory=list)- def stop_task(self, *, task_id: str, invoke_from: InvokeFrom, user_id: str, app_mode: AppMode) -> None:+ def stop_task(self, *, task_id: str, invoke_from: InvokeFrom, user_id: str) -> None:assert invoke_from == InvokeFrom.EXPLORE- self.chat_calls.append((task_id, user_id, app_mode))-- def stop_workflow_task_no_user_check(self, *, task_id: str) -> None:- self.calls.append(task_id)+ self.chat_calls.append((task_id, user_id))++ def stop_workflow_task(+ self,+ *,+ task_id: str,+ tenant_id: str,+ app_id: str,+ app_mode: AppMode,+ owner: tuple[CreatorUserRole, str] | None,+ ) -> None:+ assert tenant_id+ assert app_id+ if app_mode == AppMode.ADVANCED_CHAT:+ assert owner is not None+ assert owner[0] == CreatorUserRole.ACCOUNT+ self.chat_calls.append((task_id, owner[1]))+ else:+ assert owner is None+ assert app_mode == AppMode.WORKFLOW+ self.calls.append(task_id)@dataclass(frozen=True)@@ -436,7 +454,7 @@ def test_last_trial_can_be_stopped_without_allowing_another_generation(harness:stopped = harness.post("chat-messages/task-last/stop")assert stopped.status_code == 200- assert harness.services.app_tasks.chat_calls == [("task-last", harness.account.id, mode)]+ assert harness.services.app_tasks.chat_calls == [("task-last", harness.account.id)]assert harness.usage() == 3unavailable = harness.app.test_client().post(harness.url("chat-messages/task-last/stop", app_id=str(uuid4())))_assert_error(unavailable, 403, "trial_app_not_allowed")diff --git a/api/tests/unit_tests/controllers/console/explore/test_workflow.py b/api/tests/unit_tests/controllers/console/explore/test_workflow.pyindex e44c82310f7..1896d35f85e 100644--- a/api/tests/unit_tests/controllers/console/explore/test_workflow.py+++ b/api/tests/unit_tests/controllers/console/explore/test_workflow.py@@ -9,9 +9,13 @@ from redis.exceptions import ConnectionError as RedisConnectionErrorfrom sqlalchemy.orm import Session, sessionmakerimport controllers.console.explore.workflow as workflow_module+import services.app_task_service as app_task_modulefrom core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError+from graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom graphon.model_runtime.errors.invoke import InvokeErrorfrom models import App, AppMode, InstalledApp, Tenant+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.workflow import WorkflowRunfrom repositories.installed_app_repository import SQLAlchemyInstalledAppRepositoryfrom services.account_errors import AccountNotFoundErrorfrom services.app_definition_query_service import AppDefinitionUnavailableError@@ -24,14 +28,15 @@ from tests.unit_tests.controllers.console.explore.test_installed_app_admission i_assert_json_response,_Harness,_set_app_mode,+ _stop_global_redis,harness,)from tests.unit_tests.services.test_app_task_service import _StopRedis-__all__ = ["harness"]+__all__ = ["_stop_global_redis", "harness"]_LAST_USED_AT = datetime(2026, 9, 1, 12, 0, 0)-_TASK_ID = "workflow-task-with-non-uuid-id"+_TASK_ID = "a1111111-1111-1111-1111-111111111111"_STOP_KEY = f"generate_task_stopped:{_TASK_ID}"_COMMAND_KEY = f"workflow:{_TASK_ID}:commands"@@ -94,12 +99,29 @@ def runtime(monkeypatch: pytest.MonkeyPatch,sqlite_session_factory: sessionmaker[Session],stop_redis: _StopRedis,+ _stop_global_redis: _StopRedis,) -> _Runtime:_set_app_mode(harness, sqlite_session_factory, AppMode.WORKFLOW)with sqlite_session_factory.begin() as session:installation = session.get(InstalledApp, harness.installed_app.id)assert installation is not Noneinstallation.last_used_at = _LAST_USED_AT+ with sqlite_session_factory.begin() as session:+ session.add(+ WorkflowRun(+ id="run-1",+ tenant_id=harness.target_app.tenant_id,+ app_id=harness.target_app.id,+ workflow_id="workflow-1",+ task_id=_TASK_ID,+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.RUNNING,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=harness.account.id,+ )+ )runtime = _Runtime(sqlite_session_factory, harness.installed_app.id)services = _Services(installed_apps=_InstalledAppServices(@@ -111,6 +133,7 @@ def runtime(app_tasks=AppTaskControlService(redis_client=stop_redis),)monkeypatch.setattr(workflow_module, "application_services", lambda: services)+ monkeypatch.setattr(type(app_task_module.db), "engine", property(lambda _: sqlite_session_factory.kw["bind"]))harness.api.add_resource(workflow_module.InstalledAppWorkflowRunApi,"/installed-apps/<uuid:installed_app_id>/workflows/run",@@ -439,7 +462,7 @@ def test_workflow_stop_sets_both_signals_in_order_without_reading_task_ownershipassert stop_redis.expirations[_STOP_KEY] == 600assert set(stop_redis.commands) == {_COMMAND_KEY}assert [json.loads(command) for command in stop_redis.commands[_COMMAND_KEY]] == [- {"command_type": "abort", "payload": None, "reason": "User requested stop"}+ {"command_type": "abort", "reason": "User requested stop"}]assert stop_redis.expirations[_COMMAND_KEY] == 3600assert runtime.calls == []diff --git a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.pyindex f1ca241c03d..a3b3ddfd15f 100644--- a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py+++ b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py@@ -596,18 +596,12 @@ def test_draft_node_last_run_raises_not_found_when_execution_missing(handler(api, snippet=snippet, node_id="llm-1")-def test_workflow_task_stop_uses_queue_flag_and_graph_command(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- set_stop_flag = Mock()- send_stop_command = Mock()+def test_workflow_task_stop_uses_snippet_owner_scope(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:+ stop_task = Mock()monkeypatch.setattr(- snippet_workflow_module.AppQueueManager,- "set_stop_flag_no_user_check",- set_stop_flag,- )- monkeypatch.setattr(- snippet_workflow_module,- "GraphEngineManager",- Mock(return_value=Mock(send_stop_command=send_stop_command)),+ snippet_workflow_module.AppTaskService,+ "stop_workflow_task",+ stop_task,)api = snippet_workflow_module.SnippetWorkflowTaskStopApi()@@ -617,5 +611,10 @@ def test_workflow_task_stop_uses_queue_flag_and_graph_command(app: Flask, monkeyresult = handler(api, snippet=_snippet(), task_id="task-1")assert result == {"result": "success"}- set_stop_flag.assert_called_once_with("task-1")- send_stop_command.assert_called_once_with("task-1")+ stop_task.assert_called_once_with(+ tenant_id="tenant-1",+ app_id="snippet-1",+ task_id="task-1",+ app_mode=snippet_workflow_module.AppMode.WORKFLOW,+ owner=None,+ )diff --git a/api/tests/unit_tests/controllers/console/test_human_input_form.py b/api/tests/unit_tests/controllers/console/test_human_input_form.pyindex d799c634505..cf6137f1f80 100644--- a/api/tests/unit_tests/controllers/console/test_human_input_form.py+++ b/api/tests/unit_tests/controllers/console/test_human_input_form.py@@ -21,6 +21,8 @@ from controllers.console.human_input_form import (_jsonify_form_definition,)from core.workflow.human_input_policy import HumanInputSurface+from core.workflow.nodes.human_input.entities import SelectInputConfig, StringListSource+from core.workflow.nodes.human_input.enums import ValueSourceTypefrom graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom models import Accountfrom models.enums import CreatorUserRole, WorkflowRunTriggeredFrom@@ -98,6 +100,10 @@ def test_get_form_definition_success(app: Flask, monkeypatch: pytest.MonkeyPatchexpiration = datetime(2024, 1, 1, tzinfo=UTC)definition = SimpleNamespace(model_dump=lambda: {"fields": ["a"]})form = SimpleNamespace(tenant_id="tenant-1", get_definition=lambda: definition, expiration_time=expiration)+ resolved_input = SelectInputConfig(+ output_variable_name="choice",+ option_source=StringListSource(type=ValueSourceType.CONSTANT, value=["a"]),+ )class _ServiceStub:def __init__(self, *_args, **_kwargs):@@ -106,6 +112,10 @@ def test_get_form_definition_success(app: Flask, monkeypatch: pytest.MonkeyPatchdef get_form_definition_by_token_for_console(self, _token):return form+ def resolve_form_inputs(self, resolved_form):+ assert resolved_form is form+ return [resolved_input]+monkeypatch.setattr("controllers.console.human_input_form.HumanInputService", _ServiceStub)api = ConsoleHumanInputFormApi()@@ -116,6 +126,7 @@ def test_get_form_definition_success(app: Flask, monkeypatch: pytest.MonkeyPatchpayload = json.loads(response.get_data(as_text=True))assert payload["fields"] == ["a"]+ assert payload["inputs"] == [resolved_input.model_dump(mode="json")]def test_get_form_definition_not_found(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:@@ -378,9 +389,11 @@ def test_workflow_events_snapshot_can_continue_across_pauses(app: Flask, monkeypdef get_workflow_run_by_id_and_tenant_id(self, **_kwargs):return workflow_run- workflow_generator = Mock()- workflow_generator.convert_to_event_stream.return_value = iter(["data: snapshot\n\n"])- snapshot_builder = Mock(return_value=["snapshot-events"])+ snapshot_calls: list[dict[str, object]] = []++ def snapshot_builder(**kwargs: object) -> list[dict[str, str]]:+ snapshot_calls.append(kwargs)+ return [{"event": "snapshot"}]monkeypatch.setattr(DifyAPIRepositoryFactory,@@ -391,10 +404,6 @@ def test_workflow_events_snapshot_can_continue_across_pauses(app: Flask, monkeyp"controllers.console.human_input_form._retrieve_app_for_workflow_run",lambda *_args, **_kwargs: app_model,)- monkeypatch.setattr(- "controllers.console.human_input_form.WorkflowAppGenerator",- lambda: workflow_generator,- )monkeypatch.setattr("controllers.console.human_input_form.build_workflow_event_stream",snapshot_builder,@@ -409,13 +418,15 @@ def test_workflow_events_snapshot_can_continue_across_pauses(app: Flask, monkeyp):response = handler(api, "t1", _account(), workflow_run_id="run-1")- assert response.get_data(as_text=True) == "data: snapshot\n\n"- snapshot_builder.assert_called_once_with(- app_mode=AppMode.WORKFLOW,- workflow_run=workflow_run,- tenant_id="t1",- app_id="app-1",- session_maker=ANY,- human_input_surface=HumanInputSurface.CONSOLE,- close_on_pause=False,- )+ assert response.get_data(as_text=True) == 'data: {"event":"snapshot"}\n\n'+ assert snapshot_calls == [+ {+ "app_mode": AppMode.WORKFLOW,+ "workflow_run": workflow_run,+ "tenant_id": "t1",+ "app_id": "app-1",+ "session_maker": ANY,+ "human_input_surface": HumanInputSurface.CONSOLE,+ "close_on_pause": False,+ }+ ]diff --git a/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py b/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.pyindex 314852da2e3..715c78ca7b1 100644--- a/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py+++ b/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py@@ -1,4 +1,5 @@from collections.abc import Callable, Generator+from dataclasses import replacefrom types import SimpleNamespacefrom unittest.mock import MagicMock, patchfrom uuid import uuid4@@ -19,6 +20,7 @@ from controllers.openapi.auth.subjects import AccountSubject, ResourceAccessSubjfrom core.db.session_factory import get_session_makerfrom extensions.application_services.app import AppServicesfrom extensions.application_services.resource_access_token import build_resource_access_token_service+from extensions.ext_application_services import ApplicationServicesfrom libs.oauth_bearer import Scope, TokenTypefrom models.account import Tenant, TenantStatusfrom models.enums import CreatorUserRole@@ -252,7 +254,9 @@ def test_other_workspace_endpoints_reject_resource_token(resource_fixture: Resoumethods[endpoint](workspace_id=tenant_id)-def test_stop_task_rejects_other_token_owner(resource_fixture: ResourceFixture) -> None:+def test_stop_task_rejects_other_token_owner(+ resource_fixture: ResourceFixture, account_application_services: ApplicationServices+) -> None:import inspectfrom werkzeug.exceptions import NotFound@@ -262,19 +266,20 @@ def test_stop_task_rejects_other_token_owner(resource_fixture: ResourceFixture)flask_app, app_id, _, _, end_user = resource_fixturedata = call(flask_app, app_id)end_user.return_value.id = "machine-user"- with (- patch("controllers.openapi.app_run.redis_client") as redis,- patch("controllers.openapi.app_run.AppQueueManager.set_stop_flag_no_user_check") as stop,- patch("controllers.openapi.app_run.GraphEngineManager") as engine,+ from services.app_task_service import AppTaskControlService+ from tests.unit_tests.services.test_app_task_service import _StopRedis++ redis = _StopRedis(values={"generate_task_belong:task": b"end-user-other-user"})+ with patch(+ "controllers.openapi.app_run.application_services",+ return_value=replace(account_application_services, app_tasks=AppTaskControlService(redis_client=redis)),):- redis.get.return_value = b"end-user-other-user"with pytest.raises(NotFound):inspect.unwrap(AppRunTaskStopApi.post)(AppRunTaskStopApi(), data, app_id, "task")- stop.assert_not_called()- engine.assert_not_called()- redis.get.return_value = b"end-user-machine-user"+ assert redis.operations == []+ redis.values["generate_task_belong:task"] = b"end-user-machine-user"inspect.unwrap(AppRunTaskStopApi.post)(AppRunTaskStopApi(), data, app_id, "task")- stop.assert_called_once_with("task")+ assert redis.values["generate_task_stopped:task"] == b"1"@pytest.mark.parametrize("search", ["list", "bound", "unbound"])diff --git a/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py b/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.pyindex b22e03c0a73..dbfc7c875ea 100644--- a/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py+++ b/api/tests/unit_tests/controllers/openapi/test_app_run_streaming.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport jsonimport sysimport uuid+from dataclasses import replacefrom io import BytesIOfrom types import SimpleNamespacefrom unittest.mock import Mock@@ -12,8 +13,12 @@ from unittest.mock import Mockimport pytestfrom flask import Flaskfrom pydantic import BaseModel, ValidationError+from sqlalchemy import event+from sqlalchemy.orm import Sessionfrom werkzeug.datastructures import FileStorage+from werkzeug.exceptions import NotFound+from constants.oauth_bearer import TokenTypefrom controllers.openapi import app_runfrom controllers.openapi._errors import AppModeMismatchfrom controllers.openapi._models import (@@ -24,11 +29,19 @@ from controllers.openapi._models import (WorkflowRunPayload,)from controllers.openapi.app_run import AdvancedChatRunApi, AppRunTaskStopApi, ChatRunApi, CompletionRunApi+from controllers.openapi.auth.subjects import Subject, subject_from_authfrom core.app.entities.app_invoke_entities import InvokeFrom+from extensions.ext_application_services import ApplicationServices+from graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom graphon.file import FileType+from libs.oauth_bearer import AuthContextfrom models import Account-from models.enums import CreatorUserRole-from models.model import App, AppMode+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.model import App, AppMode, EndUser, UploadFile+from models.workflow import WorkflowRun+from services.app_task_service import AppTaskControlService+from tests.unit_tests.model_factories import make_upload_file+from tests.unit_tests.services.test_app_task_service import _StopRedis_TEST_APP_ID = str(uuid.uuid4())_TEST_TENANT_ID = str(uuid.uuid4())@@ -52,6 +65,20 @@ def _make_account() -> Account:return account+def _subject(token_type: TokenType = TokenType.OAUTH_ACCOUNT) -> Subject:+ return subject_from_auth(+ AuthContext(+ token_type=token_type,+ subject_email="[email protected]" if token_type == TokenType.OAUTH_EXTERNAL_SSO else None,+ subject_issuer=None,+ account_id=uuid.UUID(_TEST_ACCOUNT_ID) if token_type == TokenType.OAUTH_ACCOUNT else None,+ client_id=None,+ token_id=uuid.uuid4(),+ expires_at=None,+ )+ )++@pytest.mark.parametrize(("model", "payload"),[@@ -73,29 +100,69 @@ def test_chat_payload_normalizes_conversation_id():assert ChatRunPayload(inputs={}, query="hi", conversation_id=cid).conversation_id == cid-def test_stop_task_calls_queue_manager_and_graph_engine(app: Flask, monkeypatch: pytest.MonkeyPatch):- queue_mock = Mock()- graph_mock = Mock()- graph_instance = Mock()- graph_mock.return_value = graph_instance-- run_module = sys.modules["controllers.openapi.app_run"]- monkeypatch.setattr(run_module, "AppQueueManager", queue_mock)- monkeypatch.setattr(run_module, "GraphEngineManager", graph_mock)- monkeypatch.setattr(run_module, "redis_client", object())-- api = AppRunTaskStopApi()- with app.test_request_context("/openapi/v1/apps/app-1/tasks/task-1:stop", method="POST"):- result = api.post.__handler__(- api,- _SealableContext(subject=SimpleNamespace(caller_role=CreatorUserRole.ACCOUNT)),- app_id="app-1",- task_id="task-1",+ "token_type", [TokenType.OAUTH_ACCOUNT, TokenType.OAUTH_EXTERNAL_SSO, TokenType.RESOURCE_ACCESS]+)[email protected]("mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT, AppMode.CHAT, AppMode.COMPLETION])[email protected]("owns_task", [False, True])+def test_stop_task_enforces_resolved_caller_ownership(+ app: Flask,+ account_application_services: ApplicationServices,+ monkeypatch: pytest.MonkeyPatch,+ sqlite_session: Session,+ token_type: TokenType,+ mode: AppMode,+ owns_task: bool,+):+ import services.app_task_service as task_module++ subject = _subject(token_type)+ caller = _make_account() if subject.caller_role == CreatorUserRole.ACCOUNT else EndUser(id="user-1")+ owner_id = caller.id if owns_task else "another-user"+ task_id = str(uuid.uuid4())+ redis = _StopRedis(values={f"generate_task_belong:{task_id}": f"end-user-{owner_id}".encode()})+ if mode in (AppMode.WORKFLOW, AppMode.ADVANCED_CHAT):+ sqlite_session.add(+ WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ task_id=task_id,+ type=WorkflowType.CHAT if mode == AppMode.ADVANCED_CHAT else WorkflowType.WORKFLOW,+ version="1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ status=WorkflowExecutionStatus.RUNNING,+ created_by_role=CreatorUserRole.END_USER,+ created_by=owner_id,+ ))-- queue_mock.set_stop_flag_no_user_check.assert_called_once_with("task-1")- graph_instance.send_stop_command.assert_called_once_with("task-1")- assert result == TaskStopResponse(result="success")+ sqlite_session.commit()+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_session.get_bind()))+ monkeypatch.setattr(+ sys.modules["controllers.openapi.app_run"],+ "application_services",+ lambda: replace(account_application_services, app_tasks=AppTaskControlService(redis_client=redis)),+ )+ ctx = _SealableContext(+ subject=subject,+ caller=caller,+ app=App(id="app-1", tenant_id="tenant-1", mode=mode),+ )+ api = AppRunTaskStopApi()+ allowed = owns_task or subject.caller_role == CreatorUserRole.ACCOUNT+ with app.test_request_context(f"/openapi/v1/apps/app-1/tasks/{task_id}:stop", method="POST"):+ if allowed:+ assert api.post.__handler__(api, ctx, app_id="app-1", task_id=task_id) == TaskStopResponse(result="success")+ else:+ with pytest.raises(NotFound, match="Task not found"):+ api.post.__handler__(api, ctx, app_id="app-1", task_id=task_id)+ if allowed:+ assert redis.values[f"generate_task_stopped:{task_id}"] == b"1"+ assert bool(redis.commands) == (mode in (AppMode.WORKFLOW, AppMode.ADVANCED_CHAT))+ else:+ assert redis.operations == []+ assert redis.commands == {}class _SealableContext:@@ -128,14 +195,14 @@ def _generate_stub(monkeypatch: pytest.MonkeyPatch, chunks: list[str]) -> Mock:return generate_mock-def _ctx(mode: AppMode, session: Mock | None = None) -> _SealableContext:+def _ctx(mode: AppMode, session: Session | Mock | None = None) -> _SealableContext:app_model = _make_app()app_model.mode = modereturn _SealableContext(app=app_model,caller=_make_account(),- session=session or Mock(),- subject=SimpleNamespace(caller_role=CreatorUserRole.ACCOUNT),+ session=session or Session(),+ subject=_subject(),)@@ -165,12 +232,25 @@ def test_per_mode_route_refuses_an_app_of_another_mode(app: Flask, monkeypatch:generate_mock.assert_not_called()-def test_run_hands_the_generator_file_mappings_for_inputs_and_attachments(app: Flask, monkeypatch: pytest.MonkeyPatch):+def test_run_hands_the_generator_file_mappings_for_inputs_and_attachments(+ app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session+):generate_mock = _generate_stub(monkeypatch, [])upload_service = Mock()- upload_service.upload_file.side_effect = lambda **kw: SimpleNamespace(- id=f"uf-{kw['filename']}", extension=kw["filename"].rsplit(".", 1)[-1], mime_type=kw["mimetype"]- )+ operations: list[str] = []+ sqlite_session.begin()+ event.listen(sqlite_session, "after_commit", lambda _session: operations.append("commit"))++ def upload_file(*, filename: str, content: bytes, mimetype: str, user: Account) -> UploadFile:+ assert not sqlite_session.in_transaction()+ operations.append("upload")+ return make_upload_file(+ file_id=f"uf-{filename}",+ extension=filename.rsplit(".", 1)[-1],+ mime_type=mimetype,+ )++ upload_service.upload_file.side_effect = upload_filemonkeypatch.setattr(sys.modules["controllers.openapi._files"], "application_services", lambda: SimpleNamespace(files=upload_service))@@ -179,17 +259,12 @@ def test_run_hands_the_generator_file_mappings_for_inputs_and_attachments(app: Ffiles={"doc": FileStorage(stream=BytesIO(b"pdf"), filename="r.pdf", content_type="application/pdf")},attachments=[FileStorage(stream=BytesIO(b"jpg"), filename="p.jpg", content_type="image/jpeg")],)- order = Mock()- session = Mock()- order.attach_mock(session, "session")- order.attach_mock(upload_service, "uploads")-api = CompletionRunApi()with app.test_request_context(f"/openapi/v1/apps/{_TEST_APP_ID}/completion:run", method="POST"):- api.post.__handler__(api, _ctx(AppMode.COMPLETION, session), app_id=_TEST_APP_ID, body=body)+ api.post.__handler__(api, _ctx(AppMode.COMPLETION, sqlite_session), app_id=_TEST_APP_ID, body=body)# The read transaction ends before the first object-storage write (api/AGENTS.md).- assert [call[0] for call in order.mock_calls][:2] == ["session.commit", "uploads.upload_file"]+ assert operations == ["commit", "upload", "upload"]args = generate_mock.call_args.kwargs["args"]assert args["inputs"]["doc"] == {"transfer_method": "local_file",diff --git a/api/tests/unit_tests/controllers/openapi/test_workflow_events_openapi.py b/api/tests/unit_tests/controllers/openapi/test_workflow_events_openapi.pyindex 9049b5ff4c9..64026d07fff 100644--- a/api/tests/unit_tests/controllers/openapi/test_workflow_events_openapi.py+++ b/api/tests/unit_tests/controllers/openapi/test_workflow_events_openapi.py@@ -119,9 +119,6 @@ class TestOpenApiWorkflowEventsApi:def _bind_generators(self, monkeypatch: pytest.MonkeyPatch) -> None:module = sys.modules["controllers.openapi.workflow_events"]- generator_mock = Mock()- generator_mock.convert_to_event_stream.return_value = iter([])- monkeypatch.setattr(module, "WorkflowAppGenerator", lambda: generator_mock)msg_gen_mock = Mock()msg_gen_mock.retrieve_events.return_value = iter([])monkeypatch.setattr(module, "MessageGenerator", lambda: msg_gen_mock)@@ -243,13 +240,10 @@ class TestOpenApiWorkflowEventsApi:module = sys.modules["controllers.openapi.workflow_events"]self._bind_repo(monkeypatch, _make_workflow_run(created_by_role=CreatorUserRole.ACCOUNT, created_by="acct-1"))- generator_mock = Mock()- generator_mock.convert_to_event_stream.return_value = iter(["event: a\n\n", "event: b\n\n"])- monkeypatch.setattr(module, "WorkflowAppGenerator", lambda: generator_mock)msg_gen_mock = Mock()- msg_gen_mock.retrieve_events.return_value = iter([])+ msg_gen_mock.retrieve_events.return_value = iter(["a", "b"])monkeypatch.setattr(module, "MessageGenerator", lambda: msg_gen_mock)- monkeypatch.setattr(module, "build_workflow_event_stream", Mock(return_value=iter([])))+ monkeypatch.setattr(module, "build_workflow_event_stream", lambda **_kwargs: iter(["a", "b"]))ctx = _context(_make_account(), CreatorUserRole.ACCOUNT)api = OpenApiWorkflowEventsApi()@@ -268,9 +262,6 @@ class TestOpenApiWorkflowEventsApi:):module = sys.modules["controllers.openapi.workflow_events"]self._bind_repo(monkeypatch, _make_workflow_run(created_by_role=CreatorUserRole.ACCOUNT, created_by="acct-1"))- generator_mock = Mock()- generator_mock.convert_to_event_stream.return_value = iter([])- monkeypatch.setattr(module, "WorkflowAppGenerator", lambda: generator_mock)msg_gen_mock = Mock()msg_gen_mock.retrieve_events.return_value = iter([])monkeypatch.setattr(module, "MessageGenerator", lambda: msg_gen_mock)diff --git a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.pyindex 29a1f8722a1..23f5360b33e 100644--- a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py+++ b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py@@ -40,7 +40,7 @@ from core.workflow.system_variables import build_system_variablesfrom enums import DeploymentEditionfrom graphon.entities import WorkflowStartReasonfrom graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom libs.datetime_utils import to_utc_timestampfrom models.account import Accountfrom models.enums import CreatorUserRole, EndUserType, MessageStatus@@ -319,7 +319,7 @@ def _build_resumption_context(task_id: str) -> WorkflowResumptionContext:call_depth=0,workflow_execution_id="run-1",)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)runtime_state.set_output("result", "value")wrapper = _WorkflowGenerateEntityWrapper(entity=generate_entity)return WorkflowResumptionContext(@@ -344,11 +344,8 @@ class TestHitlServiceApi:sqlite_engine=sqlite_engine,)msg_generator = Mock()- msg_generator.retrieve_events.return_value = ["raw-event"]- workflow_generator = Mock()- workflow_generator.convert_to_event_stream.return_value = iter(["data: streamed\n\n"])+ msg_generator.retrieve_events.return_value = [{"event": "streamed"}]monkeypatch.setattr(workflow_events_module, "MessageGenerator", lambda: msg_generator)- monkeypatch.setattr(workflow_events_module, "WorkflowAppGenerator", lambda: workflow_generator)api = WorkflowEventsApi()handler = unwrap(api.get)@@ -358,13 +355,12 @@ class TestHitlServiceApi:with app.test_request_context("/workflow/run-1/events?user=u1&continue_on_pause=true", method="GET"):response = handler(api, app_model=app_model, end_user=end_user, workflow_run_id="run-1")- assert response.get_data(as_text=True) == "data: streamed\n\n"+ assert response.get_data(as_text=True) == 'data: {"event":"streamed"}\n\n'msg_generator.retrieve_events.assert_called_once_with(AppMode.WORKFLOW,"run-1",terminal_events=[],)- workflow_generator.convert_to_event_stream.assert_called_once_with(["raw-event"])def test_workflow_events_snapshot_continue_on_pause_keeps_pause_open(self,@@ -380,11 +376,13 @@ class TestHitlServiceApi:sqlite_engine=sqlite_engine,)msg_generator = Mock()- workflow_generator = Mock()- workflow_generator.convert_to_event_stream.return_value = iter(["data: snapshot\n\n"])- snapshot_builder = Mock(return_value=["snapshot-events"])+ snapshot_calls: list[dict[str, object]] = []++ def snapshot_builder(**kwargs: object) -> list[dict[str, str]]:+ snapshot_calls.append(kwargs)+ return [{"event": "snapshot"}]+monkeypatch.setattr(workflow_events_module, "MessageGenerator", lambda: msg_generator)- monkeypatch.setattr(workflow_events_module, "WorkflowAppGenerator", lambda: workflow_generator)monkeypatch.setattr(workflow_events_module, "build_workflow_event_stream", snapshot_builder)api = WorkflowEventsApi()@@ -398,21 +396,22 @@ class TestHitlServiceApi:):response = handler(api, app_model=app_model, end_user=end_user, workflow_run_id="run-1")- assert response.get_data(as_text=True) == "data: snapshot\n\n"+ assert response.get_data(as_text=True) == 'data: {"event":"snapshot"}\n\n'msg_generator.retrieve_events.assert_not_called()- snapshot_builder.assert_called_once_with(- app_mode=AppMode.WORKFLOW,- workflow_run=workflow_run,- tenant_id="tenant-1",- app_id="app-1",- session_maker=ANY,- human_input_surface=HumanInputSurface.SERVICE_API,- close_on_pause=False,- )- snapshot_session_maker = snapshot_builder.call_args.kwargs["session_maker"]+ assert snapshot_calls == [+ {+ "app_mode": AppMode.WORKFLOW,+ "workflow_run": workflow_run,+ "tenant_id": "tenant-1",+ "app_id": "app-1",+ "session_maker": ANY,+ "human_input_surface": HumanInputSurface.SERVICE_API,+ "close_on_pause": False,+ }+ ]+ snapshot_session_maker = snapshot_calls[0]["session_maker"]assert isinstance(snapshot_session_maker, sessionmaker)assert snapshot_session_maker.kw["bind"] is sqlite_engine- workflow_generator.convert_to_event_stream.assert_called_once_with(["snapshot-events"])def test_advanced_chat_blocking_injects_pause_state_config(self,@@ -430,7 +429,7 @@ class TestHitlServiceApi:generator_instance = MagicMock()generator_instance.generate.return_value = {"result": "advanced-blocking"}generator_instance.convert_to_event_stream.side_effect = lambda payload: payload- monkeypatch.setattr(ags_module, "AdvancedChatAppGenerator", lambda: generator_instance)+ monkeypatch.setattr(ags_module, "AdvancedChatAppGenerator", lambda **_kwargs: generator_instance)app_model = _app(app_id="app-id", tenant_id="tenant-id", mode=AppMode.ADVANCED_CHAT)user = _end_user(user_id="user-id", app_id="app-id", tenant_id="tenant-id")diff --git a/api/tests/unit_tests/controllers/service_api/app/test_workflow.py b/api/tests/unit_tests/controllers/service_api/app/test_workflow.pyindex c0bb353fa4f..4a0b92f2867 100644--- a/api/tests/unit_tests/controllers/service_api/app/test_workflow.py+++ b/api/tests/unit_tests/controllers/service_api/app/test_workflow.py@@ -35,8 +35,6 @@ from controllers.service_api.app.error import (WorkflowVersionExecutionNotAllowedError,)from controllers.service_api.app.workflow import (- AppQueueManager,- GraphEngineManager,WorkflowAppLogApi,WorkflowLogQuery,WorkflowRunApi,@@ -725,9 +723,8 @@ class TestWorkflowTaskStopApi:def test_success(self, app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:stop_mock = Mock()- send_mock = Mock()- monkeypatch.setattr(AppQueueManager, "set_stop_flag_no_user_check", stop_mock)- monkeypatch.setattr(GraphEngineManager, "send_stop_command", send_mock)+ workflow_module = sys.modules["controllers.service_api.app.workflow"]+ monkeypatch.setattr(workflow_module.AppTaskService, "stop_workflow_task", stop_mock)api = WorkflowTaskStopApi()handler = unwrap(api.post)@@ -738,8 +735,13 @@ class TestWorkflowTaskStopApi:response = handler(api, app_model=app_model, end_user=end_user, task_id="t1")assert response == {"result": "success"}- stop_mock.assert_called_once_with("t1")- send_mock.assert_called_once_with("t1")+ stop_mock.assert_called_once_with(+ tenant_id=app_model.tenant_id,+ app_id=app_model.id,+ task_id="t1",+ app_mode=AppMode.WORKFLOW,+ owner=None,+ )class TestWorkflowAppLogApi:diff --git a/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.py b/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.pyindex 5419a34cb8b..d9ad0b758ff 100644--- a/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.py+++ b/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.py@@ -152,11 +152,8 @@ class TestWorkflowEventsApi:workflow_run = _workflow_run()workflow_events_module = _mock_repo_for_run(monkeypatch, workflow_run=workflow_run, sqlite_engine=sqlite_engine)msg_generator = Mock()- msg_generator.retrieve_events.return_value = ["raw-event"]- workflow_generator = Mock()- workflow_generator.convert_to_event_stream.return_value = iter(["data: streamed\n\n"])+ msg_generator.retrieve_events.return_value = [{"event": "streamed"}]monkeypatch.setattr(workflow_events_module, "MessageGenerator", lambda: msg_generator)- monkeypatch.setattr(workflow_events_module, "WorkflowAppGenerator", lambda: workflow_generator)api = WorkflowEventsApi()handler = unwrap(api.get)@@ -166,13 +163,12 @@ class TestWorkflowEventsApi:with app.test_request_context("/workflow/run-1/events?user=u1", method="GET"):response = handler(api, app_model=app_model, end_user=end_user, workflow_run_id="run-1")- assert response.get_data(as_text=True) == "data: streamed\n\n"+ assert response.get_data(as_text=True) == 'data: {"event":"streamed"}\n\n'msg_generator.retrieve_events.assert_called_once_with(AppMode.WORKFLOW,"run-1",terminal_events=None,)- workflow_generator.convert_to_event_stream.assert_called_once_with(["raw-event"])def test_running_run_with_snapshot(self, app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine@@ -180,11 +176,13 @@ class TestWorkflowEventsApi:workflow_run = _workflow_run()workflow_events_module = _mock_repo_for_run(monkeypatch, workflow_run=workflow_run, sqlite_engine=sqlite_engine)msg_generator = Mock()- workflow_generator = Mock()- workflow_generator.convert_to_event_stream.return_value = iter(["data: snapshot\n\n"])- snapshot_builder = Mock(return_value=["snapshot-events"])+ snapshot_calls: list[dict[str, object]] = []++ def snapshot_builder(**kwargs: object) -> list[dict[str, str]]:+ snapshot_calls.append(kwargs)+ return [{"event": "snapshot"}]+monkeypatch.setattr(workflow_events_module, "MessageGenerator", lambda: msg_generator)- monkeypatch.setattr(workflow_events_module, "WorkflowAppGenerator", lambda: workflow_generator)monkeypatch.setattr(workflow_events_module, "build_workflow_event_stream", snapshot_builder)api = WorkflowEventsApi()@@ -195,7 +193,6 @@ class TestWorkflowEventsApi:with app.test_request_context("/workflow/run-1/events?user=u1&include_state_snapshot=true", method="GET"):response = handler(api, app_model=app_model, end_user=end_user, workflow_run_id="run-1")- assert response.get_data(as_text=True) == "data: snapshot\n\n"+ assert response.get_data(as_text=True) == 'data: {"event":"snapshot"}\n\n'msg_generator.retrieve_events.assert_not_called()- snapshot_builder.assert_called_once()- workflow_generator.convert_to_event_stream.assert_called_once_with(["snapshot-events"])+ assert len(snapshot_calls) == 1diff --git a/api/tests/unit_tests/controllers/web/test_workflow.py b/api/tests/unit_tests/controllers/web/test_workflow.pyindex 49c6ad74c6b..35459762f5c 100644--- a/api/tests/unit_tests/controllers/web/test_workflow.py+++ b/api/tests/unit_tests/controllers/web/test_workflow.py@@ -109,12 +109,12 @@ class TestWorkflowTaskStopApi:with pytest.raises(NotWorkflowAppError):WorkflowTaskStopApi().post(_chat_app(), _end_user(), "task-1")- @patch("controllers.web.workflow.GraphEngineManager.send_stop_command")- @patch("controllers.web.workflow.AppQueueManager.set_stop_flag_no_user_check")- def test_stop_calls_both_mechanisms(self, mock_legacy: MagicMock, mock_graph: MagicMock, app: Flask) -> None:+ @patch("controllers.web.workflow.AppTaskService.stop_workflow_task")+ def test_stop_passes_app_scope(self, stop: MagicMock, app: Flask) -> None:with app.test_request_context("/workflows/tasks/task-1/stop", method="POST"):result = WorkflowTaskStopApi().post(_workflow_app(), _end_user(), "task-1")assert result == {"result": "success"}- mock_legacy.assert_called_once_with("task-1")- mock_graph.assert_called_once_with("task-1")+ stop.assert_called_once_with(+ tenant_id="tenant-1", app_id="app-1", task_id="task-1", app_mode=AppMode.WORKFLOW, owner=None+ )diff --git a/api/tests/unit_tests/controllers/web/test_workflow_events.py b/api/tests/unit_tests/controllers/web/test_workflow_events.pyindex e6a4abf86df..343f11552a2 100644--- a/api/tests/unit_tests/controllers/web/test_workflow_events.py+++ b/api/tests/unit_tests/controllers/web/test_workflow_events.py@@ -3,7 +3,7 @@from __future__ import annotationsfrom datetime import datetime-from unittest.mock import ANY, MagicMock, Mock, patch+from unittest.mock import ANY, MagicMock, patchimport pytestfrom flask import Flask@@ -145,21 +145,25 @@ class TestWorkflowEventsApi:mock_repo.get_workflow_run_by_id_and_tenant_id.return_value = runmock_factory.create_api_workflow_run_repository.return_value = mock_repo- workflow_generator = Mock()- workflow_generator.convert_to_event_stream.return_value = iter(["data: snapshot\n\n"])- snapshot_builder = Mock(return_value=["snapshot-events"])- monkeypatch.setattr("controllers.web.workflow_events.WorkflowAppGenerator", lambda: workflow_generator)+ snapshot_calls: list[dict[str, object]] = []++ def snapshot_builder(**kwargs: object) -> list[dict[str, str]]:+ snapshot_calls.append(kwargs)+ return [{"event": "snapshot"}]+monkeypatch.setattr("controllers.web.workflow_events.build_workflow_event_stream", snapshot_builder)with app.test_request_context("/workflow/run-1/events?include_state_snapshot=true&continue_on_pause=true"):response = WorkflowEventsApi().get(_workflow_app(), _end_user(), "run-1")- assert response.get_data(as_text=True) == "data: snapshot\n\n"- snapshot_builder.assert_called_once_with(- app_mode=AppMode.WORKFLOW,- workflow_run=run,- tenant_id="tenant-1",- app_id="app-1",- session_maker=ANY,- close_on_pause=False,- )+ assert response.get_data(as_text=True) == 'data: {"event":"snapshot"}\n\n'+ assert snapshot_calls == [+ {+ "app_mode": AppMode.WORKFLOW,+ "workflow_run": run,+ "tenant_id": "tenant-1",+ "app_id": "app-1",+ "session_maker": ANY,+ "close_on_pause": False,+ }+ ]diff --git a/api/tests/unit_tests/core/agent/test_base_agent_runner.py b/api/tests/unit_tests/core/agent/test_base_agent_runner.pyindex c1bbdf8e2d9..476d7432148 100644--- a/api/tests/unit_tests/core/agent/test_base_agent_runner.py+++ b/api/tests/unit_tests/core/agent/test_base_agent_runner.py@@ -42,6 +42,7 @@ from graphon.file import FileTransferMethod, FileTypefrom graphon.model_runtime.entities import LLMUsage, PromptMessageToolfrom models.enums import ConversationFromSource, CreatorUserRole, MessageFileBelongsTofrom models.model import AppMode, AppModelConfig, Conversation, Message, MessageAgentThought, MessageFile, UploadFile+from services.workflow_run_agg import WorkflowRunAggfrom tests.tool_fixtures import make_runtime_toolfrom tests.unit_tests.core.model_fixtures import make_model_config@@ -239,6 +240,7 @@ def runner(mocker.patch.object(model_instance.model_type_instance, "get_model_schema", return_value=None)return BaseAgentRunner(+ execution_driver=WorkflowRunAgg.run,session=sqlite_session,tenant_id="tenant",application_generate_entity=_app_generate(app_config=app_config),@@ -745,6 +747,7 @@ def test_init_uses_real_session_for_count_and_dependencies(message = _message(message_id="msg1")initialized = BaseAgentRunner(+ execution_driver=WorkflowRunAgg.run,session=sqlite_session,tenant_id="tenant",application_generate_entity=app_generate,diff --git a/api/tests/unit_tests/core/agent/test_fc_agent_runner.py b/api/tests/unit_tests/core/agent/test_fc_agent_runner.pyindex 6db592f26f9..89eb6c4674d 100644--- a/api/tests/unit_tests/core/agent/test_fc_agent_runner.py+++ b/api/tests/unit_tests/core/agent/test_fc_agent_runner.py@@ -36,6 +36,7 @@ from graphon.model_runtime.entities.message_entities import (from libs.datetime_utils import naive_utc_nowfrom models.enums import ConversationFromSource, CreatorUserRole, MessageStatusfrom models.model import AppMode, Conversation, Message, StorageType, UploadFile+from services.workflow_run_agg import WorkflowRunAgg# ==============================# Dummy Helper Classes@@ -131,6 +132,7 @@ def real_runner(assert app_config.agent is not Noneapp_config.agent.strategy = AgentEntity.Strategy.FUNCTION_CALLINGreturn FunctionCallAgentRunner(+ execution_driver=WorkflowRunAgg.run,session=sqlite_session,tenant_id=app_config.tenant_id,application_generate_entity=agent_generate_entity,@@ -189,6 +191,7 @@ def runner(mocker: MockerFixture, sqlite_engine: Engine) -> Iterator[FunctionCalconversation = _make_conversation()runner = FunctionCallAgentRunner(+ execution_driver=WorkflowRunAgg.run,tenant_id="tenant",application_generate_entity=application_generate_entity,conversation=conversation,diff --git a/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_generator.py b/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_generator.pyindex 542deab229a..fec64f59ff4 100644--- a/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_generator.py+++ b/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_generator.py@@ -28,6 +28,7 @@ from models.account import Accountfrom models.enums import ConversationFromSource, EndUserType, MessageStatusfrom models.model import App, AppMode, Conversation, EndUser, Messagefrom models.workflow import Workflow, WorkflowType+from services.workflow_run_agg import WorkflowRunAggfrom tests.unit_tests.config_override import apply_config_overrides@@ -110,7 +111,7 @@ def _make_message(class TestAdvancedChatAppGeneratorValidation:def test_generate_requires_query(self, unbound_session: Session):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)with pytest.raises(ValueError, match="query is required"):generator.generate(@@ -125,7 +126,7 @@ class TestAdvancedChatAppGeneratorValidation:)def test_generate_requires_string_query(self, unbound_session: Session):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)with pytest.raises(ValueError, match="query must be a string"):generator.generate(@@ -140,7 +141,7 @@ class TestAdvancedChatAppGeneratorValidation:)def test_single_iteration_generate_validates_args(self, unbound_session: Session):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)with pytest.raises(ValueError, match="node_id is required"):generator.single_iteration_generate(@@ -165,7 +166,7 @@ class TestAdvancedChatAppGeneratorValidation:)def test_single_loop_generate_validates_args(self, unbound_session: Session):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)with pytest.raises(ValueError, match="node_id is required"):generator.single_loop_generate(@@ -205,7 +206,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_loads_conversation_and_files(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = self._build_app_config()conversation = _make_conversation()@@ -291,7 +292,7 @@ class TestAdvancedChatAppGeneratorInternals:assert get_conversation.call_args.kwargs["session"] is sessiondef test_resume_delegates_to_generate(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)existing_trace_manager = SimpleNamespace(app_id="existing-app", user_id="existing-user")application_generate_entity = AdvancedChatAppGenerateEntity.model_construct(task_id="task",@@ -340,7 +341,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_single_iteration_generate_builds_debug_task(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = self._build_app_config()captured: dict[str, object] = {}prefill_calls: list[object] = []@@ -403,7 +404,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_single_loop_generate_builds_debug_task(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = self._build_app_config()captured: dict[str, object] = {}prefill_calls: list[object] = []@@ -466,7 +467,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_internal_flow_initial_conversation_with_pause_layer(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 0app_config = self._build_app_config()@@ -510,10 +511,6 @@ class TestAdvancedChatAppGeneratorInternals:"core.app.apps.advanced_chat.app_generator.MessageBasedAppQueueManager",lambda **kwargs: SimpleNamespace(**kwargs),)- monkeypatch.setattr(- "core.app.apps.advanced_chat.app_generator.PauseStatePersistenceLayer",- lambda **kwargs: "pause-layer",- )monkeypatch.setattr("core.app.apps.advanced_chat.app_generator.current_app",SimpleNamespace(_get_current_object=lambda: SimpleNamespace(name="flask")),@@ -571,7 +568,7 @@ class TestAdvancedChatAppGeneratorInternals:assert thread_data["started"] is Trueassert thread_data["joined"] is Trueassert thread_data["join_timeout"] == 300- assert "pause-layer" in thread_data["kwargs"]["graph_engine_layers"]+ assert thread_data["kwargs"]["pause_state_config"] is pause_state_configassert generator._dialogue_count == 3assert init_records.call_args.kwargs["session"] is sqlite_sessionget_thread_messages_length.assert_called_once_with(conversation.id, session=sqlite_session)@@ -585,7 +582,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_internal_flow_with_existing_records_skips_init(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 0app_config = self._build_app_config()@@ -689,7 +686,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_worker_raises_when_workflow_not_found(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1app_config = self._build_app_config()@@ -732,14 +729,13 @@ class TestAdvancedChatAppGeneratorInternals:variable_loader=SimpleNamespace(),workflow_execution_repository=SimpleNamespace(),workflow_node_execution_repository=SimpleNamespace(),- graph_engine_layers=(),graph_runtime_state=None,)def test_generate_worker_raises_when_app_not_found_for_internal_call(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1app_config = self._build_app_config()@@ -784,14 +780,13 @@ class TestAdvancedChatAppGeneratorInternals:variable_loader=SimpleNamespace(),workflow_execution_repository=SimpleNamespace(),workflow_node_execution_repository=SimpleNamespace(),- graph_engine_layers=(),graph_runtime_state=None,)def test_generate_worker_handles_stopped_error(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1app_config = self._build_app_config()@@ -826,7 +821,7 @@ class TestAdvancedChatAppGeneratorInternals:def __init__(self, **kwargs):_ = kwargs- def run(self):+ def prepare(self):raise GenerateTaskStoppedError()monkeypatch.setattr("core.app.apps.advanced_chat.app_generator.AdvancedChatAppRunner", _Runner)@@ -847,7 +842,6 @@ class TestAdvancedChatAppGeneratorInternals:variable_loader=SimpleNamespace(),workflow_execution_repository=SimpleNamespace(),workflow_node_execution_repository=SimpleNamespace(),- graph_engine_layers=(),graph_runtime_state=SimpleNamespace(),)@@ -858,7 +852,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_worker_handles_validation_error(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1app_config = self._build_app_config()@@ -902,7 +896,7 @@ class TestAdvancedChatAppGeneratorInternals:def __init__(self, **kwargs):_ = kwargs- def run(self):+ def prepare(self):raise validation_errormonkeypatch.setattr("core.app.apps.advanced_chat.app_generator.AdvancedChatAppRunner", _Runner)@@ -921,7 +915,6 @@ class TestAdvancedChatAppGeneratorInternals:variable_loader=SimpleNamespace(),workflow_execution_repository=SimpleNamespace(),workflow_node_execution_repository=SimpleNamespace(),- graph_engine_layers=(),graph_runtime_state=None,)@@ -943,13 +936,13 @@ class TestAdvancedChatAppGeneratorInternals:def __init__(self, **kwargs):_ = kwargs- def run(self):+ def prepare(self):raise errorreturn _Runnerfor raised_error in [ValueError("bad input"), RuntimeError("unexpected")]:- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1application_generate_entity = AdvancedChatAppGenerateEntity.model_construct(task_id="task",@@ -990,14 +983,13 @@ class TestAdvancedChatAppGeneratorInternals:variable_loader=SimpleNamespace(),workflow_execution_repository=SimpleNamespace(),workflow_node_execution_repository=SimpleNamespace(),- graph_engine_layers=(),graph_runtime_state=None,)queue_manager.publish_error.assert_called_once()def test_handle_response_closed_file_raises_stopped(self, monkeypatch: pytest.MonkeyPatch):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1app_config = WorkflowUIBasedAppConfig(@@ -1055,7 +1047,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_handle_response_re_raises_value_error(self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1app_config = self._build_app_config()application_generate_entity = AdvancedChatAppGenerateEntity.model_construct(@@ -1105,7 +1097,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_worker_handles_invoke_auth_error(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)generator._dialogue_count = 1app_config = WorkflowUIBasedAppConfig(@@ -1141,7 +1133,7 @@ class TestAdvancedChatAppGeneratorInternals:def __init__(self, **kwargs) -> None:_ = kwargs- def run(self):+ def prepare(self):from graphon.model_runtime.errors.invoke import InvokeAuthorizationErrorraise InvokeAuthorizationError("bad key")@@ -1169,7 +1161,6 @@ class TestAdvancedChatAppGeneratorInternals:variable_loader=SimpleNamespace(),workflow_execution_repository=SimpleNamespace(),workflow_node_execution_repository=SimpleNamespace(),- graph_engine_layers=(),graph_runtime_state=None,)@@ -1178,7 +1169,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_debugger_enables_retrieve_source(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = WorkflowUIBasedAppConfig(tenant_id="tenant",@@ -1253,7 +1244,7 @@ class TestAdvancedChatAppGeneratorInternals:def test_generate_service_api_sets_parent_message_id(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session, sqlite_engine: Engine):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = WorkflowUIBasedAppConfig(tenant_id="tenant",@@ -1339,7 +1330,7 @@ class TestAdvancedChatAppGeneratorResume:def test_resume_restores_trace_manager_when_missing(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)application_generate_entity = AdvancedChatAppGenerateEntity.model_construct(task_id="task",app_config=self._build_app_config(),@@ -1399,7 +1390,7 @@ class TestAdvancedChatAppGeneratorResume:assert trace_manager.user_id == "session-id"def test_resume_preserves_existing_trace_manager(self, monkeypatch: pytest.MonkeyPatch, unbound_session: Session):- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)existing_trace_manager = SimpleNamespace(app_id="existing-app", user_id="existing-user")application_generate_entity = AdvancedChatAppGenerateEntity.model_construct(task_id="task",diff --git a/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_conversation_variables.py b/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_conversation_variables.pyindex 5ed29a74063..4fc147d7355 100644--- a/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_conversation_variables.py+++ b/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_conversation_variables.py@@ -54,8 +54,6 @@ def _runner(workflow_variables: list[object]) -> AdvancedChatAppRunner:workflow=workflow,system_user_id="44444444-4444-4444-4444-444444444444",app=App(id=APP_ID, tenant_id=workflow.tenant_id),- workflow_execution_repository=MagicMock(),- workflow_node_execution_repository=MagicMock(),)diff --git a/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_input_moderation.py b/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_input_moderation.pyindex 5e420ed6c5f..76309013bf9 100644--- a/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_input_moderation.py+++ b/api/tests/unit_tests/core/app/apps/advanced_chat/test_app_runner_input_moderation.py@@ -94,8 +94,6 @@ def build_runner(sqlite_session: Session):workflow=workflow,system_user_id=str(uuid4()),app=app,- workflow_execution_repository=MagicMock(),- workflow_node_execution_repository=MagicMock(),)return runner@@ -108,7 +106,7 @@ def _patch_common_run_deps(runner: AdvancedChatAppRunner):RedisChannel=MagicMock(),redis_client=MagicMock(),WorkflowEntry=MagicMock(**{"return_value.run.return_value": iter([])}),- GraphRuntimeState=MagicMock(),+ RuntimeState=MagicMock(),)@@ -140,6 +138,7 @@ def test_handle_input_moderation_stops_on_moderation_error(build_runner):def test_run_applies_overridden_inputs_and_query_from_moderation(build_runner):runner = build_runner+ runner.application_generate_entity.call_depth = 5overridden_inputs = {"q": "sanitized"}overridden_query = "sanitized-query"@@ -154,7 +153,7 @@ def test_run_applies_overridden_inputs_and_query_from_moderation(build_runner):patch.object(runner, "handle_annotation_reply", return_value=False) as mock_anno,patch.object(runner, "_init_graph", return_value=MagicMock()) as mock_init_graph,):- runner.run()+ runner.prepare()# moderation called with original valuesmock_moderate.assert_called_once()@@ -169,6 +168,7 @@ def test_run_applies_overridden_inputs_and_query_from_moderation(build_runner):# since not stopped, graph initialization should proceedassert mock_init_graph.called+ assert mock_init_graph.call_args.kwargs["call_depth"] == 5def test_run_returns_early_when_direct_output_via_handle_input_moderation(build_runner):@@ -185,7 +185,7 @@ def test_run_returns_early_when_direct_output_via_handle_input_moderation(build_patch.object(runner, "_init_graph") as mock_init_graph,patch.object(runner, "handle_annotation_reply") as mock_anno,):- runner.run()+ runner.prepare()mock_handle.assert_called_once()# Ensure no further steps executed@@ -224,7 +224,7 @@ def test_run_publishes_annotation_after_commit(build_runner, sqlite_engine: Engipatch.object(runner, "_publish_event", side_effect=publish),patch.object(runner, "_complete_with_stream_output"),):- runner.run()+ runner.prepare()event.remove(Session, "after_commit", record_commit)assert events == ["commit", "publish"]@@ -242,7 +242,7 @@ def test_run_closes_scoped_session_before_workflow_run(build_runner, sqlite_sessworkflow_entry = MagicMock()- def run_workflow():+ def run_workflow(**kwargs):events.append("run")return iter([])@@ -262,6 +262,9 @@ def test_run_closes_scoped_session_before_workflow_run(build_runner, sqlite_sesspatch.object(runner, "_initialize_conversation_variables", return_value=[]),patch.object(runner, "_init_graph", return_value=MagicMock()),):- runner.run()+ prepared = runner.prepare()+ assert prepared is not None+ assert events[-1] == "close"+ list(prepared.entry.run())assert events[-2:] == ["close", "run"]diff --git a/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline.py b/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline.pyindex 5166d33fbb7..4314aa8b55a 100644--- a/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline.py+++ b/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline.py@@ -1,16 +1,17 @@from __future__ import annotations-from contextlib import nullcontextfrom datetime import datetimefrom types import SimpleNamespacefrom unittest import mockimport pytest+from pytest_mock import MockerFixturefrom sqlalchemy import select-from sqlalchemy.orm import Session+from sqlalchemy.orm import Session, sessionmaker+from core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.apps.advanced_chat import generate_task_pipeline as pipeline_module-from core.app.entities.app_invoke_entities import InvokeFrom+from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFromfrom core.app.entities.queue_entities import (QueuePingEvent,QueueTextChunkEvent,@@ -20,10 +21,28 @@ from core.app.entities.queue_entities import ()from core.app.entities.task_entities import StreamEventfrom core.workflow.nodes.human_input.pause_reason import HumanInputRequired-from graphon.enums import WorkflowExecutionStatus-from models.enums import MessageStatus+from graphon.enums import WorkflowExecutionStatus, WorkflowType+from models.enums import CreatorUserRole, MessageStatus, WorkflowRunTriggeredFromfrom models.execution_extra_content import HumanInputContentfrom models.model import AppMode, EndUser, Message+from models.workflow import WorkflowRun+from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository+++def _build_generate_entity() -> AdvancedChatAppGenerateEntity:+ return AdvancedChatAppGenerateEntity(+ task_id="task-1",+ app_config=WorkflowUIBasedAppConfig(+ app_id="app-1", tenant_id="tenant-1", app_mode=AppMode.ADVANCED_CHAT, workflow_id="workflow-1"+ ),+ inputs={},+ files=[],+ user_id="user-1",+ stream=True,+ invoke_from=InvokeFrom.WEB_APP,+ query="hello",+ workflow_run_id="run-1",+ )def _build_pipeline() -> pipeline_module.AdvancedChatAppGenerateTaskPipeline:@@ -63,13 +82,10 @@ def _message(*, status: MessageStatus, answer: str = "") -> Message:)-def test_persist_human_input_extra_content_adds_record(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:+def test_persist_human_input_extra_content_adds_record(sqlite_session: Session) -> None:pipeline = _build_pipeline()- monkeypatch.setattr(pipeline, "_load_human_input_form_id", lambda **kwargs: "form-1")- pipeline._persist_human_input_extra_content(node_id="node-1")+ pipeline._persist_human_input_extra_content(form_id="form-1")content = sqlite_session.scalar(select(HumanInputContent))assert content is not None@@ -78,37 +94,44 @@ def test_persist_human_input_extra_content_adds_record(assert content.form_id == "form-1"-def test_persist_human_input_extra_content_skips_when_form_missing(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:+def test_persist_human_input_extra_content_skips_when_existing(sqlite_session: Session) -> None:pipeline = _build_pipeline()- monkeypatch.setattr(pipeline, "_load_human_input_form_id", lambda **kwargs: None)-- pipeline._persist_human_input_extra_content(node_id="node-1")-- assert sqlite_session.scalar(select(HumanInputContent)) is None---def test_persist_human_input_extra_content_skips_when_existing(- monkeypatch: pytest.MonkeyPatch, sqlite_session: Session-) -> None:- pipeline = _build_pipeline()- monkeypatch.setattr(pipeline, "_load_human_input_form_id", lambda **kwargs: "form-1")existing = HumanInputContent.new(workflow_run_id="run-1", message_id="message-1", form_id="form-1")sqlite_session.add(existing)sqlite_session.commit()- pipeline._persist_human_input_extra_content(node_id="node-1")+ pipeline._persist_human_input_extra_content(form_id="form-1")contents = list(sqlite_session.scalars(select(HumanInputContent)))assert [content.id for content in contents] == [existing.id]-def test_handle_workflow_paused_event_persists_human_input_extra_content(unbound_session: Session) -> None:[email protected]("stop_after_pause", [False, True])+def test_handle_workflow_paused_event_does_not_overwrite_a_concurrent_stop(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ stop_after_pause: bool,+ mocker: MockerFixture,+) -> None:+ run = WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ task_id="task-1",+ type=WorkflowType.CHAT,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.PAUSED,+ created_by_role=CreatorUserRole.END_USER,+ created_by="user-1",+ )+ sqlite_session.add(run)+ sqlite_session.commit()pipeline = _build_pipeline()- pipeline._application_generate_entity = SimpleNamespace(task_id="task-1")+ pipeline._application_generate_entity = _build_generate_entity()pipeline._workflow_response_converter = mock.Mock()- pipeline._workflow_response_converter.workflow_pause_to_stream_response.return_value = []+ pipeline._workflow_response_converter.workflow_pause_to_stream_response.return_value = ["pause"]pipeline._ensure_graph_runtime_initialized = mock.Mock(return_value=SimpleNamespace(total_tokens=0,@@ -118,13 +141,11 @@ def test_handle_workflow_paused_event_persists_human_input_extra_content(unboundpipeline._save_message = mock.Mock()message = _message(status=MessageStatus.NORMAL)pipeline._get_message = mock.Mock(return_value=message)- pipeline._persist_human_input_extra_content = mock.Mock()+ persist_content = mocker.patch.object(pipeline, "_persist_human_input_extra_content")pipeline._base_task_pipeline = mock.Mock()pipeline._base_task_pipeline.queue_manager = mock.Mock()pipeline._message_saved_on_pause = False- pipeline._database_session = lambda: nullcontext(unbound_session) # type: ignore[method-assign]-reason = HumanInputRequired(form_id="form-1",form_content="content",@@ -136,10 +157,15 @@ def test_handle_workflow_paused_event_persists_human_input_extra_content(unbound)event = QueueWorkflowPausedEvent(reasons=[reason], outputs={}, paused_nodes=["node-1"])- list(pipeline._handle_workflow_paused_event(event))+ events = pipeline._handle_workflow_paused_event(event)+ assert next(events) == "pause"+ if stop_after_pause:+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ repository.stop_paused_workflow_task(tenant_id="tenant-1", app_id="app-1", task_id="task-1", owner=None)+ assert list(events) == []- pipeline._persist_human_input_extra_content.assert_called_once_with(form_id="form-1", node_id="node-1")- assert message.status == MessageStatus.PAUSED+ persist_content.assert_called_once_with(form_id="form-1")+ assert message.status == (MessageStatus.NORMAL if stop_after_pause else MessageStatus.PAUSED)def test_resume_appends_chunks_to_paused_answer(unbound_session: Session) -> None:diff --git a/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_core.py b/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_core.pyindex 69f80a393a8..6ce0d5aeb26 100644--- a/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_core.py+++ b/api/tests/unit_tests/core/app/apps/advanced_chat/test_generate_task_pipeline_core.py@@ -57,7 +57,7 @@ from core.workflow.system_variables import build_system_variablesfrom graphon.enums import BuiltinNodeTypesfrom graphon.file import FileTransferMethod, FileTypefrom graphon.model_runtime.entities.llm_entities import LLMUsage-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom libs.datetime_utils import naive_utc_nowfrom models.enums import MessageStatusfrom models.model import AppMode, EndUser, Message, MessageFile@@ -170,7 +170,8 @@ class TestAdvancedChatGenerateTaskPipeline:pipeline = _make_pipeline()pipeline._task_state.answer = "partial answer"pipeline._workflow_run_id = "run-id"- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=build_test_variable_pool(variables=build_system_variables(workflow_execution_id="run-id"),),@@ -300,7 +301,8 @@ class TestAdvancedChatGenerateTaskPipeline:pipeline = _make_pipeline()message = _persist_message(sqlite_session)other_message = _persist_message(sqlite_session, message_id="other-message-id")- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=build_test_variable_pool(variables=build_system_variables(workflow_execution_id="run-id")),start_at=0.0,)@@ -427,7 +429,6 @@ class TestAdvancedChatGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)iter_next = QueueIterationNextEvent(index=1,@@ -435,7 +436,6 @@ class TestAdvancedChatGenerateTaskPipeline:node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="LLM",- node_run_index=1,)iter_done = QueueIterationCompletedEvent(node_execution_id="exec",@@ -443,7 +443,6 @@ class TestAdvancedChatGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)loop_start = QueueLoopStartEvent(node_execution_id="exec",@@ -451,7 +450,6 @@ class TestAdvancedChatGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)loop_next = QueueLoopNextEvent(index=1,@@ -459,7 +457,6 @@ class TestAdvancedChatGenerateTaskPipeline:node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="LLM",- node_run_index=1,)loop_done = QueueLoopCompletedEvent(node_execution_id="exec",@@ -467,7 +464,6 @@ class TestAdvancedChatGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)assert list(pipeline._handle_iteration_start_event(iter_start)) == ["iter_start"]@@ -480,7 +476,8 @@ class TestAdvancedChatGenerateTaskPipeline:def test_workflow_finish_handlers(self):pipeline = _make_pipeline()pipeline._workflow_run_id = "run-id"- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-id")),@@ -628,7 +625,7 @@ class TestAdvancedChatGenerateTaskPipeline:pipeline._workflow_response_converter.human_input_form_timeout_to_stream_response = lambda **kwargs: "timeout"filled_event = QueueHumanInputFormFilledEvent(- node_execution_id="exec",+ form_id="form",node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="title",@@ -637,6 +634,7 @@ class TestAdvancedChatGenerateTaskPipeline:action_text="action",)timeout_event = QueueHumanInputFormTimeoutEvent(+ form_id="form",node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="title",@@ -665,7 +663,8 @@ class TestAdvancedChatGenerateTaskPipeline:message = _persist_message(sqlite_session)- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-id")),@@ -697,7 +696,8 @@ class TestAdvancedChatGenerateTaskPipeline:def test_handle_message_end_event_applies_output_moderation(self):pipeline = _make_pipeline()- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-id")),diff --git a/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.pyindex db498ebaac6..1da9f8f77c4 100644--- a/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.py+++ b/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_generator.py@@ -23,6 +23,7 @@ from core.ops.ops_trace_manager import TraceQueueManagerfrom graphon.model_runtime.errors.invoke import InvokeAuthorizationErrorfrom models import Accountfrom models.model import App, AppMode, AppModelConfig, Conversation, Message+from services.workflow_run_agg import WorkflowRunAggfrom tests.unit_tests.config_override import apply_config_overrides@@ -74,7 +75,7 @@ def generator(assert agent_records[0].id == "conv"# Keep trace configuration lookup and initialization real, but do not schedule background timers.monkeypatch.setattr(TraceQueueManager, "start_timer", lambda _manager: None)- return AgentChatAppGenerator()+ return AgentChatAppGenerator(execution_driver=WorkflowRunAgg.run)@pytest.fixturediff --git a/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_runner.py b/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_runner.pyindex 29405e16780..a8ea37696c8 100644--- a/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_runner.py+++ b/api/tests/unit_tests/core/app/apps/agent_chat/test_agent_chat_app_runner.py@@ -22,6 +22,7 @@ from graphon.model_runtime.entities.llm_entities import LLMMode, LLMResultChunk,from graphon.model_runtime.entities.message_entities import AssistantPromptMessagefrom graphon.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKeyfrom models.model import App, Conversation, Message, MessageAgentThought, MessageAnnotation+from services.workflow_run_agg import WorkflowRunAggdef _assert_direct_output(agent_queue_manager: MessageBasedAppQueueManager, text: str) -> None:@@ -37,7 +38,7 @@ def _assert_direct_output(agent_queue_manager: MessageBasedAppQueueManager, text@pytest.fixturedef runner(agent_records: tuple[Conversation, Message]) -> AgentChatAppRunner:assert agent_records[0].id == "conv"- return AgentChatAppRunner()+ return AgentChatAppRunner(execution_driver=WorkflowRunAgg.run)def _records(session: Session) -> tuple[Conversation, Message]:diff --git a/api/tests/unit_tests/core/app/apps/common/test_graph_runtime_state_support.py b/api/tests/unit_tests/core/app/apps/common/test_graph_runtime_state_support.pyindex b373ecd3b6e..74a58b06962 100644--- a/api/tests/unit_tests/core/app/apps/common/test_graph_runtime_state_support.py+++ b/api/tests/unit_tests/core/app/apps/common/test_graph_runtime_state_support.py@@ -5,17 +5,17 @@ import pytestfrom core.app.apps.common.graph_runtime_state_support import GraphRuntimeStateSupportfrom core.workflow.system_variables import build_system_variablesfrom core.workflow.variable_pool_initializer import add_variables_to_pool-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePool-def _make_state(workflow_run_id: str | None) -> GraphRuntimeState:+def _make_state(workflow_run_id: str | None) -> RuntimeState:variable_pool = VariablePool()add_variables_to_pool(variable_pool, build_system_variables(workflow_execution_id=workflow_run_id))- return GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)class _StubPipeline(GraphRuntimeStateSupport):- def __init__(self, *, cached_state: GraphRuntimeState | None, queue_state: GraphRuntimeState | None):+ def __init__(self, *, cached_state: RuntimeState | None, queue_state: RuntimeState | None):self._graph_runtime_state = cached_stateself._base_task_pipeline = SimpleNamespace(queue_manager=SimpleNamespace(graph_runtime_state=queue_state))diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.pyindex 4ecfa826809..6f5ae204f83 100644--- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py+++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py@@ -51,7 +51,7 @@ def test_human_input_form_filled_stream_response_contains_rendered_content() ->)queue_event = QueueHumanInputFormFilledEvent(- node_execution_id="exec-1",+ form_id="form-1",node_id="node-1",node_type="human-input",node_title="Human Input",@@ -63,6 +63,7 @@ def test_human_input_form_filled_stream_response_contains_rendered_content() ->resp = converter.human_input_form_filled_to_stream_response(event=queue_event, task_id="task-1")assert resp.workflow_run_id == "run-1"+ assert resp.data.form_id == "form-1"assert resp.data.node_id == "node-1"assert resp.data.node_title == "Human Input"assert resp.data.rendered_content.startswith("# Title")@@ -79,7 +80,7 @@ def test_human_input_form_filled_stream_response_serializes_submitted_data() ->)queue_event = QueueHumanInputFormFilledEvent(- node_execution_id="exec-1",+ form_id="form-1",node_id="node-1",node_type="human-input",node_title="Human Input",@@ -110,6 +111,7 @@ def test_human_input_form_timeout_stream_response_contains_timeout_metadata() ->)queue_event = QueueHumanInputFormTimeoutEvent(+ form_id="form-1",node_id="node-1",node_type="human-input",node_title="Human Input",@@ -119,6 +121,7 @@ def test_human_input_form_timeout_stream_response_contains_timeout_metadata() ->resp = converter.human_input_form_timeout_to_stream_response(event=queue_event, task_id="task-1")assert resp.workflow_run_id == "run-1"+ assert resp.data.form_id == "form-1"assert resp.data.node_id == "node-1"assert resp.data.node_title == "Human Input"assert resp.data.expiration_time == 1735689600diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.pyindex f2167100fb2..2b97a528104 100644--- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py+++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py@@ -1,8 +1,19 @@+from datetime import datetime, timedelta++import pytest+from core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.apps.common.workflow_response_converter import WorkflowResponseConverterfrom core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity+from core.app.entities.queue_entities import (+ NodeExecutionSnapshot,+ QueueNodeStartedEvent,+ QueueNodeSucceededEvent,+ QueueWorkflowStartedEvent,+)from core.workflow.system_variables import build_system_variablesfrom graphon.entities import WorkflowStartReason+from graphon.enums import BuiltinNodeTypesfrom models.account import Accountfrom models.model import AppMode@@ -58,3 +69,95 @@ def test_workflow_start_stream_response_carries_initial_reason() -> None:reason=WorkflowStartReason.INITIAL,)assert resp.data.reason is WorkflowStartReason.INITIAL+++def test_node_start_preserves_the_allocated_index_when_events_arrive_out_of_order():+ converter = _build_converter()+ converter.workflow_start_to_stream_response(+ task_id="task-1", workflow_run_id="run-1", workflow_id="wf-1", reason=WorkflowStartReason.INITIAL+ )+ for index in (7, 3):+ response = converter.workflow_node_start_to_stream_response(+ task_id="task-1",+ event=QueueNodeStartedEvent(+ node_execution_id=f"execution-{index}",+ node_id=f"node-{index}",+ node_title="Start",+ node_type=BuiltinNodeTypes.START,+ node_run_index=index,+ start_at=datetime(2026, 9, 7),+ provider_type="",+ provider_id="",+ ),+ )+ assert response is not None+ assert response.data.index == index++[email protected]("next_node_already_persisted", [False, True])+def test_resumed_node_metadata_and_sequence(next_node_already_persisted):+ started_at = datetime(2026, 9, 7)+ converter = _build_converter()+ converter.workflow_start_to_stream_response(+ task_id="task-1", workflow_run_id="run-1", workflow_id="wf-1", reason=WorkflowStartReason.INITIAL+ )++ def node_start(execution_id, title, node_type, index):+ return QueueNodeStartedEvent(+ node_execution_id=execution_id,+ node_id=execution_id,+ node_run_index=index,+ node_title=title,+ node_type=node_type,+ start_at=started_at,+ provider_type="workflow",+ provider_id="tool-1",+ )++ request = node_start("request", "Request", BuiltinNodeTypes.START, 1)+ tool = node_start("tool", "Approval tool", BuiltinNodeTypes.TOOL, 2)+ for index, event in enumerate((request, tool), start=1):+ response = converter.workflow_node_start_to_stream_response(event=event, task_id="task-1")+ assert response is not None+ assert response.data.index == index++ snapshots = [+ NodeExecutionSnapshot(execution_id="request", title="Request", index=1, start_at=started_at),+ NodeExecutionSnapshot(execution_id="tool", title="Approval tool", index=2, start_at=started_at),+ ]+ if next_node_already_persisted:+ snapshots.append(NodeExecutionSnapshot(execution_id="end", title="Result", index=3, start_at=started_at))+ # Resume history survives the actual queued event's serialization boundary.+ queued = QueueWorkflowStartedEvent.model_validate_json(+ QueueWorkflowStartedEvent(node_execution_snapshots=tuple(snapshots)).model_dump_json()+ )++ converter = _build_converter()+ converter.workflow_start_to_stream_response(+ task_id="task-1",+ workflow_run_id="run-1",+ workflow_id="wf-1",+ reason=WorkflowStartReason.RESUMPTION,+ node_execution_snapshots=queued.node_execution_snapshots,+ )+ response = converter.workflow_node_finish_to_stream_response(+ event=QueueNodeSucceededEvent(+ node_execution_id="tool",+ node_id="tool",+ node_type=BuiltinNodeTypes.TOOL,+ start_at=started_at,+ finished_at=started_at + timedelta(seconds=4),+ outputs={"decision": "approve"},+ ),+ task_id="task-1",+ )+ assert response is not None+ assert (response.data.title, response.data.index, response.data.elapsed_time) == ("Approval tool", 2, 4)+ assert response.data.outputs == {"decision": "approve"}++ for index, execution_id in enumerate(("end", "next"), start=3):+ response = converter.workflow_node_start_to_stream_response(+ event=node_start(execution_id, "Result", BuiltinNodeTypes.END, index), task_id="task-1"+ )+ assert response is not None+ assert response.data.index == indexdiff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.pyindex b15f8500bc4..93149691cbb 100644--- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.py+++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_truncation.py@@ -800,7 +800,6 @@ class TestWorkflowResponseConverterServiceApiTruncation:node_id="test_iteration",node_type=BuiltinNodeTypes.ITERATION,node_title="Test Iteration",- node_run_index=0,start_at=naive_utc_now(),inputs=large_value,metadata={},@@ -831,7 +830,6 @@ class TestWorkflowResponseConverterServiceApiTruncation:start_at=naive_utc_now(),inputs=large_inputs,metadata={},- node_run_index=0,)response = converter.workflow_loop_start_to_stream_response(diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.pyindex 587e8da9424..71cf0564e34 100644--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py@@ -20,6 +20,7 @@ from models.workflow import Workflow, WorkflowTypefrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom services.data_source.credential_gateway import DatasourceProviderCredentialStorefrom services.data_source.provider_service import DatasourceProviderService+from services.workflow_run_agg import WorkflowRunAggTENANT_ID = "00000000-0000-0000-0000-000000000001"PIPELINE_ID = "00000000-0000-0000-0000-000000000002"@@ -42,6 +43,7 @@ class FakeRagPipelineGenerateEntity(SimpleNamespace):@pytest.fixturedef generator(mocker: MockerFixture, sqlite_engine: Engine):gen = module.PipelineGenerator(+ execution_driver=WorkflowRunAgg.run,documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),datasource_providers=DatasourceProviderService(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)@@ -386,7 +388,7 @@ def test_generate_worker_handles_errors()runner_instance = MagicMock()- runner_instance.run.side_effect = ValueError("bad")+ runner_instance.prepare.side_effect = ValueError("bad")mocker.patch.object(module, "PipelineRunner", return_value=runner_instance)queue_manager = MagicMock()@@ -422,6 +424,9 @@ def test_generate_worker_sets_system_user_id_for_external_call()runner_instance = MagicMock()+ execution_driver = mocker.patch.object(generator, "_execution_driver")+ workflow_execution_repository = MagicMock()+ workflow_node_execution_repository = MagicMock()mocker.patch.object(module, "PipelineRunner", return_value=runner_instance)generator._generate_worker(@@ -430,11 +435,19 @@ def test_generate_worker_sets_system_user_id_for_external_call(queue_manager=MagicMock(),context=contextlib.nullcontext(),variable_loader=MagicMock(),- workflow_execution_repository=MagicMock(),- workflow_node_execution_repository=MagicMock(),+ workflow_execution_repository=workflow_execution_repository,+ workflow_node_execution_repository=workflow_node_execution_repository,)- assert module.PipelineRunner.call_args.kwargs["system_user_id"] == "session"+ runner_kwargs = module.PipelineRunner.call_args.kwargs+ assert runner_kwargs["system_user_id"] == "session"+ assert runner_kwargs["execution_driver"] is execution_driver+ assert runner_kwargs["documents"] is generator._documents+ assert "workflow_execution_repository" not in runner_kwargs+ assert "workflow_node_execution_repository" not in runner_kwargs+ execution_driver.assert_called_once_with(+ runner_instance, None, workflow_execution_repository, workflow_node_execution_repository+ )def test_generate_raises_when_workflow_not_found(generator, mocker: MockerFixture, sqlite_session: Session):diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.pyindex 64a354f2a45..9632de39766 100644--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py@@ -25,18 +25,15 @@ from core.app.apps.pipeline.pipeline_queue_manager import PipelineQueueManagerfrom core.app.apps.pipeline.pipeline_runner import PipelineRunnerfrom core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntityfrom core.app.entities.queue_entities import AppQueueEvent, QueueWorkflowFailedEvent-from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository-from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepositoryfrom core.workflow.workflow_entry import WorkflowEntry+from graphon.engine_events import EngineEvent, GraphRunFailedEventfrom graphon.graph import Graph-from graphon.graph_events import GraphEngineEvent, GraphRunFailedEvent-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variable_loader import DUMMY_VARIABLE_LOADER-from models.account import Accountfrom models.dataset import Dataset, Document, Pipeline-from models.enums import DocumentCreatedFrom, WorkflowRunTriggeredFrom+from models.enums import DocumentCreatedFromfrom models.model import AppMode, EndUser-from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowType+from models.workflow import Workflow, WorkflowTypefrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom tests.unit_tests.model_factories import make_dataset, make_document, make_end_user, make_workflow@@ -136,33 +133,17 @@ def isolate_queue_redis(monkeypatch: pytest.MonkeyPatch) -> None:def _build_runner(app_generate_entity: RagPipelineGenerateEntity, sqlite_engine: Engine, workflow: Workflow) -> PipelineRunner:- user = Account(name="User", email="[email protected]")- user.id = app_generate_entity.user_idreturn PipelineRunner(application_generate_entity=app_generate_entity,queue_manager=PipelineQueueManager(task_id=app_generate_entity.task_id,- user_id=user.id,+ user_id=app_generate_entity.user_id,invoke_from=app_generate_entity.invoke_from,app_mode=AppMode.RAG_PIPELINE,),variable_loader=DUMMY_VARIABLE_LOADER,workflow=workflow,system_user_id="sys",- workflow_execution_repository=SQLAlchemyWorkflowExecutionRepository(- session_factory=sqlite_engine,- tenant_id="tenant",- user=user,- app_id="pipe",- triggered_from=WorkflowRunTriggeredFrom.APP_RUN,- ),- workflow_node_execution_repository=SQLAlchemyWorkflowNodeExecutionRepository(- session_factory=sqlite_engine,- tenant_id="tenant",- user=user,- app_id="pipe",- triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,- ),documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),)@@ -189,19 +170,22 @@ def test_init_rag_pipeline_graph_invalid_config(mocker, runner):with pytest.raises(ValueError, match="nodes or edges not found in workflow graph"):runner._init_rag_pipeline_graph(- workflow=workflow, graph_runtime_state=GraphRuntimeState(variable_pool=VariablePool(), start_at=0)+ workflow=workflow,+ graph_runtime_state=RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0),)workflow.graph = json.dumps({"nodes": "bad", "edges": []})with pytest.raises(ValueError, match="nodes in workflow graph must be a list"):runner._init_rag_pipeline_graph(- workflow=workflow, graph_runtime_state=GraphRuntimeState(variable_pool=VariablePool(), start_at=0)+ workflow=workflow,+ graph_runtime_state=RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0),)workflow.graph = json.dumps({"nodes": [], "edges": "bad"})with pytest.raises(ValueError, match="edges in workflow graph must be a list"):runner._init_rag_pipeline_graph(- workflow=workflow, graph_runtime_state=GraphRuntimeState(variable_pool=VariablePool(), start_at=0)+ workflow=workflow,+ graph_runtime_state=RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0),)@@ -211,10 +195,33 @@ def test_init_rag_pipeline_graph_not_found(mocker, runner):with pytest.raises(ValueError, match="Unable to determine default root node ID from workflow graph"):runner._init_rag_pipeline_graph(- workflow=workflow, graph_runtime_state=GraphRuntimeState(variable_pool=VariablePool(), start_at=0)+ workflow=workflow,+ graph_runtime_state=RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0),)+def test_init_rag_pipeline_graph_honors_canonical_root_ownership(runner):+ workflow = _workflow(+ graph={+ "nodes": [+ {+ "id": "start",+ "parentId": "stale-owner",+ "data": {"type": "start", "title": "Start", "variables": [], "container_id": ""},+ },+ {"id": "end", "data": {"type": "end", "title": "Output", "outputs": []}},+ ],+ "edges": [{"id": "start-end", "source": "start", "target": "end"}],+ }+ )+ runtime_state = RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0.0)++ graph = runner._init_rag_pipeline_graph(workflow=workflow, graph_runtime_state=runtime_state)++ assert graph.root_node.id == "start"+ assert set(graph.nodes) == {"start", "end"}++def test_update_document_status_on_failure(runner, sqlite_session: Session):document = _document()_, dataset, _ = _persist_scope(sqlite_session, documents=(document,))@@ -278,7 +285,7 @@ def test_run_pipeline_not_found(sqlite_engine: Engine):runner = _build_runner(app_generate_entity, sqlite_engine, _workflow())with pytest.raises(ValueError, match="Pipeline not found"):- runner.run()+ runner.prepare()def test_run_pipeline_from_other_tenant_is_not_found(runner: PipelineRunner, sqlite_session: Session):@@ -287,7 +294,7 @@ def test_run_pipeline_from_other_tenant_is_not_found(runner: PipelineRunner, sqlsqlite_session.commit()with pytest.raises(ValueError, match="Pipeline not found"):- runner.run()+ runner.prepare()def _unexpected_workflow_lookup(session: Session, pipeline: Pipeline, workflow_id: str) -> Workflow | None:@@ -316,7 +323,7 @@ def test_run_rejects_unowned_pipeline_dataset(monkeypatch.setattr(runner, "get_workflow", _unexpected_workflow_lookup)with pytest.raises(ValueError, match="Pipeline dataset not found"):- runner.run()+ runner.prepare()def test_run_rejects_document_outside_pipeline_dataset_after_async_boundary(@@ -330,7 +337,7 @@ def test_run_rejects_document_outside_pipeline_dataset_after_async_boundary(monkeypatch.setattr(runner, "get_workflow", _unexpected_workflow_lookup)with pytest.raises(ValueError, match="Pipeline document not found"):- runner.run()+ runner.prepare()def test_run_rejects_original_document_outside_pipeline_dataset_after_async_boundary(@@ -344,7 +351,7 @@ def test_run_rejects_original_document_outside_pipeline_dataset_after_async_bounmonkeypatch.setattr(runner, "get_workflow", _unexpected_workflow_lookup)with pytest.raises(ValueError, match="Pipeline original document not found"):- runner.run()+ runner.prepare()def test_run_workflow_not_initialized(sqlite_session: Session, sqlite_engine: Engine):@@ -358,7 +365,7 @@ def test_run_workflow_not_initialized(sqlite_session: Session, sqlite_engine: Enrunner = _build_runner(app_generate_entity, sqlite_engine, _workflow())with pytest.raises(ValueError, match="Workflow not initialized"):- runner.run()+ runner.prepare()def _start_workflow() -> Workflow:@@ -378,7 +385,7 @@ def test_run_single_iteration_path(monkeypatch: pytest.MonkeyPatch, sqlite_sessipublished: list[AppQueueEvent] = []failure = GraphRunFailedEvent(error="iteration failed")variable_pool = VariablePool()- state = GraphRuntimeState(variable_pool=variable_pool, start_at=0)+ state = RuntimeState(workflow_id=workflow.id, variable_pool=variable_pool, start_at=0)def prepare_single_node_execution(*,@@ -386,20 +393,22 @@ def test_run_single_iteration_path(monkeypatch: pytest.MonkeyPatch, sqlite_sessisingle_iteration_run: RagPipelineGenerateEntity.SingleIterationRunEntity | None,single_loop_run: RagPipelineGenerateEntity.SingleLoopRunEntity | None,user_id: str,- ) -> tuple[Graph, VariablePool, GraphRuntimeState]:+ ) -> tuple[Graph, VariablePool, RuntimeState]:prepared.append((workflow.id, single_iteration_run, single_loop_run, user_id))graph = runner._init_rag_pipeline_graph(workflow=workflow, graph_runtime_state=state, start_node_id="start")return graph, variable_pool, state- def run_workflow(entry: WorkflowEntry) -> Generator[GraphEngineEvent]:- assert entry.graph_engine.graph_runtime_state is state+ def run_workflow(entry: WorkflowEntry) -> Generator[EngineEvent]:+ assert entry.graph_engine.runtime_state is stateyield failuremonkeypatch.setattr(runner, "_prepare_single_node_execution", prepare_single_node_execution)monkeypatch.setattr(runner, "_publish_event", published.append)monkeypatch.setattr(WorkflowEntry, "run", run_workflow)- runner.run()+ prepared_run = runner.prepare()+ for engine_event in prepared_run.entry.run():+ runner.handle_event(prepared_run.entry, engine_event)assert prepared == [("wf", app_generate_entity.single_iteration_run, None, "user")]assert runner._queue_manager.graph_runtime_state is state@@ -426,7 +435,7 @@ def test_run_normal_path_builds_graph(monkeypatch: pytest.MonkeyPatch, sqlite_serunner = _build_runner(app_generate_entity, sqlite_engine, workflow)entries: list[WorkflowEntry] = []- def run_workflow(entry: WorkflowEntry) -> Generator[GraphEngineEvent]:+ def run_workflow(entry: WorkflowEntry) -> Generator[EngineEvent]:events.append("workflow_run")entries.append(entry)yield from ()@@ -438,7 +447,8 @@ def test_run_normal_path_builds_graph(monkeypatch: pytest.MonkeyPatch, sqlite_seevent.listen(sqlite_engine, "checkin", record_checkin)try:- runner.run()+ prepared_run = runner.prepare()+ list(prepared_run.entry.run())finally:event.remove(sqlite_engine, "checkin", record_checkin)@@ -447,7 +457,7 @@ def test_run_normal_path_builds_graph(monkeypatch: pytest.MonkeyPatch, sqlite_seassert len(entries) == 1graph_engine = entries[0].graph_engineassert graph_engine.graph.root_node.id == "start"- assert graph_engine.graph_runtime_state is runner._queue_manager.graph_runtime_state- input_variable = graph_engine.graph_runtime_state.variable_pool.get(["start", "input1"])+ assert graph_engine.runtime_state is runner._queue_manager.graph_runtime_state+ input_variable = graph_engine.runtime_state.variable_pool.get(["start", "input1"])assert input_variable is not Noneassert input_variable.value == "v1"diff --git a/api/tests/unit_tests/core/app/apps/test_advanced_chat_app_generator.py b/api/tests/unit_tests/core/app/apps/test_advanced_chat_app_generator.pyindex 3543022b5db..e06c58c471a 100644--- a/api/tests/unit_tests/core/app/apps/test_advanced_chat_app_generator.py+++ b/api/tests/unit_tests/core/app/apps/test_advanced_chat_app_generator.py@@ -18,6 +18,7 @@ from models import Account, Workflowfrom models.enums import ConversationFromSourcefrom models.model import App, AppMode, Conversationfrom services.errors.conversation import ConversationNotExistsError+from services.workflow_run_agg import WorkflowRunAggdef _make_app_config() -> WorkflowUIBasedAppConfig:@@ -86,7 +87,7 @@ def test_init_generate_records_sets_conversation_metadata(sqlite_session: Sessioapp_config = _make_app_config()entity = _make_generate_entity(app_config)- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)conversation, _ = generator._init_generate_records(entity,@@ -125,7 +126,7 @@ def test_init_generate_records_marks_existing_conversation(sqlite_session: Sessisqlite_session.add(existing_conversation)sqlite_session.flush()- generator = AdvancedChatAppGenerator()+ generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)conversation, _ = generator._init_generate_records(entity,@@ -187,7 +188,7 @@ def test_generate_falls_back_to_new_conversation_when_conversation_missing(monkeypatch.setattr(AdvancedChatAppGenerator, "_generate", fake_generate)- result = AdvancedChatAppGenerator().generate(+ result = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run).generate(app_model=app_model,workflow=workflow,user=user,diff --git a/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py b/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.pyindex 2debd2e3d8d..36a1da95b72 100644--- a/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py+++ b/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py@@ -10,6 +10,7 @@ from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFromfrom core.app.apps.execution_coordinator import AppExecutionStatefrom core.app.entities.app_invoke_entities import InvokeFromfrom core.app.entities.queue_entities import QueueErrorEvent+from graphon.engine.command import AbortCommandfrom models import Tenant@@ -103,7 +104,7 @@ class TestBaseAppQueueManager:def test_completed_listener_defers_graph_runtime_state_cleanup_until_listener_exits(self):with (patch("core.app.apps.base_app_queue_manager.redis_client") as mock_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):mock_redis.setex.return_value = Truemock_redis.get.return_value = None@@ -116,22 +117,26 @@ class TestBaseAppQueueManager:assert manager.graph_runtime_state is runtime_stateassert list(manager.listen()) == []assert manager.graph_runtime_state is None- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()def test_execution_coordinator_abort_is_idempotent_when_graph_stop_command_fails(self, caplog):with (patch("core.app.apps.base_app_queue_manager.redis_client") as queue_redis,patch("core.app.apps.execution_coordinator.redis_client") as execution_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.RedisChannel") as redis_channel,):queue_redis.setex.return_value = Trueexecution_redis.setex.return_value = True- graph_engine_manager.return_value.send_stop_command.side_effect = RuntimeError("redis unavailable")+ redis_channel.return_value.send_command.side_effect = RedisConnectionError("redis unavailable")manager = DummyQueueManager(task_id="t1", user_id="u1", invoke_from=InvokeFrom.SERVICE_API)assert manager._execution_coordinator.request_abort("stream closed") is Trueassert manager._execution_coordinator.request_abort("duplicate") is Falseexecution_redis.setex.assert_called_once_with("generate_task_stopped:t1", 600, 1)- graph_engine_manager.return_value.send_stop_command.assert_called_once_with("t1", reason="stream closed")- assert "Failed to send stop command for app execution task=t1" in caplog.text+ redis_channel.assert_called_once_with(execution_redis, "workflow:t1:commands")+ redis_channel.return_value.send_command.assert_called_once()+ command = redis_channel.return_value.send_command.call_args.args[0]+ assert isinstance(command, AbortCommand)+ assert command.reason == "stream closed"+ assert "Failed to send Engine abort command for task t1" in caplog.textdiff --git a/api/tests/unit_tests/core/app/apps/test_execution_coordinator.py b/api/tests/unit_tests/core/app/apps/test_execution_coordinator.pyindex 5ab95b961cb..f13b85c8d6a 100644--- a/api/tests/unit_tests/core/app/apps/test_execution_coordinator.py+++ b/api/tests/unit_tests/core/app/apps/test_execution_coordinator.py@@ -16,7 +16,7 @@ def test_listener_close_does_not_abort_running_attempt() -> None:on_timeout = Mock()with (patch("core.app.apps.execution_coordinator.redis_client") as redis_client,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):coordinator = AppExecutionCoordinator(task_id="task", on_timeout=on_timeout, timeout_seconds=1200)@@ -25,7 +25,7 @@ def test_listener_close_does_not_abort_running_attempt() -> None:assert coordinator.state is AppExecutionState.RUNNINGredis_client.setex.assert_not_called()- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()on_timeout.assert_not_called()@@ -33,7 +33,7 @@ def test_paused_attempt_ignores_listener_close_and_timeout() -> None:on_timeout = Mock()with (patch("core.app.apps.execution_coordinator.redis_client") as redis_client,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):coordinator = AppExecutionCoordinator(task_id="task", on_timeout=on_timeout, timeout_seconds=0)coordinator.mark_paused()@@ -43,7 +43,7 @@ def test_paused_attempt_ignores_listener_close_and_timeout() -> None:assert coordinator.state is AppExecutionState.PAUSEDredis_client.setex.assert_not_called()- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()on_timeout.assert_not_called()@@ -51,7 +51,7 @@ def test_watchdog_aborts_and_notifies_response_pipeline() -> None:on_timeout = Mock()with (patch("core.app.apps.execution_coordinator.redis_client") as redis_client,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):coordinator = AppExecutionCoordinator(task_id="task", on_timeout=on_timeout, timeout_seconds=0)@@ -60,7 +60,7 @@ def test_watchdog_aborts_and_notifies_response_pipeline() -> None:assert coordinator.state is AppExecutionState.ABORTINGredis_client.setex.assert_called_once_with("generate_task_stopped:task", 600, 1)- graph_engine_manager.return_value.send_stop_command.assert_called_once_with(+ send_abort_command.assert_called_once_with("task",reason="App execution exceeded 0 seconds",)@@ -92,10 +92,7 @@ def test_clearing_cancellation_signals_drops_stop_flag_and_queued_commands() ->redis_channel.assert_called_once_with(redis_client, "workflow:task:commands")channel.fetch_commands.assert_called_once_with()- assert redis_client.delete.call_args_list == [- (("generate_task_stopped:task",), {}),- (("workflow:task:commands",), {}),- ]+ redis_client.delete.assert_called_once_with("generate_task_stopped:task")def test_clearing_cancellation_signals_ignores_empty_task_id() -> None:@@ -133,14 +130,14 @@ def test_stop_flag_failure_does_not_block_graph_stop(caplog: pytest.LogCaptureFion_timeout = Mock()with (patch("core.app.apps.execution_coordinator.redis_client") as redis_client,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):redis_client.setex.side_effect = RuntimeError("redis write failed")coordinator = AppExecutionCoordinator(task_id="task", on_timeout=on_timeout, timeout_seconds=1200)coordinator.request_abort("test abort")- graph_engine_manager.return_value.send_stop_command.assert_called_once_with(+ send_abort_command.assert_called_once_with("task",reason="test abort",)diff --git a/api/tests/unit_tests/core/app/apps/test_message_based_app_queue_manager.py b/api/tests/unit_tests/core/app/apps/test_message_based_app_queue_manager.pyindex 9d480366334..90240e9b567 100644--- a/api/tests/unit_tests/core/app/apps/test_message_based_app_queue_manager.py+++ b/api/tests/unit_tests/core/app/apps/test_message_based_app_queue_manager.py@@ -93,7 +93,7 @@ class TestMessageBasedAppQueueManager:with (patch("core.app.apps.base_app_queue_manager.redis_client") as queue_redis,patch("core.app.apps.execution_coordinator.redis_client") as execution_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):queue_redis.get.return_value = Nonemanager = MessageBasedAppQueueManager(@@ -115,4 +115,4 @@ class TestMessageBasedAppQueueManager:assert isinstance(messages[0].event, QueueWorkflowPausedEvent)assert manager.execution_state is AppExecutionState.PAUSEDexecution_redis.setex.assert_not_called()- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()diff --git a/api/tests/unit_tests/core/app/apps/test_pause_resume.py b/api/tests/unit_tests/core/app/apps/test_pause_resume.pyindex 3f5ac90292d..779c79b442e 100644--- a/api/tests/unit_tests/core/app/apps/test_pause_resume.py+++ b/api/tests/unit_tests/core/app/apps/test_pause_resume.py@@ -15,30 +15,31 @@ from core.app.entities.app_invoke_entities import InvokeFromfrom core.workflow import node_factory as node_factory_modulefrom core.workflow.node_factory import DifyNodeFactoryfrom core.workflow.system_variables import build_system_variables-from graphon.entities import WorkflowStartReason-from graphon.entities.base_node_data import BaseNodeData, RetryConfig-from graphon.entities.pause_reason import SchedulingPause-from graphon.enums import BuiltinNodeTypes, NodeType, WorkflowNodeExecutionStatus-from graphon.graph import Graph-from graphon.graph_engine import GraphEngine-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import (- GraphEngineEvent,+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import (+ EngineEvent,GraphRunPausedEvent,GraphRunStartedEvent,GraphRunSucceededEvent,NodeRunSucceededEvent,)+from graphon.entities import WorkflowStartReason+from graphon.entities.base_node_data import BaseNodeData, RetryConfig+from graphon.entities.pause_reason import SchedulingPause+from graphon.enums import BuiltinNodeTypes, NodeType, WorkflowNodeExecutionStatus+from graphon.graph import Graphfrom graphon.node_events import NodeRunResult, PauseRequestedEventfrom graphon.nodes.base.entities import OutputVariableEntityfrom graphon.nodes.base.node import Nodefrom graphon.nodes.end.entities import EndNodeDatafrom graphon.nodes.start.entities import StartNodeData-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom models.account import Accountfrom models.enums import ConversationFromSourcefrom models.model import App, AppMode, Conversation, Messagefrom models.workflow import Workflow, WorkflowType+from services.workflow_run_agg import WorkflowRunAggfrom tests.workflow_test_utils import build_test_graph_init_paramsif "core.ops.ops_trace_manager" not in sys.modules:@@ -69,15 +70,15 @@ class _StubToolNode(Node[_StubToolNodeData]):node_id: str,data: _StubToolNodeData,*,- graph_init_params,- graph_runtime_state,+ init_params,+ runtime_state,**_kwargs: Any,) -> None:super().__init__(node_id=node_id,data=data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)def _get_error_strategy(self):@@ -107,7 +108,7 @@ class _StubToolNode(Node[_StubToolNodeData]):status=WorkflowNodeExecutionStatus.SUCCEEDED,outputs={"value": f"{self.id}-done"},)- yield self._convert_node_run_result_to_graph_node_event(result)+ yield self._convert_node_run_result_to_node_event(result)def _patch_tool_node(mocker: MockerFixture):@@ -163,7 +164,7 @@ def _build_graph_config(*, pause_on: str | None) -> dict[str, object]:return {"nodes": nodes, "edges": edges}-def _build_graph(runtime_state: GraphRuntimeState, *, pause_on: str | None) -> Graph:+def _build_graph(runtime_state: RuntimeState, *, pause_on: str | None) -> Graph:graph_config = _build_graph_config(pause_on=pause_on)params = build_test_graph_init_params(workflow_id="workflow",@@ -177,21 +178,21 @@ def _build_graph(runtime_state: GraphRuntimeState, *, pause_on: str | None) -> G)node_factory = DifyNodeFactory(- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)return Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")-def _build_runtime_state(run_id: str) -> GraphRuntimeState:+def _build_runtime_state(run_id: str) -> RuntimeState:variable_pool = VariablePool.from_bootstrap(system_variables=build_system_variables(user_id="user", app_id="app", workflow_id="workflow"),user_inputs={},conversation_variables=[],)variable_pool.add(("sys", "workflow_run_id"), run_id)- return GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter())def _make_app() -> App:@@ -251,23 +252,22 @@ def _make_message() -> Message:)-def _run_with_optional_pause(runtime_state: GraphRuntimeState, *, pause_on: str | None) -> list[GraphEngineEvent]:+def _run_with_optional_pause(runtime_state: RuntimeState, *, pause_on: str | None) -> list[EngineEvent]:command_channel = InMemoryChannel()graph = _build_graph(runtime_state, pause_on=pause_on)- engine = GraphEngine(- workflow_id="workflow",+ engine = Engine(graph=graph,- graph_runtime_state=runtime_state,+ runtime_state=runtime_state,command_channel=command_channel,)- events: list[GraphEngineEvent] = []+ events: list[EngineEvent] = []for event in engine.run():events.append(event)return events-def _node_successes(events: list[GraphEngineEvent]) -> list[str]:+def _node_successes(events: list[EngineEvent]) -> list[str]:return [evt.node_id for evt in events if isinstance(evt, NodeRunSucceededEvent)]@@ -286,12 +286,12 @@ def test_workflow_app_pause_resume_matches_baseline(mocker: MockerFixture):paused_nodes = _node_successes(paused_events)snapshot = paused_state.dumps()- resumed_state = GraphRuntimeState.from_snapshot(snapshot)+ resumed_state = RuntimeState.from_snapshot(snapshot)- generator = wf_app_gen_module.WorkflowAppGenerator()+ generator = wf_app_gen_module.WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)def _fake_generate(**kwargs):- state: GraphRuntimeState = kwargs["graph_runtime_state"]+ state: RuntimeState = kwargs["graph_runtime_state"]events = _run_with_optional_pause(state, pause_on=None)return _node_successes(events)@@ -330,12 +330,12 @@ def test_advanced_chat_pause_resume_matches_baseline(mocker: MockerFixture, unbopaused_nodes = _node_successes(paused_events)snapshot = paused_state.dumps()- resumed_state = GraphRuntimeState.from_snapshot(snapshot)+ resumed_state = RuntimeState.from_snapshot(snapshot)- generator = adv_app_gen_module.AdvancedChatAppGenerator()+ generator = adv_app_gen_module.AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run)def _fake_generate(**kwargs):- state: GraphRuntimeState = kwargs["graph_runtime_state"]+ state: RuntimeState = kwargs["graph_runtime_state"]events = _run_with_optional_pause(state, pause_on=None)return _node_successes(events)@@ -370,7 +370,7 @@ def test_resume_emits_resumption_start_reason(mocker: MockerFixture) -> None:initial_start = next(event for event in paused_events if isinstance(event, GraphRunStartedEvent))assert initial_start.reason == WorkflowStartReason.INITIAL- resumed_state = GraphRuntimeState.from_snapshot(paused_state.dumps())+ resumed_state = RuntimeState.from_snapshot(paused_state.dumps())resumed_events = _run_with_optional_pause(resumed_state, pause_on=None)resume_start = next(event for event in resumed_events if isinstance(event, GraphRunStartedEvent))assert resume_start.reason == WorkflowStartReason.RESUMPTIONdiff --git a/api/tests/unit_tests/core/app/apps/test_workflow_app_generator.py b/api/tests/unit_tests/core/app/apps/test_workflow_app_generator.pyindex 4e1e01780dd..af7b0ba1c4f 100644--- a/api/tests/unit_tests/core/app/apps/test_workflow_app_generator.py+++ b/api/tests/unit_tests/core/app/apps/test_workflow_app_generator.py@@ -1,6 +1,8 @@"""SQLite-backed tests for workflow app generation and worker reload behavior."""import contextlib+import contextvars+import jsonfrom types import SimpleNamespacefrom unittest.mock import MagicMock@@ -12,16 +14,20 @@ import core.app.apps.workflow.app_generator as app_generator_modulefrom core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.apps.draft_variable_saver import DraftVariableSaverFactoryfrom core.app.apps.workflow.app_generator import WorkflowAppGenerator+from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManagerfrom core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity-from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, PauseStatePersistenceLayer+from core.app.entities.queue_entities import QueueWorkflowSucceededEvent+from core.app.layers.pause_state_persist_layer import PauseStateLayerConfigfrom core.ops.ops_trace_manager import OpsTraceManager, TraceQueueManagerfrom core.repositories import SQLAlchemyWorkflowExecutionRepository, SQLAlchemyWorkflowNodeExecutionRepositoryfrom graphon.enums import WorkflowExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePool+from graphon.variable_loader import DUMMY_VARIABLE_LOADERfrom models.enums import CreatorUserRole, EndUserType, WorkflowRunTriggeredFromfrom models.model import App, AppMode, EndUserfrom models.snippet import CustomizedSnippetfrom models.workflow import Workflow, WorkflowKind, WorkflowNodeExecutionTriggeredFrom, WorkflowRun+from services.workflow_run_agg import WorkflowRunAggfrom tests.unit_tests.model_factories import make_workflow@@ -107,8 +113,8 @@ def _generate_entity()-def _runtime_state() -> GraphRuntimeState:- return GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+def _runtime_state() -> RuntimeState:+ return RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)def _repositories(@@ -132,6 +138,59 @@ def _repositories()+def test_worker_executes_and_persists_a_workflow_through_the_injected_driver(+ app: Flask, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session]+) -> None:+ app_model, workflow, end_user = _persist_generator_rows(sqlite_session)+ workflow.graph = json.dumps(+ {+ "nodes": [+ {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}},+ {+ "id": "end",+ "data": {+ "type": "end",+ "title": "End",+ "outputs": [{"variable": "caller", "value_selector": ["sys", "user_id"]}],+ },+ },+ ],+ "edges": [{"id": "start-end", "source": "start", "target": "end"}],+ }+ )+ sqlite_session.commit()+ entity = WorkflowAppGenerateEntity(+ task_id="worker-task",+ app_config=_app_config(app_model, workflow),+ inputs={},+ files=[],+ user_id=end_user.id,+ stream=True,+ invoke_from=InvokeFrom.SERVICE_API,+ workflow_execution_id="worker-run",+ )+ queue = WorkflowAppQueueManager(entity.task_id, end_user.id, entity.invoke_from, AppMode.WORKFLOW)+ execution_repository, node_repository = _repositories(sqlite_session_factory, app_model, end_user)++ WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)._generate_worker(+ flask_app=app,+ application_generate_entity=entity,+ queue_manager=queue,+ context=contextvars.copy_context(),+ variable_loader=DUMMY_VARIABLE_LOADER,+ workflow_execution_repository=execution_repository,+ workflow_node_execution_repository=node_repository,+ )++ events = [message.event for message in queue.listen()]+ assert isinstance(events[-1], QueueWorkflowSucceededEvent), events+ assert events[-1].outputs == {"caller": end_user.session_id}+ run = sqlite_session.get(WorkflowRun, entity.workflow_execution_id)+ assert run is not None+ assert run.status == "succeeded"+ assert run.outputs_dict == {"caller": end_user.session_id}++def test_ensure_snippet_start_node_in_worker_returns_standard_workflow_without_lookup(sqlite_session: Session,) -> None:@@ -191,7 +250,7 @@ def test_ensure_snippet_start_node_in_worker_applies_snippet_start_injection(def test_generate_includes_parent_trace_context_in_extras(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None:- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app, workflow, end_user = _persist_generator_rows(sqlite_session)monkeypatch.setattr(@@ -277,7 +336,7 @@ def test_resume_delegates_to_generate(sqlite_session: Session,sqlite_session_factory: sessionmaker[Session],) -> None:- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app, workflow, end_user = _persist_generator_rows(sqlite_session)mock_generate = MagicMock(return_value="ok")monkeypatch.setattr(generator, "_generate", mock_generate)@@ -319,12 +378,12 @@ def test_resume_delegates_to_generate(assert kwargs["invoke_from"] == InvokeFrom.DEBUGGER-def test_generate_appends_pause_layer_and_forwards_state(+def test_generate_forwards_pause_configuration_and_state(monkeypatch: pytest.MonkeyPatch,sqlite_session: Session,sqlite_session_factory: sessionmaker[Session],) -> None:- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app, workflow, end_user = _persist_generator_rows(sqlite_session)queue_manager = MagicMock()@@ -403,8 +462,8 @@ def test_generate_appends_pause_layer_and_forwards_state(assert result == "converted"graph_engine_layers = worker_kwargs["kwargs"]["graph_engine_layers"]- assert graph_engine_layers[0] == "base-layer"- assert isinstance(graph_engine_layers[1], PauseStatePersistenceLayer)+ assert graph_engine_layers == ("base-layer",)+ assert worker_kwargs["kwargs"]["pause_state_config"].state_owner_user_id == "owner"assert worker_kwargs["kwargs"]["graph_runtime_state"] is graph_runtime_stateassert worker_kwargs["joined"] is Trueassert worker_kwargs["join_timeout"] == 300@@ -416,7 +475,7 @@ def test_resume_path_runs_worker_with_runtime_state(sqlite_session: Session,sqlite_session_factory: sessionmaker[Session],) -> None:- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app, workflow, end_user = _persist_generator_rows(sqlite_session)workflow_run = WorkflowRun(id="run",@@ -452,6 +511,7 @@ def test_resume_path_runs_worker_with_runtime_state(monkeypatch.setattr(app_generator_module.db, "session", db_session)runner_instance = MagicMock()+ runner_instance.prepare.return_value = Nonedef runner_ctor(**kwargs):assert kwargs["graph_runtime_state"] is runtime_state@@ -507,4 +567,4 @@ def test_resume_path_runs_worker_with_runtime_state(assert result == "raw-response"assert worker_lifecycle["joined"] is Trueassert worker_lifecycle["join_timeout"] == 300- runner_instance.run.assert_called_once()+ runner_instance.prepare.assert_called_once()diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_core.py b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_core.pyindex c2ecd33d323..4c98b62b71a 100644--- a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_core.py+++ b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_core.py@@ -4,20 +4,25 @@ import jsonfrom copy import deepcopyfrom datetime import UTC, datetimefrom types import SimpleNamespace+from typing import overrideimport pytestfrom pydantic import ValidationError+from core.app.apps.execution_coordinator import AppExecutionState+from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManagerfrom core.app.apps.workflow_app_runner import WorkflowBasedAppRunnerfrom core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, InvokeFrom, UserFromfrom core.app.entities.queue_entities import (QueueAgentLogEvent,- QueueHumanInputFormFilledEvent,QueueIterationCompletedEvent,+ QueueIterationStartEvent,QueueLoopCompletedEvent,+ QueueLoopStartEvent,QueueNodeExceptionEvent,QueueNodeFailedEvent,QueueNodeRetryEvent,+ QueueNodeStartedEvent,QueueNodeSucceededEvent,QueueReasoningChunkEvent,QueueStopEvent,@@ -29,29 +34,34 @@ from core.app.entities.queue_entities import (from core.workflow.nodes.agent.events import NodeRunAgentLogEventfrom core.workflow.nodes.human_input.pause_reason import HumanInputRequiredfrom core.workflow.system_variables import default_system_variables-from graphon.entities.pause_reason import HitlRequired-from graphon.enums import BuiltinNodeTypes-from graphon.graph_events import (+from core.workflow.workflow_entry import WorkflowEntry, iter_dify_graph_engine_events+from graphon.engine.layer import Layer+from graphon.engine_events import (GraphRunAbortedEvent,GraphRunPausedEvent,GraphRunStartedEvent,GraphRunSucceededEvent,NodeRunExceptionEvent,NodeRunFailedEvent,- NodeRunHumanInputFormFilledEvent,+ NodeRunIterationStartedEvent,NodeRunIterationSucceededEvent,NodeRunLoopFailedEvent,+ NodeRunLoopStartedEvent,+ NodeRunLoopSucceededEvent,NodeRunReasoningChunkEvent,NodeRunRetryEvent,NodeRunStartedEvent,NodeRunStreamChunkEvent,NodeRunSucceededEvent,)+from graphon.entities.pause_reason import HitlRequired+from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKey+from graphon.graph import Graphfrom graphon.node_events import NodeRunResult-from graphon.runtime import GraphRuntimeState, VariablePool-from graphon.variables.segments import StringSegment+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variables.variables import StringVariablefrom models.workflow import Workflow+from services.workflow_run_index import WorkflowRunIndexfrom tests.unit_tests.model_factories import make_workflow@@ -64,7 +74,147 @@ def _workflow(graph: dict[str, object] | None = None) -> Workflow:)+def _workflow_entry(runner: WorkflowBasedAppRunner, state: RuntimeState | None = None) -> WorkflowEntry:+ state = state or RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=0.0)+ graph = runner._init_graph(+ graph_config={+ "nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}],+ "edges": [],+ },+ graph_runtime_state=state,+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ call_depth=0,+ )+ return WorkflowEntry(graph_config=graph.graph_config, graph=graph, call_depth=0, graph_runtime_state=state)+++def _nested_container_graph(ownership: str) -> dict:+ nodes = [+ {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}},+ {+ "id": "outer",+ "data": {+ "type": "loop",+ "title": "Outer loop",+ "start_node_id": "outer-start",+ "loop_count": 1,+ "break_conditions": [],+ "logical_operator": "and",+ },+ },+ {"id": "outer-start", "data": {"type": "loop-start", "title": "Loop start"}},+ {+ "id": "inner",+ "data": {+ "type": "iteration",+ "title": "Inner iteration",+ "start_node_id": "inner-start",+ "iterator_selector": ["start", "items"],+ "output_selector": ["inner", "item"],+ },+ },+ {"id": "inner-start", "data": {"type": "iteration-start", "title": "Iteration start"}},+ {+ "id": "deep",+ "data": {+ "type": "loop",+ "title": "Nested loop",+ "start_node_id": "deep-start",+ "loop_count": 1,+ "break_conditions": [],+ "logical_operator": "and",+ },+ },+ {"id": "deep-start", "data": {"type": "loop-start", "title": "Nested loop start"}},+ {"id": "end", "data": {"type": "end", "title": "Output", "outputs": []}},+ ]+ owners = {"outer-start": "outer", "inner": "outer", "inner-start": "inner", "deep": "inner", "deep-start": "deep"}+ for node in nodes:+ owner = owners.get(node["id"], "")+ if ownership == "canonical":+ node["data"]["container_id"] = owner+ # Canonical ownership, including the empty root owner, takes precedence.+ node["parentId"] = "stale-owner"+ node["data"]["loop_id"] = "stale-owner"+ elif ownership == "parent":+ if owner:+ node["parentId"] = owner+ elif owner:+ node["data"]["loop_id"] = "deep" if owner == "deep" else "outer"+ if owner in {"inner", "deep"}:+ node["data"]["iteration_id"] = "inner"+ return {+ "nodes": nodes,+ "edges": [+ {"id": "start-outer", "source": "start", "target": "outer"},+ {"id": "outer-end", "source": "outer", "target": "end"},+ {"id": "outer-start-inner", "source": "outer-start", "target": "inner"},+ {"id": "inner-start-deep", "source": "inner-start", "target": "deep"},+ ],+ }++class TestWorkflowBasedAppRunner:+ @pytest.mark.parametrize(+ ("event_class", "node_type"),+ [+ (NodeRunIterationStartedEvent, BuiltinNodeTypes.ITERATION),+ (NodeRunIterationSucceededEvent, BuiltinNodeTypes.ITERATION),+ (NodeRunLoopStartedEvent, BuiltinNodeTypes.LOOP),+ (NodeRunLoopSucceededEvent, BuiltinNodeTypes.LOOP),+ ],+ )+ def test_container_queue_events_preserve_lifecycle_and_ancestor_metadata(+ self,+ event_class: type[+ NodeRunIterationStartedEvent+ | NodeRunIterationSucceededEvent+ | NodeRunLoopStartedEvent+ | NodeRunLoopSucceededEvent+ ],+ node_type: str,+ ) -> None:+ queue = WorkflowAppQueueManager(+ task_id="metadata-contract", user_id="account", invoke_from=InvokeFrom.DEBUGGER, app_mode="workflow"+ )+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")+ entry = _workflow_entry(runner)+ event = event_class(+ id="container-execution",+ node_id="container",+ node_type=node_type,+ node_title="Container",+ start_at=datetime.now(UTC),+ metadata={"iteration_length": 2, "total_tokens": 1},+ node_run_result=NodeRunResult(+ metadata={+ WorkflowNodeExecutionMetadataKey.LOOP_ID: "outer",+ WorkflowNodeExecutionMetadataKey.LOOP_INDEX: 3,+ WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS: 2,+ }+ ),+ )+ original_event = event.model_dump()++ runner.handle_event(entry, event)+ queue.stop_listen(execution_state=AppExecutionState.TERMINAL)+ messages = list(queue.listen())++ assert len(messages) == 1+ queued_event = messages[0].event+ assert isinstance(+ queued_event,+ QueueIterationStartEvent | QueueIterationCompletedEvent | QueueLoopStartEvent | QueueLoopCompletedEvent,+ )+ assert queued_event.metadata == {+ "iteration_length": 2,+ "total_tokens": 2,+ "loop_id": "outer",+ "loop_index": 3,+ }+ assert event.model_dump() == original_event+def test_resolve_user_from(self):runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")@@ -75,7 +225,8 @@ class TestWorkflowBasedAppRunner:def test_init_graph_validates_graph_structure(self):runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)@@ -86,6 +237,7 @@ class TestWorkflowBasedAppRunner:graph_runtime_state=runtime_state,user_from=UserFrom.ACCOUNT,invoke_from=InvokeFrom.DEBUGGER,+ call_depth=0,)with pytest.raises(ValidationError, match="nodes"):@@ -94,6 +246,7 @@ class TestWorkflowBasedAppRunner:graph_runtime_state=runtime_state,user_from=UserFrom.ACCOUNT,invoke_from=InvokeFrom.DEBUGGER,+ call_depth=0,)with pytest.raises(ValidationError, match="edges"):@@ -102,18 +255,23 @@ class TestWorkflowBasedAppRunner:graph_runtime_state=runtime_state,user_from=UserFrom.ACCOUNT,invoke_from=InvokeFrom.DEBUGGER,+ call_depth=0,)def test_init_graph_includes_trace_session_id_in_run_context(self, monkeypatch: pytest.MonkeyPatch):runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)captured = {}def fake_from_graph_init_context(**kwargs):- captured["run_context"] = kwargs["graph_init_context"].run_context+ graph_init_context = kwargs["graph_init_context"]+ captured["run_context"] = graph_init_context.run_context+ captured["call_depth"] = graph_init_context.call_depth+ captured["runtime_state"] = kwargs["runtime_state"]return SimpleNamespace()monkeypatch.setattr(@@ -129,9 +287,55 @@ class TestWorkflowBasedAppRunner:invoke_from=InvokeFrom.DEBUGGER,root_node_id="root",trace_session_id="session-1",+ call_depth=3,)assert captured["run_context"][DIFY_RUN_CONTEXT_KEY].trace_session_id == "session-1"+ assert captured["call_depth"] == 3+ assert captured["runtime_state"] is runtime_state++ @pytest.mark.parametrize("ownership", ["canonical", "parent", "legacy"])+ def test_init_graph_preserves_container_execution_scopes(self, ownership: str) -> None:+ runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")+ runtime_state = RuntimeState(+ workflow_id="test-workflow",+ variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),+ start_at=0.0,+ )+ graph_config = _nested_container_graph(ownership)+ original_config = deepcopy(graph_config)+ graph = runner._init_graph(+ graph_config=graph_config,+ graph_runtime_state=runtime_state,+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ root_node_id="start",+ call_depth=0,+ )++ assert set(graph.nodes) == {"start", "outer", "end"}+ outer_graph = Graph.init(+ graph_config=graph.graph_config,+ node_factory=graph.node_factory,+ root_node_id="outer-start",+ container_id="outer",+ )+ assert set(outer_graph.nodes) == {"outer-start", "inner"}+ inner_graph = Graph.init(+ graph_config=outer_graph.graph_config,+ node_factory=outer_graph.node_factory,+ root_node_id="inner-start",+ container_id="inner",+ )+ assert set(inner_graph.nodes) == {"inner-start", "deep"}+ deep_graph = Graph.init(+ graph_config=inner_graph.graph_config,+ node_factory=inner_graph.node_factory,+ root_node_id="deep-start",+ container_id="deep",+ )+ assert set(deep_graph.nodes) == {"deep-start"}+ assert graph_config == original_configdef test_prepare_single_node_execution_requires_run(self):runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")@@ -141,64 +345,46 @@ class TestWorkflowBasedAppRunner:with pytest.raises(ValueError, match="Neither single_iteration_run nor single_loop_run"):runner._prepare_single_node_execution(workflow, None, None, user_id="00000000-0000-0000-0000-000000000001")- def test_get_graph_and_variable_pool_for_single_node_run(self, monkeypatch: pytest.MonkeyPatch):+ @pytest.mark.parametrize("ownership", ["canonical", "parent", "legacy"])+ @pytest.mark.parametrize("node_id", ["outer", "inner", "deep"])+ def test_single_container_debugging_preserves_descendants(self, ownership: str, node_id: str):runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)- graph_config = {- "nodes": [{"id": "node-1", "data": {"type": "start", "version": "1"}}],- "edges": [],- }+ graph_config = _nested_container_graph(ownership)workflow = _workflow(graph_config)workflow.id = "workflow"-- monkeypatch.setattr(- "core.app.apps.workflow_app_runner.Graph.init",- lambda **kwargs: SimpleNamespace(),- )-- class _NodeCls:- @staticmethod- def extract_variable_selector_to_variable_mapping(graph_config, config):- return {}-- from core.app.apps import workflow_app_runner-- monkeypatch.setattr(- workflow_app_runner,- "resolve_workflow_node_class",- lambda **_kwargs: _NodeCls,- )- monkeypatch.setattr(- "core.app.apps.workflow_app_runner.load_into_variable_pool",- lambda **kwargs: None,- )- monkeypatch.setattr(- "core.app.apps.workflow_app_runner.WorkflowEntry.mapping_user_inputs_to_variable_pool",- lambda **kwargs: None,- )+ graph_runtime_state.variable_pool.add(["start", "items"], ["first", "second"])graph, variable_pool = runner._get_graph_and_variable_pool_for_single_node_run(workflow=workflow,- node_id="node-1",+ node_id=node_id,user_inputs={},graph_runtime_state=graph_runtime_state,- node_type_filter_key="iteration_id",node_type_label="iteration",user_id="00000000-0000-0000-0000-000000000001",)- assert graph is not None+ assert set(graph.nodes) == {node_id}assert variable_pool is graph_runtime_state.variable_pool+ deep_graph = Graph.init(+ graph_config=graph.graph_config,+ node_factory=graph.node_factory,+ root_node_id="deep-start",+ container_id="deep",+ )+ assert set(deep_graph.nodes) == {"deep-start"}def test_get_graph_and_variable_pool_for_single_node_run_does_not_mutate_inputs(self, monkeypatch: pytest.MonkeyPatch):runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)@@ -246,7 +432,6 @@ class TestWorkflowBasedAppRunner:node_id="node-1",user_inputs=user_inputs,graph_runtime_state=graph_runtime_state,- node_type_filter_key="iteration_id",node_type_label="iteration",user_id="00000000-0000-0000-0000-000000000001",)@@ -263,7 +448,8 @@ class TestWorkflowBasedAppRunner:self, monkeypatch: pytest.MonkeyPatch):runner = WorkflowBasedAppRunner(queue_manager=SimpleNamespace(), app_id="app")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)@@ -277,6 +463,7 @@ class TestWorkflowBasedAppRunner:def fake_from_graph_init_context(**kwargs):captured["run_context"] = kwargs["graph_init_context"].run_context+ captured["runtime_state"] = kwargs["runtime_state"]return SimpleNamespace()class _NodeCls:@@ -303,13 +490,13 @@ class TestWorkflowBasedAppRunner:node_id="node-1",user_inputs={},graph_runtime_state=graph_runtime_state,- node_type_filter_key="iteration_id",node_type_label="iteration",user_id="00000000-0000-0000-0000-000000000001",trace_session_id="session-1",)assert captured["run_context"][DIFY_RUN_CONTEXT_KEY].trace_session_id == "session-1"+ assert captured["runtime_state"] is graph_runtime_statedef test_get_graph_and_variable_pool_preloads_constructor_variables_before_graph_init(self, monkeypatch: pytest.MonkeyPatch@@ -332,7 +519,8 @@ class TestWorkflowBasedAppRunner:variable_loader=variable_loader,app_id="app",)- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)@@ -388,7 +576,6 @@ class TestWorkflowBasedAppRunner:node_id="loop-node",user_inputs={},graph_runtime_state=graph_runtime_state,- node_type_filter_key="loop_id",node_type_label="loop",)@@ -403,11 +590,12 @@ class TestWorkflowBasedAppRunner:published.append((event, publish_from))runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)- workflow_entry = SimpleNamespace(graph_engine=SimpleNamespace(graph_runtime_state=graph_runtime_state))+ workflow_entry = _workflow_entry(runner, graph_runtime_state)emails: list[dict] = []@@ -419,16 +607,11 @@ class TestWorkflowBasedAppRunner:"core.app.apps.workflow_app_runner.dispatch_human_input_email_task",_Dispatch(),)- monkeypatch.setattr(- "core.app.apps.workflow_app_runner.enrich_graph_pause_reasons",- lambda **_: [- HumanInputRequired(- form_id="form",- form_content="content",- node_id="node-1",- node_title="Node",- )- ],+ enriched_reason = HumanInputRequired(+ form_id="form",+ form_content="content",+ node_id="node-1",+ node_title="Node",)reason = HitlRequired(@@ -437,9 +620,11 @@ class TestWorkflowBasedAppRunner:node_title="Node",)- runner._handle_event(workflow_entry, GraphRunStartedEvent())- runner._handle_event(workflow_entry, GraphRunSucceededEvent(outputs={"ok": True}))- runner._handle_event(workflow_entry, GraphRunPausedEvent(reasons=[reason], outputs={}))+ runner.handle_event(workflow_entry, GraphRunStartedEvent())+ runner.handle_event(workflow_entry, GraphRunSucceededEvent(outputs={"ok": True}))+ runner.handle_event(+ workflow_entry, GraphRunPausedEvent(reasons=[reason], outputs={}), pause_reasons=[enriched_reason]+ )assert any(isinstance(event, QueueWorkflowStartedEvent) for event, _ in published)assert any(isinstance(event, QueueWorkflowSucceededEvent) for event, _ in published)@@ -456,9 +641,9 @@ class TestWorkflowBasedAppRunner:published.append(event)runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")- workflow_entry = SimpleNamespace()+ workflow_entry = _workflow_entry(runner)- runner._handle_event(workflow_entry, GraphRunAbortedEvent(reason="User requested stop", outputs={}))+ runner.handle_event(workflow_entry, GraphRunAbortedEvent(reason="User requested stop", outputs={}))event = published[-1]assert isinstance(event, QueueStopEvent)@@ -472,13 +657,14 @@ class TestWorkflowBasedAppRunner:published.append(event)runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)- workflow_entry = SimpleNamespace(graph_engine=SimpleNamespace(graph_runtime_state=graph_runtime_state))+ workflow_entry = _workflow_entry(runner, graph_runtime_state)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunStartedEvent(id="exec",@@ -488,7 +674,7 @@ class TestWorkflowBasedAppRunner:start_at=datetime.now(UTC),),)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunStreamChunkEvent(id="exec",@@ -499,7 +685,7 @@ class TestWorkflowBasedAppRunner:is_final=False,),)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunReasoningChunkEvent(id="exec",@@ -510,7 +696,7 @@ class TestWorkflowBasedAppRunner:is_final=False,),)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunAgentLogEvent(id="exec",@@ -526,7 +712,7 @@ class TestWorkflowBasedAppRunner:metadata={},),)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunIterationSucceededEvent(id="exec",@@ -540,7 +726,7 @@ class TestWorkflowBasedAppRunner:steps=1,),)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunLoopFailedEvent(id="exec",@@ -562,41 +748,71 @@ class TestWorkflowBasedAppRunner:assert any(isinstance(event, QueueIterationCompletedEvent) for event in published)assert any(isinstance(event, QueueLoopCompletedEvent) for event in published)- def test_handle_human_input_form_filled_event_preserves_submitted_data(self):+ @pytest.mark.parametrize("ownership", ["canonical", "parent", "legacy"])+ def test_nested_engine_events_retain_both_container_ancestors(self, ownership: str):+ from tests.unit_tests.core.app.workflow.test_persistence_layer import _make_layer+published: list[object] = []class _QueueManager:def publish(self, event, publish_from):+ del publish_frompublished.append(event)- runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")- graph_runtime_state = GraphRuntimeState(- variable_pool=VariablePool.from_bootstrap(- system_variables=default_system_variables(),- user_inputs={},- environment_variables=[],- ),- start_at=0.0,+ graph_config = _nested_container_graph(ownership)+ graph_config["nodes"].append(+ {"id": "deep-end", "data": {"type": "end", "title": "Nested End", "outputs": [], "container_id": "deep"}})- workflow_entry = SimpleNamespace(graph_engine=SimpleNamespace(graph_runtime_state=graph_runtime_state))+ graph_config["edges"].append({"id": "deep-output", "source": "deep-start", "target": "deep-end"})+ layer, _, node_repo, state = _make_layer(graph_data=graph_config)+ persisted_starts: dict[str, dict] = {}- runner._handle_event(- workflow_entry,- NodeRunHumanInputFormFilledEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.HUMAN_INPUT,- node_title="Human Input",- rendered_content="content",- action_id="approve",- action_text="Approve",- submitted_data={"decision": StringSegment(value="approve")},- ),- )+ class _StartMetadataLayer(Layer):+ @override+ def on_event(self, event):+ if isinstance(event, NodeRunStartedEvent):+ persisted_starts[event.node_id] = dict(node_repo.saved[-1].metadata)- queue_event = published[-1]- assert isinstance(queue_event, QueueHumanInputFormFilledEvent)- assert queue_event.submitted_data == {"decision": StringSegment(value="approve")}+ state.variable_pool.add(["start", "items"], ["item"])+ runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")+ graph = runner._init_graph(+ graph_config=graph_config,+ graph_runtime_state=state,+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ call_depth=0,+ )+ entry = WorkflowEntry(graph_config=graph.graph_config, graph=graph, call_depth=0, graph_runtime_state=state)+ engine = entry.graph_engine+ index = WorkflowRunIndex(())+ layer.set_node_run_indices(index.indices)+ engine.add_layer(index)+ engine.add_layer(layer)+ engine.add_layer(_StartMetadataLayer())+ for event in iter_dify_graph_engine_events(engine):+ runner.handle_event(entry, event)++ expected = {+ "outer": (None, None),+ "inner": (None, "outer"),+ "deep": ("inner", "outer"),+ "deep-end": ("inner", "deep"),+ }+ started = {event.node_id: event for event in published if isinstance(event, QueueNodeStartedEvent)}+ persisted = {event.node_id: event for event in node_repo.saved}+ for node_id, (iteration_id, loop_id) in expected.items():+ assert (started[node_id].in_iteration_id, started[node_id].in_loop_id) == (iteration_id, loop_id)+ for metadata in (persisted_starts[node_id], persisted[node_id].metadata):+ assert (metadata.get("iteration_id"), metadata.get("loop_id")) == (iteration_id, loop_id)+ for metadata in (persisted_starts["deep-end"], persisted["deep-end"].metadata):+ assert metadata["iteration_index"] == 0+ assert metadata["loop_index"] == 0+ for event in published:+ if isinstance(+ event,+ QueueIterationStartEvent | QueueIterationCompletedEvent | QueueLoopStartEvent | QueueLoopCompletedEvent,+ ):+ assert (event.metadata.get("iteration_id"), event.metadata.get("loop_id")) == expected[event.node_id]def test_handle_nested_answer_success_publishes_text_chunk(self):published: list[object] = []@@ -606,13 +822,9 @@ class TestWorkflowBasedAppRunner:published.append(event)runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")- graph_runtime_state = GraphRuntimeState(- variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),- start_at=0.0,- )- workflow_entry = SimpleNamespace(graph_engine=SimpleNamespace(graph_runtime_state=graph_runtime_state))+ workflow_entry = _workflow_entry(runner)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunSucceededEvent(id="exec",@@ -620,8 +832,10 @@ class TestWorkflowBasedAppRunner:node_type=BuiltinNodeTypes.ANSWER,start_at=datetime.now(UTC),finished_at=datetime.now(UTC),- node_run_result=NodeRunResult(outputs={"answer": "inside iteration"}),- in_iteration_id="iteration",+ node_run_result=NodeRunResult(+ outputs={"answer": "inside iteration"}, metadata={"iteration_id": "iteration"}+ ),+ container_id="iteration",),)@@ -640,13 +854,9 @@ class TestWorkflowBasedAppRunner:published.append(event)runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")- graph_runtime_state = GraphRuntimeState(- variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),- start_at=0.0,- )- workflow_entry = SimpleNamespace(graph_engine=SimpleNamespace(graph_runtime_state=graph_runtime_state))+ workflow_entry = _workflow_entry(runner)- runner._handle_event(+ runner.handle_event(workflow_entry,NodeRunSucceededEvent(id="exec",@@ -723,11 +933,12 @@ class TestWorkflowBasedAppRunner:published.append(event)runner = WorkflowBasedAppRunner(queue_manager=_QueueManager(), app_id="app")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=default_system_variables()),start_at=0.0,)- workflow_entry = SimpleNamespace(graph_engine=SimpleNamespace(graph_runtime_state=graph_runtime_state))+ workflow_entry = _workflow_entry(runner, graph_runtime_state)started_at = datetime.now(UTC)finished_at = datetime.now(UTC)result = NodeRunResult(@@ -740,7 +951,7 @@ class TestWorkflowBasedAppRunner:},)- runner._handle_event(workflow_entry, event_factory(result, started_at, finished_at))+ runner.handle_event(workflow_entry, event_factory(result, started_at, finished_at))queue_event = published[-1]assert isinstance(queue_event, queue_event_cls)diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_notifications.py b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_notifications.pyindex f7f9bdb4ba2..d0437d0fa22 100644--- a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_notifications.py+++ b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_notifications.py@@ -5,8 +5,8 @@ import pytestfrom core.app.apps.workflow_app_runner import WorkflowBasedAppRunnerfrom core.app.entities.queue_entities import QueueWorkflowPausedEventfrom core.workflow.nodes.human_input.pause_reason import HumanInputRequired+from graphon.engine_events import GraphRunPausedEventfrom graphon.entities.pause_reason import HitlRequired-from graphon.graph_events import GraphRunPausedEventclass _DummyQueueManager:@@ -23,7 +23,7 @@ class _DummyRuntimeState:class _DummyGraphEngine:def __init__(self):- self.graph_runtime_state = _DummyRuntimeState()+ self.runtime_state = _DummyRuntimeState()class _DummyWorkflowEntry:@@ -48,13 +48,9 @@ def test_handle_pause_event_enqueues_email_task(monkeypatch: pytest.MonkeyPatch)node_id="node-1",node_title="Review",)- monkeypatch.setattr(- "core.app.apps.workflow_app_runner.enrich_graph_pause_reasons",- lambda **_: [enriched_reason],- )monkeypatch.setattr("core.app.apps.workflow_app_runner.dispatch_human_input_email_task", email_task)- runner._handle_event(workflow_entry, event)+ runner.handle_event(workflow_entry, event, pause_reasons=[enriched_reason])email_task.apply_async.assert_called_once()kwargs = email_task.apply_async.call_args.kwargs["kwargs"]diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_single_node.py b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_single_node.pyindex ce0018b8b4f..5da553ba7ed 100644--- a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_single_node.py+++ b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner_single_node.py@@ -13,7 +13,7 @@ from core.app.apps.workflow_app_runner import WorkflowBasedAppRunnerfrom core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntityfrom core.credit_usage import CreditUsageAppTypefrom core.workflow.system_variables import default_system_variables-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom models.model import AppModefrom models.workflow import Workflow, WorkflowKind@@ -25,7 +25,11 @@ def _make_graph_state():environment_variables=[],conversation_variables=[],)- return MagicMock(), variable_pool, GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)+ return (+ MagicMock(),+ variable_pool,+ RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0),+ )@pytest.fixture@@ -79,8 +83,6 @@ def test_run_uses_single_node_execution_branch(variable_loader=MagicMock(),workflow=workflow,system_user_id="system-user",- workflow_execution_repository=MagicMock(),- workflow_node_execution_repository=MagicMock(),)graph, variable_pool, graph_runtime_state = _make_graph_state()@@ -104,7 +106,7 @@ def test_run_uses_single_node_execution_branch() as prepare_single,patch.object(runner, "_init_graph") as init_graph,):- runner.run()+ runner.prepare()prepare_single.assert_called_once_with(workflow=workflow,@@ -116,10 +118,7 @@ def test_run_uses_single_node_execution_branch()init_graph.assert_not_called()- entry_kwargs = entry_class.call_args.kwargs- assert entry_kwargs["invoke_from"] == InvokeFrom.DEBUGGER- assert entry_kwargs["variable_pool"] is variable_pool- assert entry_kwargs["graph_runtime_state"] is graph_runtime_state+ assert entry_class.call_args.kwargs["graph_runtime_state"] is graph_runtime_statedef test_single_node_run_validates_target_node_config(queue_manager: WorkflowAppQueueManager) -> None:@@ -153,7 +152,51 @@ def test_single_node_run_validates_target_node_config(queue_manager: WorkflowAppnode_id="loop-node",user_inputs={},graph_runtime_state=graph_runtime_state,- node_type_filter_key="loop_id",+ node_type_label="loop",+ user_id="00000000-0000-0000-0000-000000000001",+ )+++def test_single_node_run_rejects_invalid_loop_count(queue_manager: WorkflowAppQueueManager) -> None:+ runner = WorkflowBasedAppRunner(+ queue_manager=queue_manager,+ app_id="app",+ )++ workflow = Workflow(+ id="workflow",+ tenant_id="tenant",+ graph=json.dumps(+ {+ "nodes": [+ {+ "id": "loop-node",+ "data": {+ "type": "loop",+ "title": "Loop",+ "loop_count": 0,+ "start_node_id": "loop-start",+ "break_conditions": [],+ "logical_operator": "and",+ },+ },+ {+ "id": "loop-start",+ "data": {"type": "loop-start", "title": "Loop start", "container_id": "loop-node"},+ },+ ],+ "edges": [],+ }+ ),+ )++ _, _, graph_runtime_state = _make_graph_state()+ with pytest.raises(ValidationError, match="loop_count"):+ runner._get_graph_and_variable_pool_for_single_node_run(+ workflow=workflow,+ node_id="loop-node",+ user_inputs={},+ graph_runtime_state=graph_runtime_state,node_type_label="loop",user_id="00000000-0000-0000-0000-000000000001",)@@ -171,6 +214,7 @@ def test_run_adds_inputs_with_snippet_compatible_start_aliases(queue_manager: Woworkflow_execution_id="execution-id",task_id="task-id",stream=True,+ call_depth=4,)workflow = Workflow(@@ -190,8 +234,6 @@ def test_run_adds_inputs_with_snippet_compatible_start_aliases(queue_manager: Wovariable_loader=MagicMock(),workflow=workflow,system_user_id="system-user",- workflow_execution_repository=MagicMock(),- workflow_node_execution_repository=MagicMock(),)mock_workflow_entry = MagicMock()@@ -211,12 +253,13 @@ def test_run_adds_inputs_with_snippet_compatible_start_aliases(queue_manager: Wo"core.app.apps.workflow.app_runner.get_compatible_start_aliases", return_value=("legacy-start",)) as aliases,patch("core.app.apps.workflow.app_runner.add_node_inputs_to_pool") as add_inputs,- patch.object(runner, "_init_graph", return_value=MagicMock()),+ patch.object(runner, "_init_graph", return_value=MagicMock()) as init_graph,):- runner.run()+ runner.prepare()aliases.assert_called_once_with(workflow_kind="snippet", root_node_id="root-node")add_inputs.assert_called_once()assert add_inputs.call_args.kwargs["node_id"] == "root-node"assert add_inputs.call_args.kwargs["inputs"] == {"question": "hello"}assert add_inputs.call_args.kwargs["aliases"] == ("legacy-start",)+ assert init_graph.call_args.kwargs["call_depth"] == 4diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_human_input_completion.py b/api/tests/unit_tests/core/app/apps/test_workflow_human_input_completion.pynew file mode 100644index 00000000000..ea58fb2a358--- /dev/null+++ b/api/tests/unit_tests/core/app/apps/test_workflow_human_input_completion.py@@ -0,0 +1,461 @@+import json+from collections.abc import Generator, Mapping+from datetime import UTC, datetime, timedelta+from time import perf_counter+from unittest.mock import MagicMock, PropertyMock+from urllib.parse import parse_qs, urlparse++import pytest+from pytest_mock import MockerFixture+from sqlalchemy import select+from sqlalchemy.orm import Session++from configs import dify_config+from core.app.app_config.entities import WorkflowUIBasedAppConfig+from core.app.apps.base_app_queue_manager import PublishFrom+from core.app.apps.common import workflow_response_converter+from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager+from core.app.apps.workflow_app_runner import PreparedWorkflowRun, WorkflowBasedAppRunner+from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity+from core.app.entities.queue_entities import (+ AppQueueEvent,+ QueueHumanInputFormFilledEvent,+ QueueHumanInputFormTimeoutEvent,+ QueueNodeStartedEvent,+ QueueWorkflowStartedEvent,+ QueueWorkflowSucceededEvent,+)+from core.db.session_factory import session_factory+from core.repositories.human_input_repository import HumanInputFormRepositoryImpl, HumanInputFormSubmissionRepository+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from core.workflow.node_runtime import DifyFileReferenceFactory+from core.workflow.nodes.human_input.callback import DifyHITLCallback+from core.workflow.nodes.human_input.entities import (+ FileInputConfig,+ FileListInputConfig,+ FormDefinition,+ HumanInputNodeData,+ ParagraphInputConfig,+ UserActionConfig,+)+from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus+from core.workflow.workflow_entry import WorkflowEntry+from extensions.storage.storage_type import StorageType+from graphon.engine.ready_queue import StartTask+from graphon.engine_events import EngineEvent, GraphRunFailedEvent+from graphon.entities import WorkflowStartReason+from graphon.entities.pause_reason import HitlRequired+from graphon.enums import BuiltinNodeTypes, WorkflowType+from graphon.file.helpers import verify_file_signature+from graphon.graph import Graph+from graphon.nodes.human_input.entities import HumanInputNodeData as GraphonHumanInputNodeData+from graphon.nodes.human_input.human_input_node import HumanInputNode+from graphon.runtime import RuntimeState, VariablePool+from graphon.runtime.execution import ROOT_FRAME_ID+from models import Account, UploadFile+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.execution_extra_content import HumanInputContent+from models.human_input import HumanInputForm+from models.model import AppMode+from models.workflow import WorkflowNodeExecutionTriggeredFrom+from services.workflow_run_agg import WorkflowRunAgg+from tests.unit_tests.core.app.apps.advanced_chat.test_generate_task_pipeline import (+ _build_generate_entity,+ _build_pipeline,+)+from tests.unit_tests.core.app.apps.common.test_workflow_response_converter_human_input import _build_converter+from tests.workflow_test_utils import build_test_graph_init_params, build_test_run_context+++def workflow_queue(mocker: MockerFixture) -> tuple[WorkflowAppQueueManager, MagicMock]:+ queue = WorkflowAppQueueManager("task", "user", InvokeFrom.DEBUGGER, AppMode.WORKFLOW)+ return queue, mocker.spy(queue, "publish")+++def _save_form(session: Session, *, status: HumanInputFormStatus) -> HumanInputForm:+ now = datetime.now(UTC)+ definition = FormDefinition(+ form_content="Decision: {{#$output.answer#}}",+ rendered_content="Decision: {{#$output.answer#}}",+ inputs=[ParagraphInputConfig(output_variable_name="answer")],+ user_actions=[UserActionConfig(id="approve", title="Approve")],+ expiration_time=now + timedelta(hours=1),+ node_title="Approval",+ )+ form = HumanInputForm(+ tenant_id="tenant",+ app_id="app",+ workflow_run_id="run-1",+ node_id="same-human-node",+ form_kind=HumanInputFormKind.RUNTIME,+ form_definition=definition.model_dump_json(),+ rendered_content=definition.rendered_content,+ expiration_time=definition.expiration_time,+ status=status,+ selected_action_id="approve" if status == HumanInputFormStatus.SUBMITTED else None,+ submitted_data=json.dumps({"answer": "approved"}) if status == HumanInputFormStatus.SUBMITTED else None,+ submitted_at=now if status == HumanInputFormStatus.SUBMITTED else None,+ created_at=now,+ )+ session.add(form)+ session.commit()+ return form+++def _make_paused_workflow(+ runner: WorkflowBasedAppRunner, forms: list[HumanInputForm], *, human_input_form: HumanInputForm | None = None+) -> WorkflowEntry:+ state = RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=perf_counter())+ state.variable_pool.add(("sys", "workflow_run_id"), "run-1")+ state.variable_pool.add(("sys", "app_id"), "app")+ graph = runner._init_graph(+ graph_config={+ "nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}],+ "edges": [],+ },+ graph_runtime_state=state,+ tenant_id="tenant",+ user_id="user",+ user_from=UserFrom.END_USER,+ invoke_from=InvokeFrom.WEB_APP,+ call_depth=0,+ )+ if human_input_form is not None:+ node_data = HumanInputNodeData(+ title="Approval",+ form_content="Decision: {{#$output.answer#}}",+ inputs=[ParagraphInputConfig(output_variable_name="answer")],+ user_actions=[UserActionConfig(id="approve", title="Approve")],+ )+ callback = DifyHITLCallback(+ form_repository=HumanInputFormRepositoryImpl(+ tenant_id="tenant", app_id="app", workflow_execution_id="run-1"+ ),+ node_data=node_data,+ execution_id_getter=lambda: human_input_form.id,+ )+ graph = (+ Graph.new()+ .add_root(graph.root_node)+ .add_node(+ HumanInputNode(+ node_id=human_input_form.node_id,+ data=GraphonHumanInputNodeData.model_validate(node_data.model_dump()),+ init_params=build_test_graph_init_params(tenant_id="tenant", app_id="app"),+ runtime_state=state,+ hitl_callback=callback,+ ),+ from_node_id=graph.root_node.id,+ )+ .build()+ )+ # A resumed Human Input node keeps the execution identity used to create its form.+ state.graph_execution.get_or_create_node_execution(+ frame_id=ROOT_FRAME_ID, node_id=human_input_form.node_id+ ).execution_id = human_input_form.id+ state.graph_execution.start()+ for form in forms:+ state.graph_execution.pause(HitlRequired(session_id=form.id, node_id=form.node_id, node_title="Approval"))+ state.defer_ready_task(StartTask(frame_id=ROOT_FRAME_ID, node_id=graph.root_node.id))+ return WorkflowEntry(+ graph=graph,+ graph_config=graph.graph_config or {},+ call_depth=0,+ graph_runtime_state=state,+ )+++def _resume_events(runner: WorkflowBasedAppRunner, entry: WorkflowEntry) -> Generator[EngineEvent]:+ """Resume through orchestration with actual repositories and the real Engine."""+ entity = WorkflowAppGenerateEntity(+ task_id="task",+ app_config=WorkflowUIBasedAppConfig(+ tenant_id="tenant",+ app_id="app",+ app_mode=AppMode.WORKFLOW,+ workflow_id="workflow",+ ),+ inputs={},+ files=[],+ user_id="user",+ stream=True,+ invoke_from=InvokeFrom.DEBUGGER,+ workflow_execution_id="run-1",+ )+ user = Account(name="Tester", email="[email protected]")+ user.id = "user"+ factory = session_factory.get_session_maker()+ runs = SQLAlchemyWorkflowExecutionRepository(+ factory,+ tenant_id="tenant",+ app_id="app",+ user=user,+ triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,+ )+ nodes = SQLAlchemyWorkflowNodeExecutionRepository(+ factory,+ tenant_id="tenant",+ app_id="app",+ user=user,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ )+ prepared = PreparedWorkflowRun(entry, WorkflowType.WORKFLOW, "draft", entity, ())+ return WorkflowRunAgg(+ prepared,+ runner,+ None,+ workflow_execution_repository=runs,+ workflow_node_execution_repository=nodes,+ ).iter_events()++[email protected]("status", [HumanInputFormStatus.SUBMITTED, HumanInputFormStatus.TIMEOUT])[email protected]("runs_human_input_node", [False, True], ids=["hidden-form", "direct-completion"])+def test_resume_publishes_only_the_completed_form_among_repeated_node_invocations(+ sqlite_session: Session,+ workflow_queue: tuple[WorkflowAppQueueManager, MagicMock],+ status: HumanInputFormStatus,+ runs_human_input_node: bool,+) -> None:+ selected = _save_form(sqlite_session, status=status)+ waiting = _save_form(sqlite_session, status=HumanInputFormStatus.WAITING)+ expired = _save_form(sqlite_session, status=HumanInputFormStatus.EXPIRED)+ _save_form(sqlite_session, status=HumanInputFormStatus.SUBMITTED)+ queue, publish_spy = workflow_queue+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")+ entry = _make_paused_workflow(+ runner, [selected, waiting, selected, expired], human_input_form=selected if runs_human_input_node else None+ )+ original_reasons = tuple(entry.graph_engine.runtime_state.graph_execution.pause_reasons)++ for event in _resume_events(runner, entry):+ runner.handle_event(entry, event)++ published = [call.args[0] for call in publish_spy.call_args_list]+ assert isinstance(published[0], QueueWorkflowStartedEvent)+ completions = [+ item for item in published if isinstance(item, QueueHumanInputFormFilledEvent | QueueHumanInputFormTimeoutEvent)+ ]+ assert len(completions) == 1+ completion = completions[0]+ assert completion.form_id == selected.id+ assert completion.node_id == "same-human-node"+ if status == HumanInputFormStatus.SUBMITTED:+ assert isinstance(completion, QueueHumanInputFormFilledEvent)+ assert (completion.action_id, completion.action_text, completion.rendered_content) == (+ "approve",+ "Approve",+ "Decision: approved",+ )+ assert completion.submitted_data is not None+ assert completion.submitted_data["answer"].to_object() == "approved"+ else:+ assert isinstance(completion, QueueHumanInputFormTimeoutEvent)+ assert len(original_reasons) == 4+ assert all(isinstance(reason, HitlRequired) and reason.node_id == "same-human-node" for reason in original_reasons)+ assert entry.graph_engine.runtime_state.graph_execution.pause_reasons == []++[email protected]("owner_field", ["tenant_id", "app_id", "workflow_run_id"])+def test_resume_rejects_form_outside_the_trusted_execution_owner(+ sqlite_session: Session, workflow_queue: tuple[WorkflowAppQueueManager, MagicMock], owner_field: str+) -> None:+ form = _save_form(sqlite_session, status=HumanInputFormStatus.SUBMITTED)+ setattr(form, owner_field, "another-owner")+ sqlite_session.commit()+ queue, publish_spy = workflow_queue+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")+ entry = _make_paused_workflow(runner, [form])++ events = list(_resume_events(runner, entry))+ assert isinstance(events[-1], GraphRunFailedEvent)+ assert "does not belong" in events[-1].error++ assert not any(isinstance(call.args[0], QueueHumanInputFormFilledEvent) for call in publish_spy.call_args_list)+++def test_resume_refreshes_expired_file_and_file_list_urls_before_form_completion(+ sqlite_session: Session,+ workflow_queue: tuple[WorkflowAppQueueManager, MagicMock],+ monkeypatch: pytest.MonkeyPatch,+ mocker: MockerFixture,+) -> None:+ mocker.patch.object(+ type(workflow_response_converter.db),+ "engine",+ new_callable=PropertyMock,+ return_value=sqlite_session.get_bind(),+ )+ submitted_at = 1_700_000_000+ monkeypatch.setattr("core.app.workflow.file_runtime.time.time", lambda: submitted_at)+ factory = DifyFileReferenceFactory(build_test_run_context())+ saved_files = []+ for name in ("first.txt", "second.txt"):+ upload = UploadFile(+ tenant_id="tenant",+ storage_type=StorageType.LOCAL,+ key=name,+ name=name,+ size=12,+ extension="txt",+ mime_type="text/plain",+ created_by="user",+ created_by_role=CreatorUserRole.END_USER,+ created_at=datetime.now(UTC),+ used=False,+ )+ sqlite_session.add(upload)+ sqlite_session.commit()+ saved_files.append(+ factory.build_from_mapping(mapping={"transfer_method": "local_file", "upload_file_id": upload.id}).to_dict()+ )++ def url_is_valid(file: Mapping[str, object]) -> bool:+ url, file_id = file["url"], file["related_id"]+ assert isinstance(url, str)+ assert isinstance(file_id, str)+ query = parse_qs(urlparse(url).query)+ return verify_file_signature(+ upload_file_id=file_id,+ timestamp=query["timestamp"][0],+ nonce=query["nonce"][0],+ sign=query["sign"][0],+ )++ assert all(url_is_valid(file) for file in saved_files)+ form = _save_form(sqlite_session, status=HumanInputFormStatus.SUBMITTED)+ definition = FormDefinition.model_validate_json(form.form_definition)+ definition.inputs.extend(+ [FileInputConfig(output_variable_name="attachment"), FileListInputConfig(output_variable_name="attachments")]+ )+ form.form_definition = definition.model_dump_json()+ form.submitted_data = json.dumps({"answer": "approved", "attachment": saved_files[0], "attachments": saved_files})+ sqlite_session.commit()++ monkeypatch.setattr(+ "core.app.workflow.file_runtime.time.time", lambda: submitted_at + dify_config.FILES_ACCESS_TIMEOUT + 1+ )+ assert not any(url_is_valid(file) for file in saved_files)+ queue, publish_spy = workflow_queue+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")+ entry = _make_paused_workflow(runner, [form])+ for event in _resume_events(runner, entry):+ runner.handle_event(entry, event)++ completions = [+ call.args[0] for call in publish_spy.call_args_list if isinstance(call.args[0], QueueHumanInputFormFilledEvent)+ ]+ assert len(completions) == 1+ converter = _build_converter()+ converter.workflow_start_to_stream_response(+ task_id="task", workflow_run_id="run-1", workflow_id="workflow", reason=WorkflowStartReason.RESUMPTION+ )+ response = converter.human_input_form_filled_to_stream_response(event=completions[0], task_id="task")+ data = response.data.submitted_data+ assert data is not None+ assert data["answer"] == "approved"+ assert response.data.rendered_content == "Decision: approved"+ restored_files = [data["attachment"], *data["attachments"]]+ for restored, saved in zip(restored_files, [saved_files[0], *saved_files], strict=True):+ assert (restored["related_id"], restored["filename"]) == (saved["related_id"], saved["filename"])+ assert url_is_valid(restored)++[email protected]("late_status", [HumanInputFormStatus.SUBMITTED, HumanInputFormStatus.TIMEOUT])+def test_form_completed_during_resume_is_published_once_before_the_terminal_event(+ sqlite_session: Session,+ workflow_queue: tuple[WorkflowAppQueueManager, MagicMock],+ late_status: HumanInputFormStatus,+) -> None:+ first = _save_form(sqlite_session, status=HumanInputFormStatus.SUBMITTED)+ second = _save_form(sqlite_session, status=HumanInputFormStatus.WAITING)+ queue, publish_spy = workflow_queue+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")+ entry = _make_paused_workflow(runner, [first, second])++ def complete_second_form_during_execution(event: AppQueueEvent, publish_from: PublishFrom) -> None:+ WorkflowAppQueueManager.publish(queue, event, publish_from)+ # The second submission arrives after the initial completion query,+ # while this real Engine attempt is already processing node events.+ if not isinstance(event, QueueNodeStartedEvent):+ return+ repository = HumanInputFormSubmissionRepository()+ if late_status == HumanInputFormStatus.SUBMITTED:+ repository.mark_submitted(+ form_id=second.id,+ recipient_id=None,+ selected_action_id="approve",+ form_data={"answer": "approved"},+ submission_user_id=None,+ submission_end_user_id="user",+ )+ else:+ repository.mark_timeout(form_id=second.id, timeout_status=late_status)++ publish_spy.side_effect = complete_second_form_during_execution+ for event in _resume_events(runner, entry):+ runner.handle_event(entry, event)++ published = [call.args[0] for call in publish_spy.call_args_list]+ completions = [+ event+ for event in published+ if isinstance(event, QueueHumanInputFormFilledEvent | QueueHumanInputFormTimeoutEvent)+ ]+ assert [event.form_id for event in completions] == [first.id, second.id]+ assert isinstance(published[0], QueueWorkflowStartedEvent)+ assert isinstance(published[-1], QueueWorkflowSucceededEvent)+ assert published.index(completions[-1]) < len(published) - 1+++def test_waiting_form_expiring_during_resume_publishes_timeout_before_success(+ sqlite_session: Session, workflow_queue: tuple[WorkflowAppQueueManager, MagicMock]+) -> None:+ form = _save_form(sqlite_session, status=HumanInputFormStatus.WAITING)+ form.expiration_time = datetime.now(UTC) - timedelta(seconds=1)+ sqlite_session.commit()+ queue, publish_spy = workflow_queue+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")+ entry = _make_paused_workflow(runner, [form], human_input_form=form)++ for event in _resume_events(runner, entry):+ runner.handle_event(entry, event)++ published = [call.args[0] for call in publish_spy.call_args_list]+ assert isinstance(published[-1], QueueWorkflowSucceededEvent)+ timeouts = [event for event in published if isinstance(event, QueueHumanInputFormTimeoutEvent)]+ assert [event.form_id for event in timeouts] == [form.id]+ assert timeouts[0].node_id == "same-human-node"+ assert published.index(timeouts[0]) < len(published) - 1+ sqlite_session.refresh(form)+ assert form.status == HumanInputFormStatus.TIMEOUT+++def test_chat_history_keeps_repeated_forms_by_form_id(sqlite_session: Session) -> None:+ pipeline = _build_pipeline()+ pipeline._workflow_tenant_id = "tenant"+ pipeline._application_generate_entity = _build_generate_entity()+ pipeline._workflow_response_converter = _build_converter()+ pipeline._workflow_response_converter.workflow_start_to_stream_response(+ task_id="task-1", workflow_run_id="run-1", workflow_id="workflow-1", reason=WorkflowStartReason.INITIAL+ )+ for form_id in ("first-form", "second-form", "first-form"):+ event = QueueHumanInputFormFilledEvent(+ form_id=form_id,+ node_id="same-human-node",+ node_type=BuiltinNodeTypes.HUMAN_INPUT,+ node_title="Approval",+ rendered_content="Decision: approved",+ action_id="approve",+ action_text="Approve",+ )+ list(pipeline._handle_human_input_form_filled_event(event))++ contents = sqlite_session.scalars(select(HumanInputContent)).all()+ assert {(content.form_id, content.message_id) for content in contents} == {+ ("first-form", "message-1"),+ ("second-form", "message-1"),+ }+ assert len(contents) == 2diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py b/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.pyindex 4fd4af28f38..16394e5e7ba 100644--- a/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py+++ b/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py@@ -22,14 +22,15 @@ from core.workflow.nodes.human_input.entities import (from core.workflow.nodes.human_input.enums import ValueSourceTypefrom core.workflow.nodes.human_input.pause_reason import HumanInputRequiredfrom core.workflow.system_variables import build_system_variables+from graphon.engine_events import GraphRunPausedEventfrom graphon.entities import WorkflowStartReasonfrom graphon.entities.pause_reason import HitlRequired-from graphon.graph_events import GraphRunPausedEvent-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom models.account import Accountfrom models.human_input import HumanInputForm, HumanInputFormRecipient, RecipientTypefrom models.model import AppModefrom models.workflow import Workflow, WorkflowType+from tests.unit_tests.core.app.apps.test_workflow_app_runner_core import _workflow_entryclass _RecordingWorkflowAppRunner(WorkflowAppRunner):@@ -41,10 +42,6 @@ class _RecordingWorkflowAppRunner(WorkflowAppRunner):self.published_events.append(event)-class _FakeRuntimeState:- variable_pool = object()--@pytest.fixturedef sqlite_pause_session(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> Session:"""Bind pause-response queries to the shared SQLite session's database."""@@ -117,24 +114,23 @@ def _build_runner():workflow=workflow,system_user_id="sys-user",root_node_id=None,- workflow_execution_repository=MagicMock(),- workflow_node_execution_repository=MagicMock(),graph_engine_layers=(),graph_runtime_state=None,)-def test_graph_run_paused_event_emits_queue_pause_event(monkeypatch: pytest.MonkeyPatch):[email protected]("source_node_id", ["node-human", "nested-human"])+def test_graph_run_paused_event_uses_projected_nodes_for_highlighting(+ monkeypatch: pytest.MonkeyPatch, source_node_id: str+):runner = _build_runner()graph_reason = HitlRequired(session_id="form-1",- node_id="node-human",+ node_id=source_node_id,node_title="Human Step",)event = GraphRunPausedEvent(reasons=[graph_reason], outputs={"foo": "bar"})- workflow_entry = SimpleNamespace(- graph_engine=SimpleNamespace(graph_runtime_state=_FakeRuntimeState()),- )+ workflow_entry = _workflow_entry(runner)enriched_reason = HumanInputRequired(form_id="form-1",@@ -144,13 +140,9 @@ def test_graph_run_paused_event_emits_queue_pause_event(monkeypatch: pytest.Monknode_id="node-human",node_title="Human Step",)- monkeypatch.setattr(- "core.app.apps.workflow_app_runner.enrich_graph_pause_reasons",- lambda **_: [enriched_reason],- )monkeypatch.setattr("core.app.apps.workflow_app_runner.dispatch_human_input_email_task", MagicMock())- runner._handle_event(workflow_entry, event)+ runner.handle_event(workflow_entry, event, pause_reasons=[enriched_reason])assert len(runner.published_events) == 1queue_event = runner.published_events[0]@@ -158,6 +150,7 @@ def test_graph_run_paused_event_emits_queue_pause_event(monkeypatch: pytest.Monkassert queue_event.reasons == [enriched_reason]assert queue_event.outputs == {"foo": "bar"}assert queue_event.paused_nodes == ["node-human"]+ assert graph_reason.node_id == source_node_iddef _build_converter(*, invoke_from: InvokeFrom = InvokeFrom.SERVICE_API) -> WorkflowResponseConverter:@@ -224,7 +217,7 @@ def test_queue_workflow_paused_event_to_stream_responses(sqlite_pause_session: Spaused_nodes=["node-id"],)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)responses = converter.workflow_pause_to_stream_response(event=queue_event,task_id="task",@@ -280,7 +273,7 @@ def _build_paused_human_input_response(paused_nodes=["node-id"],)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)responses = converter.workflow_pause_to_stream_response(event=queue_event,task_id="task",@@ -373,7 +366,7 @@ def test_queue_workflow_paused_event_resolves_variable_select_options(sqlite_paupaused_nodes=["node-id"],)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)runtime_state.variable_pool.add(("start", "options"), ["approve", "reject"])responses = converter.workflow_pause_to_stream_response(event=queue_event,diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_app_generator_extra.py b/api/tests/unit_tests/core/app/apps/workflow/test_app_generator_extra.pyindex 695408e2ccf..98333d15881 100644--- a/api/tests/unit_tests/core/app/apps/workflow/test_app_generator_extra.py+++ b/api/tests/unit_tests/core/app/apps/workflow/test_app_generator_extra.py@@ -19,6 +19,7 @@ from models.enums import EndUserTypefrom models.model import App, AppMode, EndUserfrom models.snippet import CustomizedSnippetfrom models.workflow import Workflow, WorkflowKind, WorkflowType+from services.workflow_run_agg import WorkflowRunAgg@pytest.fixture(autouse=True)@@ -162,7 +163,7 @@ class TestWorkflowAppGeneratorValidation:def test_generate_stream_joins_worker_after_response_exhaustion(self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app = _persist_app(sqlite_session)workflow = _persist_workflow(sqlite_session)user = _persist_end_user(sqlite_session)@@ -274,7 +275,7 @@ class TestWorkflowAppGeneratorValidation:ensure_start_node.assert_called_once_with(workflow, snippet)def test_single_iteration_generate_validates_args(self, sqlite_session: Session):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)with pytest.raises(ValueError, match="node_id is required"):generator.single_iteration_generate(@@ -299,7 +300,7 @@ class TestWorkflowAppGeneratorValidation:)def test_single_loop_generate_validates_args(self, sqlite_session: Session):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)with pytest.raises(ValueError, match="node_id is required"):generator.single_loop_generate(@@ -318,7 +319,7 @@ class TestWorkflowAppGeneratorValidation:monkeypatch: pytest.MonkeyPatch,sqlite_session: Session,):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app = _persist_app(sqlite_session)workflow = _persist_workflow(sqlite_session)user = _persist_end_user(sqlite_session)@@ -378,7 +379,7 @@ class TestWorkflowAppGeneratorValidation:monkeypatch: pytest.MonkeyPatch,sqlite_session: Session,):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app = _persist_app(sqlite_session)workflow = _persist_workflow(sqlite_session)user = _persist_end_user(sqlite_session)@@ -446,7 +447,7 @@ class TestWorkflowAppGeneratorValidation:class TestWorkflowAppGeneratorHandleResponse:def test_handle_response_closed_file_raises_stopped(self, monkeypatch: pytest.MonkeyPatch):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = WorkflowUIBasedAppConfig(tenant_id="tenant",@@ -497,7 +498,7 @@ class TestWorkflowAppGeneratorGenerate:@pytest.fixturedef generation(self, monkeypatch: pytest.MonkeyPatch, sqlite_generator_session: Session):"""Keep input preparation real and stop at the execution boundary."""- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app = _persist_app(sqlite_generator_session)workflow = _persist_workflow(sqlite_generator_session)user = _persist_end_user(sqlite_generator_session)@@ -586,7 +587,7 @@ class TestWorkflowAppGeneratorGenerate:class TestWorkflowAppGeneratorResume:def test_resume_restores_trace_manager_when_missing(self, monkeypatch: pytest.MonkeyPatch):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = WorkflowUIBasedAppConfig(tenant_id="tenant",app_id="app",@@ -648,7 +649,7 @@ class TestWorkflowAppGeneratorResume:assert trace_manager.user_id == "session-id"def test_resume_preserves_existing_trace_manager(self, monkeypatch: pytest.MonkeyPatch):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)app_config = WorkflowUIBasedAppConfig(tenant_id="tenant",app_id="app",@@ -701,7 +702,7 @@ class TestWorkflowAppGeneratorWorker:monkeypatch: pytest.MonkeyPatch,sqlite_session: Session,):- generator = WorkflowAppGenerator()+ generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)_persist_app(sqlite_session)_persist_workflow(sqlite_session)_persist_end_user(sqlite_session)@@ -712,7 +713,7 @@ class TestWorkflowAppGeneratorWorker:def __init__(self, **kwargs):runner_kwargs.update(kwargs)- def run(self):+ def prepare(self):return Nonemonkeypatch.setattr(diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_app_queue_manager.py b/api/tests/unit_tests/core/app/apps/workflow/test_app_queue_manager.pyindex 6d329724fc5..d6b01a8742b 100644--- a/api/tests/unit_tests/core/app/apps/workflow/test_app_queue_manager.py+++ b/api/tests/unit_tests/core/app/apps/workflow/test_app_queue_manager.py@@ -37,7 +37,7 @@ class TestWorkflowAppQueueManager:with (patch("core.app.apps.base_app_queue_manager.redis_client") as queue_redis,patch("core.app.apps.execution_coordinator.redis_client") as execution_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):queue_redis.get.return_value = Nonemanager = WorkflowAppQueueManager(@@ -54,7 +54,7 @@ class TestWorkflowAppQueueManager:assert manager.execution_state is AppExecutionState.RUNNINGexecution_redis.setex.assert_not_called()- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()manager._execution_coordinator.mark_terminal()def test_execution_timeout_aborts_graph_before_stop_event(self, config_overrides: Callable[..., None]):@@ -62,7 +62,7 @@ class TestWorkflowAppQueueManager:with (patch("core.app.apps.base_app_queue_manager.redis_client") as queue_redis,patch("core.app.apps.execution_coordinator.redis_client") as execution_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):queue_redis.get.return_value = Nonemanager = WorkflowAppQueueManager(@@ -77,7 +77,7 @@ class TestWorkflowAppQueueManager:assert any(isinstance(message.event, QueueStopEvent) for message in messages)execution_redis.setex.assert_called_once_with("generate_task_stopped:task", 600, 1)- graph_engine_manager.return_value.send_stop_command.assert_called_once_with(+ send_abort_command.assert_called_once_with("task",reason="App execution exceeded 0 seconds",)@@ -86,7 +86,7 @@ class TestWorkflowAppQueueManager:with (patch("core.app.apps.base_app_queue_manager.redis_client") as queue_redis,patch("core.app.apps.execution_coordinator.redis_client") as execution_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):queue_redis.get.return_value = Nonemanager = WorkflowAppQueueManager(@@ -100,13 +100,13 @@ class TestWorkflowAppQueueManager:_ = list(manager.listen())execution_redis.setex.assert_not_called()- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()def test_pause_completes_listener_without_aborting_resumable_execution(self):with (patch("core.app.apps.base_app_queue_manager.redis_client") as queue_redis,patch("core.app.apps.execution_coordinator.redis_client") as execution_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):queue_redis.get.return_value = Nonemanager = WorkflowAppQueueManager(@@ -126,13 +126,13 @@ class TestWorkflowAppQueueManager:assert isinstance(messages[0].event, QueueWorkflowPausedEvent)assert manager.execution_state is AppExecutionState.PAUSEDexecution_redis.setex.assert_not_called()- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()def test_workflow_pause_does_not_abort_execution(self):with (patch("core.app.apps.base_app_queue_manager.redis_client") as queue_redis,patch("core.app.apps.execution_coordinator.redis_client") as execution_redis,- patch("core.app.apps.execution_coordinator.GraphEngineManager") as graph_engine_manager,+ patch("core.app.apps.execution_coordinator.send_abort_command") as send_abort_command,):queue_redis.get.return_value = Nonemanager = WorkflowAppQueueManager(@@ -149,4 +149,4 @@ class TestWorkflowAppQueueManager:assert manager.execution_state is AppExecutionState.PAUSEDexecution_redis.setex.assert_not_called()- graph_engine_manager.return_value.send_stop_command.assert_not_called()+ send_abort_command.assert_not_called()diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_command_channels.py b/api/tests/unit_tests/core/app/apps/workflow/test_command_channels.pyindex f555fe945c1..9b5e700728a 100644--- a/api/tests/unit_tests/core/app/apps/workflow/test_command_channels.py+++ b/api/tests/unit_tests/core/app/apps/workflow/test_command_channels.py@@ -5,12 +5,13 @@ from unittest.mock import patchimport pytest+from core.app.apps.execution_coordinator import send_abort_commandfrom core.app.apps.workflow.command_channels import (CelerySignalCommandChannel,CombinedCommandChannel,StopFlagCommandChannel,)-from graphon.graph_engine.entities.commands import AbortCommand, PauseCommand+from graphon.engine.command import AbortCommand, PauseCommand, RedisChannelclass _CommandChannelStub:@@ -27,6 +28,29 @@ class _CommandChannelStub:self.sent.append(command)+def test_send_abort_command_uses_workflow_redis_channel() -> None:+ with patch.object(RedisChannel, "send_command", autospec=True) as send:+ send_abort_command("task-id", reason="stop")++ send.assert_called_once()+ channel, command = send.call_args.args+ assert channel._key == "workflow:task-id:commands"+ assert isinstance(command, AbortCommand)+ assert command.reason == "stop"+++def test_send_abort_command_preserves_legacy_fallback(caplog: pytest.LogCaptureFixture) -> None:+ with patch.object(+ RedisChannel, "send_command", autospec=True, side_effect=RuntimeError("redis unavailable")+ ) as send:+ send_abort_command("task-id")+ send_abort_command("")++ send.assert_called_once()+ assert send.call_args.args[1].reason == "User requested stop"+ assert "Failed to send Engine abort command for task task-id" in caplog.text++def test_combined_command_channel_fetches_from_all_sources() -> None:abort = AbortCommand(reason="stop")pause = PauseCommand(reason="pause")@@ -66,7 +90,7 @@ def test_combined_command_channel_continues_after_source_failure(caplog: pytest.combined = CombinedCommandChannel((failing, _CommandChannelStub([abort])))assert combined.fetch_commands() == [abort]- assert "Failed to fetch GraphEngine commands" in caplog.text+ assert "Failed to fetch Engine commands" in caplog.textdef test_celery_signal_command_channel_emits_abort_when_shutdown_starts() -> None:diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline.py b/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline.pyindex 83506ce42bc..06f8bd462a6 100644--- a/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline.py+++ b/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline.py@@ -13,7 +13,7 @@ from core.app.entities.queue_entities import QueueWorkflowStartedEventfrom core.workflow.system_variables import build_system_variablesfrom extensions.ext_redis import RedisClientWrapperfrom graphon.entities import WorkflowStartReason-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom models.account import Accountfrom models.model import AppModefrom models.workflow import Workflow, WorkflowType@@ -47,9 +47,9 @@ def _build_generate_entity(run_id: str) -> WorkflowAppGenerateEntity:)-def _build_runtime_state(run_id: str) -> GraphRuntimeState:+def _build_runtime_state(run_id: str) -> RuntimeState:variable_pool = build_test_variable_pool(variables=build_system_variables(workflow_execution_id=run_id))- return GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter())def _build_pipeline(run_id: str, unbound_session_factory: sessionmaker[Session]) -> WorkflowAppGenerateTaskPipeline:@@ -94,7 +94,9 @@ def test_workflow_app_log_saved_only_on_initial_start(unbound_session_factory: sassert pipeline._workflow_execution_id == run_id-def test_workflow_app_log_skipped_on_resumption_start(unbound_session_factory: sessionmaker[Session]) -> None:+def test_workflow_app_log_skipped_on_resumption_start(+ unbound_session_factory: sessionmaker[Session],+) -> None:run_id = "run-resume"pipeline = _build_pipeline(run_id, unbound_session_factory)pipeline._save_workflow_app_log = MagicMock()diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline_core.py b/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline_core.pyindex b63e323af1a..a8da5b417ab 100644--- a/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline_core.py+++ b/api/tests/unit_tests/core/app/apps/workflow/test_generate_task_pipeline_core.py@@ -50,7 +50,7 @@ from core.base.tts.app_generator_tts_publisher import AudioTrunkfrom core.workflow.system_variables import build_system_variables, system_variables_to_mappingfrom graphon.enums import BuiltinNodeTypes, WorkflowExecutionStatusfrom graphon.model_runtime.entities.llm_entities import LLMUsage-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom libs.datetime_utils import naive_utc_nowfrom models.enums import CreatorUserRolefrom models.model import AppMode, EndUser@@ -107,7 +107,8 @@ def _make_pipeline():class TestWorkflowGenerateTaskPipeline:def test_to_blocking_response_falls_back_to_human_input_required_when_pause_event_missing(self):pipeline = _make_pipeline()- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=build_test_variable_pool(variables=build_system_variables(workflow_execution_id="run-id"),),@@ -205,7 +206,8 @@ class TestWorkflowGenerateTaskPipeline:def test_handle_workflow_started_event_sets_run_id(self, monkeypatch: pytest.MonkeyPatch, sqlite_engine):pipeline = _make_pipeline()- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=build_test_variable_pool(variables=build_system_variables(workflow_execution_id="run-id")),start_at=0.0,)@@ -245,7 +247,8 @@ class TestWorkflowGenerateTaskPipeline:def test_handle_workflow_failed_event_yields_error(self):pipeline = _make_pipeline()pipeline._workflow_execution_id = "run-id"- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=build_test_variable_pool(variables=build_system_variables(workflow_execution_id="run-id")),start_at=0.0,)@@ -332,7 +335,8 @@ class TestWorkflowGenerateTaskPipeline:def test_handle_stop_event_yields_finish(self):pipeline = _make_pipeline()pipeline._workflow_execution_id = "run-id"- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-id")),@@ -380,7 +384,6 @@ class TestWorkflowGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)iter_next = QueueIterationNextEvent(index=1,@@ -388,7 +391,6 @@ class TestWorkflowGenerateTaskPipeline:node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="LLM",- node_run_index=1,)iter_done = QueueIterationCompletedEvent(node_execution_id="exec",@@ -396,7 +398,6 @@ class TestWorkflowGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)loop_start = QueueLoopStartEvent(node_execution_id="exec",@@ -404,7 +405,6 @@ class TestWorkflowGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)loop_next = QueueLoopNextEvent(index=1,@@ -412,7 +412,6 @@ class TestWorkflowGenerateTaskPipeline:node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="LLM",- node_run_index=1,)loop_done = QueueLoopCompletedEvent(node_execution_id="exec",@@ -420,10 +419,9 @@ class TestWorkflowGenerateTaskPipeline:node_type=BuiltinNodeTypes.LLM,node_title="LLM",start_at=naive_utc_now(),- node_run_index=1,)filled_event = QueueHumanInputFormFilledEvent(- node_execution_id="exec",+ form_id="form",node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="title",@@ -432,6 +430,7 @@ class TestWorkflowGenerateTaskPipeline:action_text="action",)timeout_event = QueueHumanInputFormTimeoutEvent(+ form_id="form",node_id="node",node_type=BuiltinNodeTypes.LLM,node_title="title",@@ -768,7 +767,8 @@ class TestWorkflowGenerateTaskPipeline:def test_success_partial_and_pause_handlers(self):pipeline = _make_pipeline()pipeline._workflow_execution_id = "run-id"- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-id")),@@ -798,7 +798,8 @@ class TestWorkflowGenerateTaskPipeline:def test_dispatch_event_direct_failed_and_unhandled_paths(self):pipeline = _make_pipeline()pipeline._workflow_execution_id = "run-id"- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-id")),@@ -816,7 +817,8 @@ class TestWorkflowGenerateTaskPipeline:def test_process_stream_response_main_match_paths_and_cleanup(self):pipeline = _make_pipeline()- pipeline._graph_runtime_state = GraphRuntimeState(+ pipeline._graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-id")),diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_stop_aware_ready_queue.py b/api/tests/unit_tests/core/app/apps/workflow/test_stop_aware_ready_queue.pyindex 6dc1781cc00..3034a328d84 100644--- a/api/tests/unit_tests/core/app/apps/workflow/test_stop_aware_ready_queue.py+++ b/api/tests/unit_tests/core/app/apps/workflow/test_stop_aware_ready_queue.py@@ -6,10 +6,9 @@ from core.app.apps.workflow.stop_aware_ready_queue import (StopAwareReadyQueue,attach_stop_aware_ready_queue,)-from graphon.graph_engine.domain.graph_execution import GraphExecution-from graphon.graph_engine.ready_queue import StartTask-from graphon.runtime.graph_runtime_state import GraphRuntimeState-from graphon.runtime.variable_pool import VariablePool+from graphon.engine.ready_queue import StartTask+from graphon.runtime import RuntimeState, VariablePool+from graphon.runtime.execution import GraphExecutiondef _start_task(node_id: str = "next-node") -> StartTask:@@ -63,7 +62,7 @@ def test_ready_queue_rejects_next_node_when_stop_flag_is_set() -> None:def test_attach_stop_aware_ready_queue_wraps_once() -> None:inner = Mock()- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(variable_pool=VariablePool(),start_at=0,ready_queue=inner,diff --git a/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.pyindex cfbe27f191b..b4d5808f4c5 100644--- a/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py+++ b/api/tests/unit_tests/core/app/layers/test_conversation_variable_persist_layer.py@@ -3,18 +3,19 @@ from unittest.mock import Mockfrom core.app.layers.conversation_variable_persist_layer import ConversationVariablePersistenceLayerfrom core.workflow.system_variables import build_system_variablesfrom core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import NodeRunSucceededEvent, NodeRunVariableUpdatedEventfrom graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import NodeRunSucceededEvent, NodeRunVariableUpdatedEventfrom graphon.node_events import NodeRunResult-from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool+from graphon.runtime import ReadOnlyRuntimeStateWrapper, RuntimeState, VariablePoolfrom graphon.variables import StringVariablefrom libs.datetime_utils import naive_utc_now-def _build_graph_runtime_state(conversation_id: str) -> ReadOnlyGraphRuntimeStateWrapper:- return ReadOnlyGraphRuntimeStateWrapper(- GraphRuntimeState(+def _build_graph_runtime_state(conversation_id: str) -> ReadOnlyRuntimeStateWrapper:+ return ReadOnlyRuntimeStateWrapper(+ RuntimeState(+ workflow_id="workflow-1",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(conversation_id=conversation_id)),diff --git a/api/tests/unit_tests/core/app/layers/test_execution_context_layer.py b/api/tests/unit_tests/core/app/layers/test_execution_context_layer.pynew file mode 100644index 00000000000..74175c1a282--- /dev/null+++ b/api/tests/unit_tests/core/app/layers/test_execution_context_layer.py@@ -0,0 +1,89 @@+from concurrent.futures import ThreadPoolExecutor+from contextvars import ContextVar+from typing import override++import pytest+from flask import current_app, g++from context.flask_app_context import capture_flask_context+from core.app.layers import execution_context_layer as module+from core.app.layers.execution_context_layer import ExecutionContextLayer+from core.app.workflow.file_runtime import create_dify_workflow_file_runtime, init_app+from core.workflow.node_factory import DifyNodeFactory+from dify_app import DifyApp+from graphon.engine import Engine+from graphon.engine.layer import Layer+from graphon.engine_events import GraphRunSucceededEvent+from graphon.entities.base_node_data import BaseNodeData+from graphon.file.runtime import peek_workflow_file_runtime, use_workflow_file_runtime+from graphon.graph import Graph+from graphon.nodes.base.node import Node+from graphon.runtime import RuntimeState, VariablePool+from tests.workflow_test_utils import build_test_graph_init_params+++def test_host_context_preserves_each_engine_file_runtime(monkeypatch: pytest.MonkeyPatch) -> None:+ app = DifyApp("execution-context")+ init_app(app)+ monkeypatch.setattr(module, "capture_current_context", capture_flask_context)+ request_id: ContextVar[str] = ContextVar("request_id")+ caller_runtime = create_dify_workflow_file_runtime()++ def build_engine(request: str) -> tuple[Engine, list[str]]:+ runtime = create_dify_workflow_file_runtime()+ with app.app_context():+ g._login_user = request+ request_id.set(request)+ host = ExecutionContextLayer()+ checked_nodes: list[str] = []++ def check_context() -> None:+ assert current_app == app+ assert g._login_user == request+ assert request_id.get() == request+ assert peek_workflow_file_runtime() is runtime++ class CheckNodeContext(Layer):+ @override+ def on_node_run_start(self, node: Node[BaseNodeData]) -> None:+ check_context()+ checked_nodes.append(node.id)++ graph_config: dict[str, object] = {+ "nodes": [+ {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}},+ {"id": "end", "data": {"type": "end", "title": "End", "outputs": []}},+ ],+ "edges": [{"source": "start", "target": "end", "sourceHandle": "source"}],+ }+ state = RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=0.0)+ engine = Engine(+ graph=Graph.init(+ graph_config=graph_config,+ root_node_id="start",+ node_factory=DifyNodeFactory(+ init_params=build_test_graph_init_params(workflow_id="workflow", graph_config=graph_config),+ runtime_state=state,+ ),+ ),+ runtime_state=state,+ workers=2,+ file_runtime=runtime,+ )+ engine.add_layer(host)+ engine.add_layer(CheckNodeContext())+ return engine, checked_nodes++ engines = [build_engine("first"), build_engine("second")]++ def run_engine(item: tuple[Engine, list[str]]) -> None:+ engine, checked_nodes = item+ with use_workflow_file_runtime(caller_runtime):+ for _event in engine.run():+ assert peek_workflow_file_runtime() is caller_runtime+ assert isinstance(_event, GraphRunSucceededEvent)+ assert set(checked_nodes) == {"start", "end"}+ assert peek_workflow_file_runtime() is caller_runtime++ with ThreadPoolExecutor(max_workers=2) as pool:+ list(pool.map(run_engine, engines))diff --git a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.pyindex 5b4cdd2b824..b96c7cc600f 100644--- a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py+++ b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py@@ -1,432 +1,17 @@+"""Persisted resume-envelope compatibility remains independent of engine layers."""+import json-from collections.abc import Sequence-from time import time-from unittest.mock import Mockimport pytest-from sqlalchemy import Engine-from sqlalchemy.orm import Session, sessionmakerfrom core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom, WorkflowAppGenerateEntity-from core.app.layers import pause_state_persist_layer as pause_layer_modulefrom core.app.layers.pause_state_persist_layer import (- PauseStatePersistenceLayer,WorkflowResumptionContext,_AdvancedChatAppGenerateEntityWrapper,_WorkflowGenerateEntityWrapper,)-from core.workflow.nodes.human_input.pause_reason import HumanInputRequired-from core.workflow.system_variables import SystemVariableKey-from graphon.entities.pause_reason import HitlRequired, SchedulingPause-from graphon.filters import GraphEventFilterContext, ResponseStreamFilter-from graphon.graph_engine.entities.commands import GraphEngineCommand-from graphon.graph_engine.layers.base import GraphEngineLayerNotInitializedError-from graphon.graph_events import (- GraphRunFailedEvent,- GraphRunPausedEvent,- GraphRunStartedEvent,- GraphRunSucceededEvent,-)-from graphon.runtime import ReadOnlyVariablePool-from graphon.variables import build_segment-from graphon.variables.segments import Segmentfrom models.model import AppMode-from repositories.factory import DifyAPIRepositoryFactory---def sqlite_session_factory(sqlite_engine: Engine) -> sessionmaker[Session]:- """Provide the real session factory injected into the persistence layer."""-- return sessionmaker(sqlite_engine, expire_on_commit=False)---def _create_initialized_response_stream_filter() -> ResponseStreamFilter:- """Build a `ResponseStreamFilter` that has already run `initialize()`.-- `ResponseStreamFilter.dumps()` raises `RuntimeError` unless the filter has- processed a `GraphEventFilterContext` first. In production this always- happens before any event (including `GraphRunPausedEvent`) reaches- `PauseStatePersistenceLayer.on_event`, so tests that exercise `on_event`- or a subsequent `dumps()` call need a filter in that same state. A- nodeless graph is enough to satisfy the precondition.- """- response_stream_filter = ResponseStreamFilter()- context = GraphEventFilterContext(graph=Mock(nodes={}), runtime_state=Mock())- response_stream_filter.initialize(context)- return response_stream_filter---class TestDataFactory:- """Factory helpers for constructing graph events used in tests."""-- @staticmethod- def create_graph_run_paused_event(outputs: dict[str, object] | None = None) -> GraphRunPausedEvent:- return GraphRunPausedEvent(reasons=[SchedulingPause(message="test pause")], outputs=outputs or {})-- @staticmethod- def create_graph_run_started_event() -> GraphRunStartedEvent:- return GraphRunStartedEvent()-- @staticmethod- def create_graph_run_succeeded_event(outputs: dict[str, object] | None = None) -> GraphRunSucceededEvent:- return GraphRunSucceededEvent(outputs=outputs or {})-- @staticmethod- def create_graph_run_failed_event(- error: str = "Test error",- exceptions_count: int = 1,- ) -> GraphRunFailedEvent:- return GraphRunFailedEvent(error=error, exceptions_count=exceptions_count)---class MockReadOnlyVariablePool:- """Mock implementation of ReadOnlyVariablePool for testing."""-- def __init__(self, variables: dict[tuple[str, str], object] | None = None):- self._variables = variables or {}-- def get(self, selector: Sequence[str]) -> Segment | None:- if len(selector) < 2:- return None- value = self._variables.get((selector[0], selector[1]))- if value is None:- return None- return build_segment(value)-- def get_all_by_node(self, node_id: str) -> dict[str, object]:- return {key: value for (nid, key), value in self._variables.items() if nid == node_id}-- def get_by_prefix(self, prefix: str) -> dict[str, object]:- return {key: value for (nid, key), value in self._variables.items() if nid == prefix}---class MockReadOnlyGraphRuntimeState:- """Mock implementation of ReadOnlyGraphRuntimeState for testing."""-- def __init__(- self,- start_at: float | None = None,- total_tokens: int = 0,- node_run_steps: int = 0,- ready_queue_size: int = 0,- exceptions_count: int = 0,- outputs: dict[str, object] | None = None,- variables: dict[tuple[str, str], object] | None = None,- workflow_execution_id: str | None = None,- ):- self._start_at = start_at or time()- self._total_tokens = total_tokens- self._node_run_steps = node_run_steps- self._ready_queue_size = ready_queue_size- self._exceptions_count = exceptions_count- self._outputs = outputs or {}- resolved_variables = dict(variables or {})- if workflow_execution_id is not None:- resolved_variables[("sys", SystemVariableKey.WORKFLOW_EXECUTION_ID.value)] = workflow_execution_id- self._variable_pool = MockReadOnlyVariablePool(resolved_variables)-- @property- def variable_pool(self) -> ReadOnlyVariablePool:- return self._variable_pool-- @property- def start_at(self) -> float:- return self._start_at-- @property- def total_tokens(self) -> int:- return self._total_tokens-- @property- def node_run_steps(self) -> int:- return self._node_run_steps-- @property- def ready_queue_size(self) -> int:- return self._ready_queue_size-- @property- def exceptions_count(self) -> int:- return self._exceptions_count-- @property- def outputs(self) -> dict[str, object]:- return self._outputs.copy()-- @property- def llm_usage(self):- mock_usage = Mock()- mock_usage.prompt_tokens = 10- mock_usage.completion_tokens = 20- mock_usage.total_tokens = 30- return mock_usage-- def get_output(self, key: str, default: object = None) -> object:- return self._outputs.get(key, default)-- def dumps(self) -> str:- return json.dumps(- {- "start_at": self._start_at,- "total_tokens": self._total_tokens,- "node_run_steps": self._node_run_steps,- "ready_queue_size": self._ready_queue_size,- "exceptions_count": self._exceptions_count,- "outputs": self._outputs,- "variables": {f"{k[0]}.{k[1]}": v for k, v in self._variable_pool._variables.items()},- "workflow_execution_id": self._variable_pool._variables.get(- ("sys", SystemVariableKey.WORKFLOW_EXECUTION_ID.value)- ),- }- )---class MockCommandChannel:- """Mock implementation of CommandChannel for testing."""-- def __init__(self):- self._commands: list[GraphEngineCommand] = []-- def fetch_commands(self) -> list[GraphEngineCommand]:- return self._commands.copy()-- def send_command(self, command: GraphEngineCommand) -> None:- self._commands.append(command)---class TestPauseStatePersistenceLayer:- """Unit tests for PauseStatePersistenceLayer."""-- @staticmethod- def _create_generate_entity(workflow_execution_id: str = "run-123") -> WorkflowAppGenerateEntity:- app_config = WorkflowUIBasedAppConfig(- tenant_id="tenant-123",- app_id="app-123",- app_mode=AppMode.WORKFLOW,- workflow_id="workflow-123",- )- return WorkflowAppGenerateEntity(- task_id="task-123",- app_config=app_config,- inputs={},- files=[],- user_id="user-123",- stream=False,- invoke_from=InvokeFrom.DEBUGGER,- workflow_execution_id=workflow_execution_id,- )-- def test_init_with_dependency_injection(self, sqlite_session_factory: sessionmaker[Session]):- state_owner_user_id = "user-123"-- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id=state_owner_user_id,- generate_entity=self._create_generate_entity(),- response_stream_filter=ResponseStreamFilter(),- )-- assert layer._session_maker is sqlite_session_factory- assert layer._state_owner_user_id == state_owner_user_id- with pytest.raises(GraphEngineLayerNotInitializedError):- _ = layer.graph_runtime_state- assert layer.command_channel is None-- def test_initialize_sets_dependencies(self, sqlite_session_factory: sessionmaker[Session]):- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id="owner",- generate_entity=self._create_generate_entity(),- response_stream_filter=ResponseStreamFilter(),- )-- graph_runtime_state = MockReadOnlyGraphRuntimeState()- command_channel = MockCommandChannel()-- layer.initialize(graph_runtime_state, command_channel)-- assert layer.graph_runtime_state is graph_runtime_state- assert layer.command_channel is command_channel-- def test_on_event_with_graph_run_paused_event(- self, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]- ):- generate_entity = self._create_generate_entity(workflow_execution_id="run-123")- clock = [10.0]- monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])- generate_entity.workflow_execution_timing.elapsed_time = 5.0- generate_entity.workflow_execution_timing.start()- clock[0] = 12.0- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id="owner-123",- generate_entity=generate_entity,- response_stream_filter=_create_initialized_response_stream_filter(),- )-- mock_repo = Mock()- mock_factory = Mock(return_value=mock_repo)- monkeypatch.setattr(DifyAPIRepositoryFactory, "create_api_workflow_run_repository", mock_factory)-- graph_runtime_state = MockReadOnlyGraphRuntimeState(- outputs={"result": "test_output"},- total_tokens=100,- workflow_execution_id="run-123",- )- command_channel = MockCommandChannel()- layer.initialize(graph_runtime_state, command_channel)-- event = TestDataFactory.create_graph_run_paused_event(outputs={"intermediate": "result"})- expected_state = graph_runtime_state.dumps()-- layer.on_event(event)-- mock_factory.assert_called_once_with(sqlite_session_factory)- assert mock_repo.create_workflow_pause.call_count == 1- call_kwargs = mock_repo.create_workflow_pause.call_args.kwargs- assert call_kwargs["workflow_run_id"] == "run-123"- assert call_kwargs["state_owner_user_id"] == "owner-123"- serialized_state = call_kwargs["state"]- resumption_context = WorkflowResumptionContext.loads(serialized_state)- assert resumption_context.elapsed_time == 7.0- assert resumption_context.get_generate_entity().workflow_execution_timing.elapsed_time == 7.0- assert resumption_context.serialized_graph_runtime_state == expected_state- assert resumption_context.get_generate_entity().model_dump() == generate_entity.model_dump()- pause_reasons = call_kwargs["pause_reasons"]-- assert isinstance(pause_reasons, list)-- def test_on_event_enriches_hitl_pause_reasons_before_persisting(- self, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]- ):- generate_entity = self._create_generate_entity(workflow_execution_id="run-123")- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id="owner-123",- generate_entity=generate_entity,- response_stream_filter=_create_initialized_response_stream_filter(),- )-- mock_repo = Mock()- mock_factory = Mock(return_value=mock_repo)- mock_form_repository = Mock(name="form_repository")- enriched_reason = HumanInputRequired(- form_id="form-123",- form_content="Rendered content",- inputs=[],- actions=[],- node_id="node-123",- node_title="Ask for approval",- )- enrich_mock = Mock(return_value=[enriched_reason])- monkeypatch.setattr(DifyAPIRepositoryFactory, "create_api_workflow_run_repository", mock_factory)- monkeypatch.setattr(- pause_layer_module,- "HumanInputFormSubmissionRepository",- Mock(return_value=mock_form_repository),- raising=False,- )- monkeypatch.setattr(- pause_layer_module,- "enrich_graph_pause_reasons",- enrich_mock,- raising=False,- )-- graph_runtime_state = MockReadOnlyGraphRuntimeState(- workflow_execution_id="run-123",- )- command_channel = MockCommandChannel()- layer.initialize(graph_runtime_state, command_channel)-- raw_reason = HitlRequired(- session_id="session-123",- node_id="node-123",- node_title="Ask for approval",- )- event = GraphRunPausedEvent(reasons=[raw_reason], outputs={})-- layer.on_event(event)-- enrich_mock.assert_called_once_with(- reasons=[raw_reason],- form_repository=mock_form_repository,- variable_pool=graph_runtime_state.variable_pool,- )- assert mock_repo.create_workflow_pause.call_args.kwargs["pause_reasons"] == [enriched_reason]-- def test_on_event_ignores_non_paused_events(- self, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]- ):- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id="owner-123",- generate_entity=self._create_generate_entity(),- response_stream_filter=ResponseStreamFilter(),- )-- mock_repo = Mock()- mock_factory = Mock(return_value=mock_repo)- monkeypatch.setattr(DifyAPIRepositoryFactory, "create_api_workflow_run_repository", mock_factory)-- graph_runtime_state = MockReadOnlyGraphRuntimeState()- command_channel = MockCommandChannel()- layer.initialize(graph_runtime_state, command_channel)-- events = [- TestDataFactory.create_graph_run_started_event(),- TestDataFactory.create_graph_run_succeeded_event(),- TestDataFactory.create_graph_run_failed_event(),- ]-- for event in events:- layer.on_event(event)-- mock_factory.assert_not_called()- mock_repo.create_workflow_pause.assert_not_called()-- def test_on_event_raises_when_graph_runtime_state_is_uninitialized(- self, sqlite_session_factory: sessionmaker[Session]- ):- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id="owner-123",- generate_entity=self._create_generate_entity(),- response_stream_filter=ResponseStreamFilter(),- )-- event = TestDataFactory.create_graph_run_paused_event()-- with pytest.raises(GraphEngineLayerNotInitializedError):- layer.on_event(event)-- def test_on_event_asserts_when_workflow_execution_id_missing(- self, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]- ):- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id="owner-123",- generate_entity=self._create_generate_entity(),- response_stream_filter=_create_initialized_response_stream_filter(),- )-- mock_repo = Mock()- mock_factory = Mock(return_value=mock_repo)- monkeypatch.setattr(DifyAPIRepositoryFactory, "create_api_workflow_run_repository", mock_factory)-- graph_runtime_state = MockReadOnlyGraphRuntimeState(workflow_execution_id=None)- command_channel = MockCommandChannel()- layer.initialize(graph_runtime_state, command_channel)-- event = TestDataFactory.create_graph_run_paused_event()-- with pytest.raises(AssertionError):- layer.on_event(event)-- mock_factory.assert_not_called()- mock_repo.create_workflow_pause.assert_not_called()def _build_workflow_generate_entity_for_roundtrip() -> WorkflowResumptionContext:@@ -499,7 +84,7 @@ def _build_advanced_chat_generate_entity_for_roundtrip() -> WorkflowResumptionCo),],)-def test_workflow_resumption_context_dumps_loads_roundtrip(state: WorkflowResumptionContext):+def test_workflow_resumption_context_dumps_loads_roundtrip(state: WorkflowResumptionContext) -> None:"""WorkflowResumptionContext roundtrip preserves workflow generate entity metadata."""dumped = state.dumps()loaded = WorkflowResumptionContext.loads(dumped)@@ -511,57 +96,6 @@ def test_workflow_resumption_context_dumps_loads_roundtrip(state: WorkflowResumpassert restored_entity.extras["trace_session_id"] == "session-1"-def test_on_event_persists_response_stream_filter_dump(- monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]-) -> None:- generate_entity = TestPauseStatePersistenceLayer._create_generate_entity(workflow_execution_id="run-123")- response_stream_filter = _create_initialized_response_stream_filter()- layer = PauseStatePersistenceLayer(- session_factory=sqlite_session_factory,- state_owner_user_id="owner-123",- generate_entity=generate_entity,- response_stream_filter=response_stream_filter,- )-- mock_repo = Mock()- mock_factory = Mock(return_value=mock_repo)- monkeypatch.setattr(DifyAPIRepositoryFactory, "create_api_workflow_run_repository", mock_factory)-- graph_runtime_state = MockReadOnlyGraphRuntimeState(workflow_execution_id="run-123")- layer.initialize(graph_runtime_state, MockCommandChannel())-- event = TestDataFactory.create_graph_run_paused_event()- layer.on_event(event)-- serialized_state = mock_repo.create_workflow_pause.call_args.kwargs["state"]- resumption_context = WorkflowResumptionContext.loads(serialized_state)- assert resumption_context.serialized_response_stream_filter_state == response_stream_filter.dumps()---def test_get_response_stream_filter_restores_dumped_state() -> None:- original = _create_initialized_response_stream_filter()- context = WorkflowResumptionContext(- serialized_graph_runtime_state=json.dumps({"state": "workflow"}),- generate_entity=_WorkflowGenerateEntityWrapper(entity=TestPauseStatePersistenceLayer._create_generate_entity()),- serialized_response_stream_filter_state=original.dumps(),- )-- restored = context.get_response_stream_filter()-- assert restored.dumps() == original.dumps()---def test_get_response_stream_filter_defaults_when_state_missing() -> None:- context = WorkflowResumptionContext(- serialized_graph_runtime_state=json.dumps({"state": "workflow"}),- generate_entity=_WorkflowGenerateEntityWrapper(entity=TestPauseStatePersistenceLayer._create_generate_entity()),- )-- restored = context.get_response_stream_filter()-- assert isinstance(restored, ResponseStreamFilter)--def test_resumption_context_defaults_elapsed_time_for_legacy_snapshots() -> None:context = _build_workflow_generate_entity_for_roundtrip()payload = context.model_dump()diff --git a/api/tests/unit_tests/core/app/layers/test_suspend_layer.py b/api/tests/unit_tests/core/app/layers/test_suspend_layer.pydeleted file mode 100644index 12d49be0f1a..00000000000--- a/api/tests/unit_tests/core/app/layers/test_suspend_layer.py+++ /dev/null@@ -1,19 +0,0 @@-from core.app.layers.suspend_layer import SuspendLayer-from graphon.graph_events import GraphRunPausedEvent---class TestSuspendLayer:- def test_on_event_accepts_paused_event(self):- layer = SuspendLayer()- assert layer.is_paused() is False- layer.on_graph_start()- assert layer.is_paused() is False- layer.on_event(GraphRunPausedEvent())- assert layer.is_paused() is True-- def test_on_event_ignores_other_events(self):- layer = SuspendLayer()- layer.on_graph_start()- initial_state = layer.is_paused()- layer.on_event(object())- assert layer.is_paused() is initial_statediff --git a/api/tests/unit_tests/core/app/layers/test_timeslice_layer.py b/api/tests/unit_tests/core/app/layers/test_timeslice_layer.pyindex 61ced61d8c8..c54404aa2be 100644--- a/api/tests/unit_tests/core/app/layers/test_timeslice_layer.py+++ b/api/tests/unit_tests/core/app/layers/test_timeslice_layer.py@@ -4,7 +4,7 @@ from unittest.mock import Mock, patchimport pytestfrom core.app.layers.timeslice_layer import TimeSliceLayer-from graphon.graph_engine.entities.commands import CommandType, GraphEngineCommand+from graphon.engine.command import PauseCommandfrom services.workflow.entities import WorkflowScheduleCFSPlanEntityfrom services.workflow.scheduler import SchedulerCommand@@ -95,5 +95,5 @@ class TestTimeSliceLayer:scheduler.remove_job.assert_called_once_with("job-1")layer.command_channel.send_command.assert_called_once()sent_command = layer.command_channel.send_command.call_args[0][0]- assert isinstance(sent_command, GraphEngineCommand)- assert sent_command.command_type == CommandType.PAUSE+ assert isinstance(sent_command, PauseCommand)+ assert sent_command.reason == SchedulerCommand.RESOURCE_LIMIT_REACHEDdiff --git a/api/tests/unit_tests/core/app/layers/test_trigger_post_layer.py b/api/tests/unit_tests/core/app/layers/test_trigger_post_layer.pyindex 150f05081ea..1a7c21cf870 100644--- a/api/tests/unit_tests/core/app/layers/test_trigger_post_layer.py+++ b/api/tests/unit_tests/core/app/layers/test_trigger_post_layer.py@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session, sessionmakerfrom core.app.layers.trigger_post_layer import TriggerPostLayerfrom core.workflow.system_variables import build_system_variables-from graphon.graph_events import (+from graphon.engine_events import (GraphRunAbortedEvent,GraphRunFailedEvent,GraphRunSucceededEvent,diff --git a/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.py b/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.pydeleted file mode 100644index 61876bc05fb..00000000000--- a/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.py+++ /dev/null@@ -1,144 +0,0 @@-import pytest-from sqlalchemy.orm import Session, sessionmaker--from core.app.app_config.entities import WorkflowUIBasedAppConfig-from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter-from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity-from core.app.entities.queue_entities import QueueWorkflowPausedEvent-from core.app.entities.task_entities import WorkflowPauseStreamResponse-from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext, _WorkflowGenerateEntityWrapper-from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer-from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository-from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository-from core.workflow.system_variables import build_system_variables-from graphon.entities import WorkflowStartReason-from graphon.enums import WorkflowExecutionStatus, WorkflowType-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import (- GraphRunAbortedEvent,- GraphRunFailedEvent,- GraphRunPartialSucceededEvent,- GraphRunPausedEvent,- GraphRunStartedEvent,- GraphRunSucceededEvent,-)-from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool-from models.account import Account-from models.enums import WorkflowRunTriggeredFrom-from models.model import AppMode-from models.workflow import WorkflowNodeExecutionTriggeredFrom, WorkflowRun--- ("terminal_event", "status"),- [- (GraphRunSucceededEvent(), WorkflowExecutionStatus.SUCCEEDED),- (GraphRunPartialSucceededEvent(exceptions_count=1), WorkflowExecutionStatus.PARTIAL_SUCCEEDED),- (GraphRunFailedEvent(error="failed"), WorkflowExecutionStatus.FAILED),- (GraphRunAbortedEvent(reason="stopped"), WorkflowExecutionStatus.STOPPED),- ],-)-def test_elapsed_time_accumulates_across_pauses_and_matches_delayed_stream_responses(- monkeypatch: pytest.MonkeyPatch,- sqlite_session_factory: sessionmaker[Session],- terminal_event: GraphRunSucceededEvent | GraphRunPartialSucceededEvent | GraphRunFailedEvent | GraphRunAbortedEvent,- status: WorkflowExecutionStatus,-) -> None:- clock = [100.0]- monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])- monkeypatch.setattr("core.app.workflow.layers.persistence._inspector_publish_workflow_completed", lambda **_: None)- entity = WorkflowAppGenerateEntity(- task_id="task-1",- workflow_execution_id="run-1",- app_config=WorkflowUIBasedAppConfig(- tenant_id="tenant-1", app_id="app-1", workflow_id="workflow-1", app_mode=AppMode.WORKFLOW- ),- inputs={},- files=[],- user_id="user-1",- stream=True,- invoke_from=InvokeFrom.SERVICE_API,- )- system_variables = build_system_variables(workflow_execution_id="run-1")- total = 0.0- account = Account(name="Test", email="[email protected]")- account.id = "user-1"-- for index, duration in enumerate([2.0, 3.0, 4.0]):- pool = VariablePool()- for variable in system_variables:- pool.add(variable.selector, variable)- runtime = GraphRuntimeState(variable_pool=pool, start_at=clock[0])- repository = SQLAlchemyWorkflowExecutionRepository(- session_factory=sqlite_session_factory,- tenant_id="tenant-1",- user=account,- app_id="app-1",- triggered_from=WorkflowRunTriggeredFrom.APP_RUN,- )- node_repository = SQLAlchemyWorkflowNodeExecutionRepository(- session_factory=sqlite_session_factory,- tenant_id="tenant-1",- user=account,- app_id="app-1",- triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,- )- layer = WorkflowPersistenceLayer(- application_generate_entity=entity,- workflow_info=PersistenceWorkflowInfo(- workflow_id="workflow-1", workflow_type=WorkflowType.WORKFLOW, version="1", graph_data={}- ),- workflow_execution_repository=repository,- workflow_node_execution_repository=node_repository,- )- layer.initialize(ReadOnlyGraphRuntimeStateWrapper(runtime), InMemoryChannel())- layer.on_graph_start()- reason = WorkflowStartReason.INITIAL if index == 0 else WorkflowStartReason.RESUMPTION- layer.on_event(GraphRunStartedEvent(reason=reason))- with sqlite_session_factory() as session:- started_execution = session.get(WorkflowRun, "run-1")- assert started_execution is not None- assert started_execution.elapsed_time == total-- clock[0] += duration- total += duration- is_paused = index < 2- layer.on_event(GraphRunPausedEvent() if is_paused else terminal_event)- with sqlite_session_factory() as session:- execution = session.get(WorkflowRun, "run-1")- assert execution is not None- assert execution.elapsed_time == total- assert execution.status == (WorkflowExecutionStatus.PAUSED if is_paused else status)- assert (execution.finished_at is None) == is_paused-- # Queue processing and pause waits must not extend a settled segment.- clock[0] += 1000- converter = WorkflowResponseConverter(- application_generate_entity=entity,- user=account,- system_variables=system_variables,- )- converter.workflow_start_to_stream_response(- task_id="task-1", workflow_run_id="run-1", workflow_id="workflow-1", reason=reason- )- if is_paused:- responses = converter.workflow_pause_to_stream_response(- event=QueueWorkflowPausedEvent(), task_id="task-1", graph_runtime_state=runtime- )- pause_response = responses[-1]- assert isinstance(pause_response, WorkflowPauseStreamResponse)- assert pause_response.data.elapsed_time == total- context = WorkflowResumptionContext(- generate_entity=_WorkflowGenerateEntityWrapper(entity=entity),- serialized_graph_runtime_state=runtime.dumps(),- elapsed_time=total,- )- entity = WorkflowResumptionContext.loads(context.dumps()).get_generate_entity()- assert isinstance(entity, WorkflowAppGenerateEntity)- assert entity.workflow_execution_timing.elapsed_time == total- else:- response = converter.workflow_finish_to_stream_response(- task_id="task-1", workflow_id="workflow-1", status=status, graph_runtime_state=runtime- )- assert response.data.elapsed_time == 9.0- assert entity.workflow_execution_timing.finish() == 9.0diff --git a/api/tests/unit_tests/core/app/workflow/layers/test_persistence.py b/api/tests/unit_tests/core/app/workflow/layers/test_persistence.pydeleted file mode 100644index b7a8754d92c..00000000000--- a/api/tests/unit_tests/core/app/workflow/layers/test_persistence.py+++ /dev/null@@ -1,101 +0,0 @@-from datetime import UTC, datetime-from unittest.mock import Mock--import pytest--from core.app.workflow.layers.persistence import (- PersistenceWorkflowInfo,- WorkflowPersistenceLayer,- _NodeRuntimeSnapshot,-)-from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus, WorkflowType-from graphon.node_events import NodeRunResult---def _build_layer() -> WorkflowPersistenceLayer:- application_generate_entity = Mock()- application_generate_entity.inputs = {}-- return WorkflowPersistenceLayer(- application_generate_entity=application_generate_entity,- workflow_info=PersistenceWorkflowInfo(- workflow_id="workflow-id",- workflow_type=WorkflowType.WORKFLOW,- version="1",- graph_data={},- ),- workflow_execution_repository=Mock(),- workflow_node_execution_repository=Mock(),- )---def test_update_node_execution_prefers_event_finished_at(monkeypatch: pytest.MonkeyPatch) -> None:- layer = _build_layer()- node_execution = Mock()- node_execution.id = "node-exec-1"- node_execution.created_at = datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC).replace(tzinfo=None)- node_execution.process_data = None- node_execution.update_from_mapping = Mock()-- layer._node_snapshots[node_execution.id] = _NodeRuntimeSnapshot(- node_id="node-id",- title="LLM",- predecessor_node_id=None,- iteration_id="iter-1",- loop_id=None,- created_at=node_execution.created_at,- )-- event_finished_at = datetime(2024, 1, 1, 0, 0, 2, tzinfo=UTC).replace(tzinfo=None)- delayed_processing_time = datetime(2024, 1, 1, 0, 0, 10, tzinfo=UTC).replace(tzinfo=None)- monkeypatch.setattr("core.app.workflow.layers.persistence.naive_utc_now", lambda: delayed_processing_time)-- layer._update_node_execution(- node_execution,- NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED),- WorkflowNodeExecutionStatus.SUCCEEDED,- finished_at=event_finished_at,- )-- assert node_execution.finished_at == event_finished_at- assert node_execution.elapsed_time == 2.0---def test_update_node_execution_projects_start_outputs() -> None:- layer = _build_layer()- node_execution = Mock()- node_execution.id = "node-exec-2"- node_execution.node_type = BuiltinNodeTypes.START- node_execution.created_at = datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC).replace(tzinfo=None)- node_execution.process_data = None- node_execution.update_from_mapping = Mock()-- layer._node_snapshots[node_execution.id] = _NodeRuntimeSnapshot(- node_id="start",- title="Start",- predecessor_node_id=None,- iteration_id=None,- loop_id=None,- created_at=node_execution.created_at,- )-- layer._update_node_execution(- node_execution,- NodeRunResult(- status=WorkflowNodeExecutionStatus.SUCCEEDED,- inputs={"question": "hello"},- outputs={- "question": "hello",- "sys.query": "hello",- "env.API_KEY": "secret",- },- ),- WorkflowNodeExecutionStatus.SUCCEEDED,- )-- node_execution.update_from_mapping.assert_called_once_with(- inputs={"question": "hello"},- process_data={},- outputs={"question": "hello"},- metadata={},- )diff --git a/api/tests/unit_tests/core/app/workflow/layers/test_persistence_inspector_publish.py b/api/tests/unit_tests/core/app/workflow/layers/test_persistence_inspector_publish.pyindex e23f4a81814..31ff06ff71d 100644--- a/api/tests/unit_tests/core/app/workflow/layers/test_persistence_inspector_publish.py+++ b/api/tests/unit_tests/core/app/workflow/layers/test_persistence_inspector_publish.py@@ -1,6 +1,6 @@"""Verify the workflow persistence layer fans Inspector deltas to redis pub/sub.-The hook lives in ``core/app/workflow/layers/persistence.py``:+The hook lives in ``services/workflow_persistence.py``:every ``_handle_node_*`` and the terminal ``_handle_graph_run_*`` handlerscall into ``services.workflow.inspector_events.publish_node_changed`` /``publish_workflow_completed`` after the DB write succeeds. Those calls are@@ -22,25 +22,20 @@ from unittest.mock import MagicMockimport pytest-from core.app.workflow.layers import persistence as persistence_mod-from core.app.workflow.layers.persistence import WorkflowPersistenceLayer+from graphon.engine_events import GraphRunPausedEvent, NodeRunStartedEvent+from services import workflow_persistence as persistence_mod+from services.workflow_persistence import WorkflowPersistenceLayer@pytest.fixturedef layer() -> WorkflowPersistenceLayer:- """Build a layer instance with all repository / trace deps stubbed.-- We bypass ``__init__`` because constructing it for real pulls in the- workflow engine's app-generate-entity, repos, and a runtime state — none- of which matter for asserting that the publish-hook fires.- """- instance = WorkflowPersistenceLayer.__new__(WorkflowPersistenceLayer)- # Minimum surface the handlers touch:- instance._workflow_execution_repository = MagicMock()- instance._workflow_node_execution_repository = MagicMock()- instance._trace_manager = None- instance._workflow_info = MagicMock(workflow_id="wf-1")- instance._application_generate_entity = MagicMock()+ """Use real layer initialization with repository and runtime test doubles."""+ instance = WorkflowPersistenceLayer(+ application_generate_entity=MagicMock(),+ workflow_info=MagicMock(workflow_id="wf-1"),+ workflow_execution_repository=MagicMock(),+ workflow_node_execution_repository=MagicMock(),+ )# Use a SimpleNamespace-like spec so Pydantic-validated callsites (e.g.# ``WorkflowNodeExecution.new`` requires real strings) get the right types.workflow_execution = MagicMock()@@ -51,12 +46,10 @@ def layer() -> WorkflowPersistenceLayer:workflow_execution.error_message = Noneworkflow_execution.exceptions_count = 0workflow_execution.finished_at = None+ instance.set_node_run_indices({"exec-1": 1})instance._workflow_execution = workflow_execution- instance._node_execution_cache = {}- instance._node_snapshots = {}- instance._node_sequence = 0- # `graph_runtime_state` is a layer-base property; stub it.- instance._graph_runtime_state = MagicMock(total_tokens=0, node_run_steps=0, outputs={}, exceptions_count=0)+ # `runtime_state` is a layer-base property; stub it.+ instance._runtime_state = MagicMock(total_tokens=0, node_run_steps=0, outputs={}, exceptions_count=0)return instance@@ -114,7 +107,7 @@ def test_graph_run_aborted_publishes_workflow_completed(layer, capture_publishesdef test_graph_run_paused_does_not_publish_completion(layer, capture_publishes):"""Pause is not a terminal state — the Inspector keeps waiting for eitherresume or a real terminal event."""- layer._handle_graph_run_paused(_graph_event(outputs={}))+ layer.on_event(GraphRunPausedEvent(outputs={}))assert capture_publishes["workflow"] == []assert capture_publishes["node"] == []@@ -124,15 +117,15 @@ def test_graph_run_paused_does_not_publish_completion(layer, capture_publishes):# ──────────────────────────────────────────────────────────────────────────────-def _node_started_event(node_id: str = "agent-1", exec_id: str = "exec-1") -> MagicMock:- return MagicMock(+def _node_started_event(node_id: str = "agent-1", exec_id: str = "exec-1") -> NodeRunStartedEvent:+ return NodeRunStartedEvent(id=exec_id,node_id=node_id,node_type="agent",node_title="Greeter",predecessor_node_id=None,- in_iteration_id=None,- in_loop_id=None,+ container_id="",+ node_version="1",start_at=datetime(2026, 5, 26, 0, 0, 0),)diff --git a/api/tests/unit_tests/core/app/workflow/test_file_runtime.py b/api/tests/unit_tests/core/app/workflow/test_file_runtime.pyindex 222c4039f61..294b3906f62 100644--- a/api/tests/unit_tests/core/app/workflow/test_file_runtime.py+++ b/api/tests/unit_tests/core/app/workflow/test_file_runtime.py@@ -4,6 +4,7 @@ import base64import hashlibimport hmacfrom collections.abc import Callable, Iterator+from concurrent.futures import ThreadPoolExecutorfrom datetime import UTC, datetimefrom unittest.mock import patchfrom urllib.parse import parse_qs, urlparse@@ -15,10 +16,12 @@ from sqlalchemy.orm import Session, sessionmakerfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFromfrom core.app.file_access import DatabaseFileAccessController, FileAccessScopefrom core.app.workflow import file_runtime-from core.app.workflow.file_runtime import DifyWorkflowFileRuntime, bind_dify_workflow_file_runtime+from core.app.workflow.file_runtime import DifyWorkflowFileRuntimefrom core.workflow.file_reference import build_file_reference+from dify_app import DifyAppfrom extensions.storage.storage_type import StorageTypefrom graphon.file import File, FileTransferMethod, FileType+from graphon.file.runtime import peek_workflow_file_runtime, use_workflow_file_runtimefrom models import ToolFile, UploadFilefrom models.base import TypeBasefrom models.enums import CreatorUserRole@@ -400,15 +403,65 @@ def test_runtime_helper_wrappers_delegate_to_config_and_io(mock_load.assert_called_once_with("path", stream=True)-def test_bind_dify_workflow_file_runtime_registers_runtime(monkeypatch: pytest.MonkeyPatch) -> None:- runtimes: list[DifyWorkflowFileRuntime] = []-- def set_runtime(runtime: DifyWorkflowFileRuntime) -> None:- runtimes.append(runtime)-- monkeypatch.setattr(file_runtime, "set_workflow_file_runtime", set_runtime)-- bind_dify_workflow_file_runtime()-- assert len(runtimes) == 1- assert isinstance(runtimes[0], DifyWorkflowFileRuntime)+def test_app_file_runtime_scopes_restore_across_nested_apps_and_workers(monkeypatch: pytest.MonkeyPatch) -> None:+ first, second = DifyApp("first"), DifyApp("second")+ runtimes = [_build_runtime(), _build_runtime()]+ monkeypatch.setattr(file_runtime, "create_dify_workflow_file_runtime", iter(runtimes).__next__)+ file_runtime.init_app(first)+ file_runtime.init_app(second)++ def in_worker() -> None:+ assert peek_workflow_file_runtime() is None+ with first.app_context():+ assert peek_workflow_file_runtime() is runtimes[0]+ assert peek_workflow_file_runtime() is None++ def failed_rendering() -> None:+ with second.app_context():+ assert peek_workflow_file_runtime() is runtimes[1]+ raise ValueError("failed rendering")++ with use_workflow_file_runtime(None):+ with first.app_context():+ assert peek_workflow_file_runtime() is runtimes[0]+ with pytest.raises(ValueError, match="failed rendering"):+ failed_rendering()+ assert peek_workflow_file_runtime() is runtimes[0]+ with ThreadPoolExecutor(max_workers=1) as pool:+ pool.submit(in_worker).result()+ assert peek_workflow_file_runtime() is runtimes[0]+ assert peek_workflow_file_runtime() is None++ context = first.app_context()+ context.push()+ context.push()+ context.pop()+ assert peek_workflow_file_runtime() is runtimes[0]+ context.pop()+ assert peek_workflow_file_runtime() is None++[email protected]("explicit_none", [False, True])+def test_app_file_runtime_restores_when_teardown_raises(monkeypatch: pytest.MonkeyPatch, explicit_none: bool) -> None:+ app = DifyApp("failed-teardown")+ runtime, previous_runtime = _build_runtime(), _build_runtime()+ monkeypatch.setattr(file_runtime, "create_dify_workflow_file_runtime", lambda: runtime)+ file_runtime.init_app(app)+ request_error = RuntimeError("request failed")++ @app.teardown_appcontext+ def fail_teardown(error: BaseException | None) -> None:+ assert peek_workflow_file_runtime() is runtime+ assert error is (None if explicit_none else request_error)+ raise ValueError("teardown failed")++ with use_workflow_file_runtime(previous_runtime):+ context = app.app_context()+ context.push()+ pop_args = (None,) if explicit_none else ()+ try:+ raise request_error+ except RuntimeError:+ with pytest.raises(ValueError, match="teardown failed"):+ context.pop(*pop_args)+ assert peek_workflow_file_runtime() is previous_runtimediff --git a/api/tests/unit_tests/core/app/workflow/test_node_factory.py b/api/tests/unit_tests/core/app/workflow/test_node_factory.pyindex d6cc4dc391e..8e4ed261e0f 100644--- a/api/tests/unit_tests/core/app/workflow/test_node_factory.py+++ b/api/tests/unit_tests/core/app/workflow/test_node_factory.py@@ -9,11 +9,11 @@ from graphon.enums import BuiltinNodeTypesclass DummyNode:- def __init__(self, *, node_id, data, graph_init_params, graph_runtime_state, **kwargs):+ def __init__(self, *, node_id, data, init_params, runtime_state, **kwargs):self.id = node_idself.data = data- self.graph_init_params = graph_init_params- self.graph_runtime_state = graph_runtime_state+ self.init_params = init_params+ self.runtime_state = runtime_stateself.kwargs = kwargs@@ -73,8 +73,8 @@ class TestDifyNodeFactory:)return DifyNodeFactory(- graph_init_params=SimpleNamespace(run_context=run_context),- graph_runtime_state=SimpleNamespace(),+ init_params=SimpleNamespace(run_context=run_context),+ runtime_state=SimpleNamespace(),)def test_create_node_unknown_type(self):diff --git a/api/tests/unit_tests/core/app/workflow/test_observability_layer_extra.py b/api/tests/unit_tests/core/app/workflow/test_observability_layer_extra.pyindex 53f2763e4a3..27a6f4a70cf 100644--- a/api/tests/unit_tests/core/app/workflow/test_observability_layer_extra.py+++ b/api/tests/unit_tests/core/app/workflow/test_observability_layer_extra.py@@ -5,7 +5,6 @@ from types import SimpleNamespaceimport pytestfrom core.app.workflow.layers.observability import ObservabilityLayer-from graphon.enums import BuiltinNodeTypesfrom tests.unit_tests.config_override import apply_config_overrides@@ -27,71 +26,18 @@ class TestObservabilityLayerExtras:assert layer._tracer is Noneassert "Failed to get OpenTelemetry tracer" in caplog.text- def test_get_parser_uses_registry_when_node_type_matches(self):- layer = ObservabilityLayer()-- parser = layer._get_parser(SimpleNamespace(node_type=BuiltinNodeTypes.TOOL))-- assert parser is layer._parsers[BuiltinNodeTypes.TOOL]-- def test_get_parser_defaults_when_node_type_missing(self):- layer = ObservabilityLayer()-- parser = layer._get_parser(SimpleNamespace(node_type=None))-- assert parser is layer._default_parser-- def test_on_graph_start_clears_contexts(self):- layer = ObservabilityLayer()- layer._node_contexts["exec"] = SimpleNamespace(span=object(), token="token")-- layer.on_graph_start()-- assert layer._node_contexts == {}-- def test_on_event_is_noop(self):- layer = ObservabilityLayer()-- layer.on_event(object())-- def test_on_graph_end_clears_unfinished_contexts(self, caplog: pytest.LogCaptureFixture):- layer = ObservabilityLayer()- layer._node_contexts["exec"] = SimpleNamespace(span=object(), token="token")-- layer.on_graph_end(error=None)-- assert layer._node_contexts == {}- assert "node spans were not properly ended" in caplog.text-- def test_on_node_run_start_skips_without_execution_id(self):- layer = ObservabilityLayer()- layer._is_disabled = False- layer._tracer = None-- layer.on_node_run_start(SimpleNamespace(execution_id=None, title="node", id="node"))-- assert layer._node_contexts == {}-- def test_on_node_run_start_skips_when_disabled(self):- layer = ObservabilityLayer()- layer._is_disabled = True- layer._tracer = SimpleNamespace(start_span=lambda *_args, **_kwargs: object())-- layer.on_node_run_start(SimpleNamespace(execution_id="exec", title="node", id="node"))-- assert layer._node_contexts == {}-- def test_on_node_run_start_skips_when_execution_id_missing_even_with_tracer(self):+ def test_node_run_context_skips_without_execution_id(self):layer = ObservabilityLayer()layer._is_disabled = Falsecalls: list[str] = []layer._tracer = SimpleNamespace(start_span=lambda *_args, **_kwargs: calls.append("called"))- layer.on_node_run_start(SimpleNamespace(execution_id=None, title="node", id="node"))+ with layer.node_run_context(SimpleNamespace(execution_id=None, title="node", id="node")):+ assert calls == []assert calls == []- def test_on_node_run_start_logs_warning_when_span_creation_fails(self, caplog: pytest.LogCaptureFixture):+ def test_node_runs_when_span_creation_fails(self, caplog: pytest.LogCaptureFixture):layer = ObservabilityLayer()layer._is_disabled = False@@ -100,98 +46,9 @@ class TestObservabilityLayerExtras:layer._tracer = SimpleNamespace(start_span=_raise)- layer.on_node_run_start(SimpleNamespace(execution_id="exec", title="node", id="node"))-- assert "Failed to create OpenTelemetry span for node" in caplog.text-- def test_on_node_run_end_without_context_noop(self):- layer = ObservabilityLayer()- layer._is_disabled = False-- layer.on_node_run_end(SimpleNamespace(execution_id="missing", id="node"), error=None)-- assert layer._node_contexts == {}-- def test_on_node_run_end_skips_when_disabled(self):- layer = ObservabilityLayer()- layer._is_disabled = True- layer._node_contexts["exec"] = SimpleNamespace(span=object(), token="token")-- layer.on_node_run_end(SimpleNamespace(execution_id="exec", id="node"), error=None)-- assert "exec" in layer._node_contexts-- def test_on_node_run_end_skips_without_execution_id(self):- layer = ObservabilityLayer()- layer._is_disabled = False-- layer.on_node_run_end(SimpleNamespace(execution_id=None, id="node"), error=None)-- assert layer._node_contexts == {}-- def test_on_node_run_end_calls_span_end(self, monkeypatch: pytest.MonkeyPatch):- layer = ObservabilityLayer()- layer._is_disabled = False- ended: list[str] = []-- class _Parser:- def parse(self, **_kwargs):- return None-- span = SimpleNamespace(end=lambda: ended.append("ended"))- layer._default_parser = _Parser()- layer._node_contexts["exec"] = SimpleNamespace(span=span, token="token")-- monkeypatch.setattr("core.app.workflow.layers.observability.context_api.detach", lambda _token: None)-- node = SimpleNamespace(execution_id="exec", title="Node", id="node", node_type=None)- layer.on_node_run_end(node, error=None)-- assert ended == ["ended"]- assert "exec" not in layer._node_contexts-- def test_on_node_run_end_logs_detach_failure(- self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture- ):- layer = ObservabilityLayer()- layer._is_disabled = False-- class _Parser:- def parse(self, **_kwargs):- return None-- layer._default_parser = _Parser()- layer._node_contexts["exec"] = SimpleNamespace(span=SimpleNamespace(end=lambda: None), token="bad-token")-- def _raise(*_args, **_kwargs):- raise RuntimeError("detach failed")-- monkeypatch.setattr("core.app.workflow.layers.observability.context_api.detach", _raise)-- node = SimpleNamespace(execution_id="exec", title="Node", id="node", node_type=None)- layer.on_node_run_end(node, error=None)-- assert "Failed to detach OpenTelemetry token" in caplog.text- assert "exec" not in layer._node_contexts-- def test_on_node_run_start_and_end_creates_span(self, monkeypatch: pytest.MonkeyPatch):- layer = ObservabilityLayer()- layer._is_disabled = False-- span = SimpleNamespace(end=lambda: None)- tracer = SimpleNamespace(start_span=lambda *args, **kwargs: span)-- monkeypatch.setattr("core.app.workflow.layers.observability.context_api.get_current", lambda: object())- monkeypatch.setattr("core.app.workflow.layers.observability.set_span_in_context", lambda s: object())- monkeypatch.setattr("core.app.workflow.layers.observability.context_api.attach", lambda ctx: "token")- monkeypatch.setattr("core.app.workflow.layers.observability.context_api.detach", lambda token: None)-- layer._tracer = tracer-- node = SimpleNamespace(execution_id="exec", title="Node", id="node", node_type=None)-- layer.on_node_run_start(node)- assert "exec" in layer._node_contexts+ node_executed = False+ with layer.node_run_context(SimpleNamespace(execution_id="exec", title="node", id="node")):+ node_executed = True- layer.on_node_run_end(node, error=None)- assert "exec" not in layer._node_contexts+ assert node_executed+ assert "start failed" in caplog.textdiff --git a/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py b/api/tests/unit_tests/core/app/workflow/test_persistence_layer.pyindex bc67bfc12d5..36cc0d37bae 100644--- a/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py+++ b/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py@@ -1,28 +1,17 @@from __future__ import annotations-from datetime import UTC, datetime+from datetime import UTC, datetime, timedeltafrom types import SimpleNamespaceimport pytestfrom core.app.entities.app_invoke_entities import WorkflowAppGenerateEntity-from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayerfrom core.ops.ops_trace_manager import TraceTask, TraceTaskNamefrom core.workflow.system_variables import SystemVariableKey, build_system_variables-from graphon.entities import WorkflowNodeExecution, WorkflowStartReason-from graphon.entities.pause_reason import SchedulingPause-from graphon.enums import (- BuiltinNodeTypes,- WorkflowExecutionStatus,- WorkflowNodeExecutionMetadataKey,- WorkflowNodeExecutionStatus,- WorkflowType,-)-from graphon.graph_events import (+from graphon.engine_events import (GraphRunAbortedEvent,GraphRunFailedEvent,GraphRunPartialSucceededEvent,- GraphRunPausedEvent,GraphRunStartedEvent,GraphRunSucceededEvent,NodeRunExceptionEvent,@@ -32,9 +21,19 @@ from graphon.graph_events import (NodeRunStartedEvent,NodeRunSucceededEvent,)+from graphon.entities import WorkflowNodeExecution, WorkflowStartReason+from graphon.entities.pause_reason import SchedulingPause+from graphon.enums import (+ BuiltinNodeTypes,+ WorkflowExecutionStatus,+ WorkflowNodeExecutionMetadataKey,+ WorkflowNodeExecutionStatus,+ WorkflowType,+)from graphon.model_runtime.entities.llm_entities import LLMUsagefrom graphon.node_events import NodeRunResult-from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool+from graphon.runtime import ReadOnlyRuntimeStateWrapper, RuntimeState, VariablePool+from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayerclass _RepoRecorder:@@ -42,7 +41,6 @@ class _RepoRecorder:self.saved: list[object] = []self.synchronously_saved: list[object] = []self.saved_exec_data: list[object] = []- self.loaded: list[object] = []def save(self, entity):self.saved.append(entity)@@ -53,9 +51,6 @@ class _RepoRecorder:def save_execution_data(self, entity):self.saved_exec_data.append(entity)- def get_by_workflow_execution(self, _workflow_execution_id):- return self.loaded-def _naive_utc_now() -> datetime:return datetime.now(UTC).replace(tzinfo=None)@@ -65,17 +60,22 @@ def _make_layer(system_variables: list | None = None,*,extras: dict | None = None,+ graph_data: dict | None = None,trace_manager: object | None = None,+ workflow_execution_repo=None,+ workflow_node_execution_repo=None,+ node_run_indices=None,):system_variables = system_variables or build_system_variables(workflow_execution_id="run-id",conversation_id="conv-id",)- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=system_variables),start_at=0.0,)- read_only_state = ReadOnlyGraphRuntimeStateWrapper(runtime_state)+ read_only_state = ReadOnlyRuntimeStateWrapper(runtime_state)application_generate_entity = WorkflowAppGenerateEntity.model_construct(task_id="task",@@ -95,11 +95,13 @@ def _make_layer(workflow_id="workflow-id",workflow_type=WorkflowType.WORKFLOW,version="1",- graph_data={"nodes": [], "edges": []},+ graph_data=graph_data if graph_data is not None else {"nodes": [], "edges": []},)- workflow_execution_repo = _RepoRecorder()- workflow_node_execution_repo = _RepoRecorder()+ workflow_execution_repo = workflow_execution_repo if workflow_execution_repo is not None else _RepoRecorder()+ workflow_node_execution_repo = (+ workflow_node_execution_repo if workflow_node_execution_repo is not None else _RepoRecorder()+ )layer = WorkflowPersistenceLayer(application_generate_entity=application_generate_entity,@@ -109,25 +111,12 @@ def _make_layer(trace_manager=trace_manager,)layer.initialize(read_only_state, command_channel=None)+ layer.set_node_run_indices(node_run_indices or {})return layer, workflow_execution_repo, workflow_node_execution_repo, runtime_stateclass TestWorkflowPersistenceLayer:- def test_on_graph_start_resets_state(self):- layer, _, _, _ = _make_layer()- layer._workflow_execution = object()- layer._node_execution_cache["cached"] = object()- layer._node_snapshots["cached"] = object()- layer._node_sequence = 9-- layer.on_graph_start()-- assert layer._workflow_execution is None- assert layer._node_execution_cache == {}- assert layer._node_snapshots == {}- assert layer._node_sequence == 0-def test_get_execution_id_requires_system_variable(self):layer, _, _, _ = _make_layer(build_system_variables())@@ -147,7 +136,16 @@ class TestWorkflowPersistenceLayer:assert "sys.conversation_id" not in inputsassert inputs[f"sys.{SystemVariableKey.WORKFLOW_EXECUTION_ID.value}"] == "run-id"- def test_fail_running_node_executions_marks_failed(self):+ @pytest.mark.parametrize(+ "status",+ [+ WorkflowNodeExecutionStatus.RUNNING,+ WorkflowNodeExecutionStatus.RETRY,+ WorkflowNodeExecutionStatus.PAUSED,+ WorkflowNodeExecutionStatus.SUCCEEDED,+ ],+ )+ def test_fail_running_node_executions_marks_only_live_attempts_failed(self, status):layer, _, node_repo, _ = _make_layer()execution = WorkflowNodeExecution(@@ -158,21 +156,22 @@ class TestWorkflowPersistenceLayer:node_id="node",node_type=BuiltinNodeTypes.START,title="Start",+ status=status,created_at=_naive_utc_now(),)layer._node_execution_cache[execution.id] = executionlayer._fail_running_node_executions(error_message="boom")- assert execution.status == WorkflowNodeExecutionStatus.FAILED- assert node_repo.saved-- def test_handle_graph_run_started_saves_execution(self):- layer, exec_repo, _, _ = _make_layer()-- layer._handle_graph_run_started()-- assert exec_repo.saved+ if status in (WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.RETRY):+ assert execution.status == WorkflowNodeExecutionStatus.FAILED+ assert execution.error == "boom"+ assert execution.finished_at is not None+ assert node_repo.saved == [execution]+ else:+ assert execution.status == status+ assert execution.finished_at is None+ assert node_repo.saved == []def test_resumption_restores_container_execution_before_terminal_event(self):layer, _, node_repo, _ = _make_layer()@@ -188,7 +187,7 @@ class TestWorkflowPersistenceLayer:status=WorkflowNodeExecutionStatus.RUNNING,created_at=started_at,)- node_repo.loaded = [execution]+ layer.set_node_execution_history([execution])layer.on_event(GraphRunStartedEvent(reason=WorkflowStartReason.RESUMPTION))layer.on_event(@@ -197,16 +196,18 @@ class TestWorkflowPersistenceLayer:node_id=execution.node_id,node_type=execution.node_type,start_at=started_at,+ finished_at=started_at + timedelta(seconds=2),node_run_result=NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED),))assert execution.status == WorkflowNodeExecutionStatus.SUCCEEDED- assert layer._next_node_sequence() == 5+ assert execution.elapsed_time == 2.0+ assert execution.index == 4def test_handle_graph_run_succeeded_updates_execution(self):layer, exec_repo, _, runtime_state = _make_layer()- layer._handle_graph_run_started()+ layer.on_event(GraphRunStartedEvent())usage = LLMUsage.empty_usage()usage.total_tokens = 3runtime_state.add_llm_usage(usage)@@ -214,16 +215,16 @@ class TestWorkflowPersistenceLayer:runtime_state.increment_node_run_steps()runtime_state.set_output("out", "v")- layer._handle_graph_run_succeeded(GraphRunSucceededEvent(outputs={"ok": True}))+ layer.on_event(GraphRunSucceededEvent(outputs={"ok": True}))- saved = exec_repo.saved[-1]+ saved = exec_repo.synchronously_saved[-1]assert saved.status == WorkflowExecutionStatus.SUCCEEDEDassert saved.total_tokens == 3assert saved.total_steps == 2def test_handle_graph_run_partial_succeeded_updates_execution(self):layer, exec_repo, _, runtime_state = _make_layer()- layer._handle_graph_run_started()+ layer.on_event(GraphRunStartedEvent())usage = LLMUsage.empty_usage()usage.total_tokens = 5runtime_state.add_llm_usage(usage)@@ -231,11 +232,9 @@ class TestWorkflowPersistenceLayer:runtime_state.increment_node_run_steps()runtime_state._graph_execution = SimpleNamespace(exceptions_count=2)- layer._handle_graph_run_partial_succeeded(- GraphRunPartialSucceededEvent(outputs={"ok": True}, exceptions_count=2)- )+ layer.on_event(GraphRunPartialSucceededEvent(outputs={"ok": True}, exceptions_count=2))- saved = exec_repo.saved[-1]+ saved = exec_repo.synchronously_saved[-1]assert saved.status == WorkflowExecutionStatus.PARTIAL_SUCCEEDEDassert saved.exceptions_count == 2assert saved.total_tokens == 5@@ -244,7 +243,7 @@ class TestWorkflowPersistenceLayer:trace_tasks: list[object] = []trace_manager = SimpleNamespace(user_id="user", add_trace_task=lambda task: trace_tasks.append(task))layer, exec_repo, node_repo, _ = _make_layer(extras={"external_trace_id": "trace"}, trace_manager=trace_manager)- layer._handle_graph_run_started()+ layer.on_event(GraphRunStartedEvent())running = WorkflowNodeExecution(id="node-exec",@@ -258,10 +257,10 @@ class TestWorkflowPersistenceLayer:)layer._node_execution_cache[running.id] = running- layer._handle_graph_run_failed(GraphRunFailedEvent(error="boom", exceptions_count=1))+ layer.on_event(GraphRunFailedEvent(error="boom", exceptions_count=1))assert node_repo.saved- assert exec_repo.saved[-1].status == WorkflowExecutionStatus.FAILED+ assert exec_repo.synchronously_saved[-1].status == WorkflowExecutionStatus.FAILEDassert trace_tasksdef test_handle_graph_run_succeeded_enqueues_parent_trace_context(self, monkeypatch: pytest.MonkeyPatch):@@ -278,7 +277,7 @@ class TestWorkflowPersistenceLayer:},trace_manager=trace_manager,)- layer._handle_graph_run_started()+ layer.on_event(GraphRunStartedEvent())captured: dict[str, object] = {}@@ -299,7 +298,7 @@ class TestWorkflowPersistenceLayer:monkeypatch.setattr(TraceTask, "workflow_trace", fake_workflow_trace)- layer._handle_graph_run_succeeded(GraphRunSucceededEvent(outputs={"ok": True}))+ layer.on_event(GraphRunSucceededEvent(outputs={"ok": True}))assert trace_taskstrace_task = trace_tasks[0]@@ -323,33 +322,17 @@ class TestWorkflowPersistenceLayer:def test_handle_graph_run_aborted_sets_status(self):layer, exec_repo, _, _ = _make_layer()- layer._handle_graph_run_started()+ layer.on_event(GraphRunStartedEvent())- layer._handle_graph_run_aborted(GraphRunAbortedEvent(reason=None, outputs={}))+ layer.on_event(GraphRunAbortedEvent(reason=None, outputs={}))- saved = exec_repo.saved[-1]+ saved = exec_repo.synchronously_saved[-1]assert saved.status == WorkflowExecutionStatus.STOPPEDassert saved.error_message- def test_handle_graph_run_paused_updates_outputs(self):- layer, exec_repo, _, runtime_state = _make_layer()- layer._handle_graph_run_started()- usage = LLMUsage.empty_usage()- usage.total_tokens = 7- runtime_state.add_llm_usage(usage)- for _ in range(5):- runtime_state.increment_node_run_steps()-- layer._handle_graph_run_paused(GraphRunPausedEvent(outputs={"pause": True}))-- saved = exec_repo.saved[-1]- assert saved.status == WorkflowExecutionStatus.PAUSED- assert saved.outputs == {"pause": True}- assert saved.finished_at is None-def test_handle_node_started_and_retry(self):- layer, _, node_repo, _ = _make_layer()- layer._handle_graph_run_started()+ layer, _, node_repo, _ = _make_layer(node_run_indices={"exec": 1})+ layer.on_event(GraphRunStartedEvent())start_event = NodeRunStartedEvent(id="exec",@@ -358,14 +341,12 @@ class TestWorkflowPersistenceLayer:node_title="Start",start_at=_naive_utc_now(),predecessor_node_id="prev",- in_iteration_id="iter",- in_loop_id="loop",)- layer._handle_node_started(start_event)+ layer.on_event(start_event)assert node_repo.savedassert "exec" in layer._node_execution_cache- assert layer._node_snapshots["exec"].node_id == "node"+ assert node_repo.saved[-1].created_at == start_event.start_atretry_event = NodeRunRetryEvent(id="exec",@@ -376,14 +357,14 @@ class TestWorkflowPersistenceLayer:error="retry",retry_index=1,)- layer._handle_node_retry(retry_event)+ layer.on_event(retry_event)assert node_repo.saved_exec_datadef test_agent_v2_caller_row_is_saved_synchronously_before_node_run(self):- layer, _, node_repo, _ = _make_layer()- layer._handle_graph_run_started()+ layer, _, node_repo, _ = _make_layer(node_run_indices={"agent-exec": 1})+ layer.on_event(GraphRunStartedEvent())- layer._handle_node_started(+ layer.on_event(NodeRunStartedEvent(id="agent-exec",node_id="agent-node",@@ -398,10 +379,10 @@ class TestWorkflowPersistenceLayer:assert node_repo.saved == []def test_retry_history_is_preserved_after_node_succeeds(self):- layer, _, node_repo, _ = _make_layer()- layer._handle_graph_run_started()+ layer, _, node_repo, _ = _make_layer(node_run_indices={"exec": 1})+ layer.on_event(GraphRunStartedEvent())started_at = _naive_utc_now()- layer._handle_node_started(+ layer.on_event(NodeRunStartedEvent(id="exec",node_id="node",@@ -412,7 +393,7 @@ class TestWorkflowPersistenceLayer:)for retry_index in (1, 2):- layer._handle_node_retry(+ layer.on_event(NodeRunRetryEvent(id="exec",node_id="node",@@ -430,7 +411,7 @@ class TestWorkflowPersistenceLayer:))- layer._handle_node_succeeded(+ layer.on_event(NodeRunSucceededEvent(id="exec",node_id="node",@@ -461,13 +442,15 @@ class TestWorkflowPersistenceLayer:[(NodeRunFailedEvent, WorkflowNodeExecutionStatus.FAILED),(NodeRunExceptionEvent, WorkflowNodeExecutionStatus.EXCEPTION),+ (GraphRunFailedEvent, WorkflowNodeExecutionStatus.FAILED),+ (GraphRunAbortedEvent, WorkflowNodeExecutionStatus.FAILED),],)def test_retry_history_is_preserved_after_terminal_error(self, event_type, expected_status):- layer, _, node_repo, _ = _make_layer()- layer._handle_graph_run_started()+ layer, _, node_repo, _ = _make_layer(node_run_indices={"exec": 1})+ layer.on_event(GraphRunStartedEvent())started_at = _naive_utc_now()- layer._handle_node_started(+ layer.on_event(NodeRunStartedEvent(id="exec",node_id="node",@@ -476,7 +459,7 @@ class TestWorkflowPersistenceLayer:start_at=started_at,))- layer._handle_node_retry(+ layer.on_event(NodeRunRetryEvent(id="exec",node_id="node",@@ -489,72 +472,62 @@ class TestWorkflowPersistenceLayer:))- terminal_event = event_type(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.LLM,- start_at=started_at,- error="terminal failure",- node_run_result=NodeRunResult(process_data={"terminal": True}),- )- if expected_status == WorkflowNodeExecutionStatus.FAILED:- layer._handle_node_failed(terminal_event)+ if event_type is GraphRunFailedEvent:+ terminal_event = event_type(error="terminal failure")+ elif event_type is GraphRunAbortedEvent:+ terminal_event = event_type(reason="terminal failure")else:- layer._handle_node_exception(terminal_event)+ terminal_event = event_type(+ id="exec",+ node_id="node",+ node_type=BuiltinNodeTypes.LLM,+ start_at=started_at,+ error="terminal failure",+ node_run_result=NodeRunResult(process_data={"terminal": True}),+ )+ node_repo.saved.clear()+ layer.on_event(terminal_event)- saved_execution = node_repo.saved_exec_data[-1]+ saved_execution = node_repo.saved[-1]assert saved_execution.status == expected_status- assert saved_execution.process_data["terminal"] is True+ assert saved_execution.error == "terminal failure"+ assert saved_execution.finished_at is not None+ if event_type in (NodeRunFailedEvent, NodeRunExceptionEvent):+ assert saved_execution.process_data["terminal"] is Trueassert saved_execution.process_data["__dify_retry_history"][0]["retry_index"] == 1- def test_handle_node_result_events_update_execution(self):- layer, _, node_repo, _ = _make_layer()- layer._handle_graph_run_started()-- start_event = NodeRunStartedEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.LLM,- node_title="LLM",- start_at=_naive_utc_now(),- )- layer._handle_node_started(start_event)-- result = NodeRunResult(inputs={"a": 1}, process_data={"b": 2}, outputs={"c": 3}, metadata={})- success_event = NodeRunSucceededEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.LLM,- start_at=_naive_utc_now(),- node_run_result=result,+ def test_pause_flush_persists_retry_history_synchronously(self):+ layer, _, node_repo, _ = _make_layer(node_run_indices={"exec": 1})+ layer.on_event(GraphRunStartedEvent())+ started_at = _naive_utc_now()+ layer.on_event(+ NodeRunStartedEvent(+ id="exec", node_id="node", node_type=BuiltinNodeTypes.LLM, node_title="LLM", start_at=started_at+ ))- layer._handle_node_succeeded(success_event)-- failed_event = NodeRunFailedEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.LLM,- start_at=_naive_utc_now(),- error="boom",- node_run_result=result,+ layer.on_event(+ NodeRunRetryEvent(+ id="exec",+ node_id="node",+ node_type=BuiltinNodeTypes.LLM,+ node_title="LLM",+ start_at=started_at,+ error="retry failed",+ retry_index=1,+ ))- layer._handle_node_failed(failed_event)+ assert node_repo.synchronously_saved == []- exception_event = NodeRunExceptionEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.LLM,- start_at=_naive_utc_now(),- error="err",- node_run_result=result,- )- layer._handle_node_exception(exception_event)+ layer.persist_unfinished_nodes_synchronously()- assert node_repo.saved_exec_data+ saved_execution = node_repo.synchronously_saved[-1]+ assert saved_execution.status == WorkflowNodeExecutionStatus.RETRY+ assert saved_execution.finished_at is None+ assert saved_execution.process_data["__dify_retry_history"][0]["error"] == "retry failed"def test_handle_node_pause_requested_skips_outputs(self):- layer, _, _, _ = _make_layer()- layer._handle_graph_run_started()+ layer, _, _, _ = _make_layer(node_run_indices={"exec": 1})+ layer.on_event(GraphRunStartedEvent())start_event = NodeRunStartedEvent(id="exec",node_id="node",@@ -562,7 +535,7 @@ class TestWorkflowPersistenceLayer:node_title="LLM",start_at=_naive_utc_now(),)- layer._handle_node_started(start_event)+ layer.on_event(start_event)domain_execution = layer._node_execution_cache["exec"]domain_execution.inputs = {"old": True}@@ -570,7 +543,10 @@ class TestWorkflowPersistenceLayer:result = NodeRunResult(inputs={"new": True},outputs={"out": 1},- process_data={"p": 1, "workflow_agent_binding_id": "workflow-binding-1"},+ process_data={+ "p": 1,+ "workflow_agent_binding_id": "workflow-binding-1",+ },metadata={},)pause_event = NodeRunPauseRequestedEvent(@@ -580,17 +556,19 @@ class TestWorkflowPersistenceLayer:reason=SchedulingPause(message="pause"),node_run_result=result,)- layer._handle_node_pause_requested(pause_event)+ layer.on_event(pause_event)assert domain_execution.status == WorkflowNodeExecutionStatus.PAUSEDassert domain_execution.inputs == {"old": True}- assert domain_execution.process_data == {"workflow_agent_binding_id": "workflow-binding-1"}+ assert domain_execution.process_data == {+ "workflow_agent_binding_id": "workflow-binding-1",+ }def test_handle_node_retry_preserves_workflow_agent_binding_identity(self):- layer, _, _, _ = _make_layer()- layer._handle_graph_run_started()+ layer, _, _, _ = _make_layer(node_run_indices={"exec": 1})+ layer.on_event(GraphRunStartedEvent())started_at = _naive_utc_now()- layer._handle_node_started(+ layer.on_event(NodeRunStartedEvent(id="exec",node_id="node",@@ -600,7 +578,7 @@ class TestWorkflowPersistenceLayer:))- layer._handle_node_retry(+ layer.on_event(NodeRunRetryEvent(id="exec",node_id="node",@@ -610,7 +588,9 @@ class TestWorkflowPersistenceLayer:error="retry",retry_index=1,node_run_result=NodeRunResult(- process_data={"workflow_agent_binding_id": "workflow-binding-1"},+ process_data={+ "workflow_agent_binding_id": "workflow-binding-1",+ },),))@@ -627,147 +607,3 @@ class TestWorkflowPersistenceLayer:with pytest.raises(ValueError, match="workflow execution not initialized"):layer._get_workflow_execution()-- def test_next_node_sequence_increments(self):- layer, _, _, _ = _make_layer()- assert layer._next_node_sequence() == 1- assert layer._next_node_sequence() == 2-- def test_on_graph_end_is_noop(self):- layer, _, _, _ = _make_layer()-- assert layer.on_graph_end(error=None) is None-- def test_on_event_dispatches_to_all_known_handlers(self):- layer, _, _, _ = _make_layer()- called: list[str] = []-- def _record(name: str):- def _handler(*_args, **_kwargs):- called.append(name)-- return _handler-- layer._handle_graph_run_started = _record("started")- layer._handle_graph_run_succeeded = _record("succeeded")- layer._handle_graph_run_partial_succeeded = _record("partial")- layer._handle_graph_run_failed = _record("failed")- layer._handle_graph_run_aborted = _record("aborted")- layer._handle_graph_run_paused = _record("paused")- layer._handle_node_started = _record("node_started")- layer._handle_node_retry = _record("node_retry")- layer._handle_node_succeeded = _record("node_succeeded")- layer._handle_node_failed = _record("node_failed")- layer._handle_node_exception = _record("node_exception")- layer._handle_node_pause_requested = _record("node_paused")-- node_result = NodeRunResult()- now = _naive_utc_now()- events = [- GraphRunStartedEvent(),- GraphRunSucceededEvent(outputs={"ok": True}),- GraphRunPartialSucceededEvent(outputs={"ok": True}, exceptions_count=1),- GraphRunFailedEvent(error="boom", exceptions_count=1),- GraphRunAbortedEvent(reason="stop", outputs={"x": 1}),- GraphRunPausedEvent(outputs={"pause": True}),- NodeRunStartedEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.START,- node_title="Start",- start_at=now,- ),- NodeRunRetryEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.START,- node_title="Start",- start_at=now,- error="retry",- retry_index=1,- ),- NodeRunSucceededEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.START,- start_at=now,- node_run_result=node_result,- ),- NodeRunFailedEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.START,- start_at=now,- error="failed",- node_run_result=node_result,- ),- NodeRunExceptionEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.START,- start_at=now,- error="error",- node_run_result=node_result,- ),- NodeRunPauseRequestedEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.START,- reason=SchedulingPause(message="pause"),- node_run_result=node_result,- ),- ]- expected_order = [- "started",- "succeeded",- "partial",- "failed",- "aborted",- "paused",- "node_started",- "node_retry",- "node_succeeded",- "node_failed",- "node_exception",- "node_paused",- ]-- for event in events:- layer.on_event(event)-- assert called == expected_order-- def test_on_event_dispatches_retry_before_started_for_retry_event(self):- layer, _, _, _ = _make_layer()- called: list[str] = []-- def _record(name: str):- def _handler(*_args, **_kwargs):- called.append(name)-- return _handler-- layer._handle_node_started = _record("node_started")- layer._handle_node_retry = _record("node_retry")-- layer.on_event(- NodeRunRetryEvent(- id="exec",- node_id="node",- node_type=BuiltinNodeTypes.START,- node_title="Start",- start_at=_naive_utc_now(),- error="retry",- retry_index=1,- )- )-- assert called == ["node_retry"]-- def test_enqueue_trace_task_skips_when_disabled(self):- trace_tasks: list[object] = []- layer, exec_repo, _, _ = _make_layer()- layer._handle_graph_run_started()- layer._handle_graph_run_succeeded(GraphRunSucceededEvent(outputs={"ok": True}))- assert exec_repo.saved- assert not trace_tasksdiff --git a/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py b/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.pyindex 46894f1882d..9c7721624e8 100644--- a/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py+++ b/api/tests/unit_tests/core/plugin/test_backwards_invocation_app.py@@ -15,6 +15,7 @@ from models import Account, Tenant, TenantAccountJoinfrom models.enums import EndUserTypefrom models.model import App, AppMode, AppModelConfig, EndUserfrom models.workflow import Workflow+from services.workflow_run_agg import WorkflowRunAggfrom tests.unit_tests.model_factories import make_app, make_end_user, make_workflow@@ -177,6 +178,7 @@ class TestPluginAppBackwardsInvocation:route = mocker.patch.object(PluginAppBackwardsInvocation, route_method, return_value={"routed": True})result = PluginAppBackwardsInvocation.invoke_app(+ execution_driver=WorkflowRunAgg.run,app_id="app",user_id="user",tenant_id="tenant",@@ -202,6 +204,7 @@ class TestPluginAppBackwardsInvocation:route = mocker.patch.object(PluginAppBackwardsInvocation, "invoke_workflow_app", return_value={"ok": True})result = PluginAppBackwardsInvocation.invoke_app(+ execution_driver=WorkflowRunAgg.run,app_id="app",user_id="",tenant_id="tenant",@@ -225,6 +228,7 @@ class TestPluginAppBackwardsInvocation:with pytest.raises(ValueError, match="missing query"):PluginAppBackwardsInvocation.invoke_app(+ execution_driver=WorkflowRunAgg.run,app_id="app",user_id="user",tenant_id="tenant",@@ -247,6 +251,7 @@ class TestPluginAppBackwardsInvocation:with pytest.raises(ValueError, match="unexpected app type"):PluginAppBackwardsInvocation.invoke_app(+ execution_driver=WorkflowRunAgg.run,app_id="app",user_id="user",tenant_id="tenant",@@ -271,6 +276,7 @@ class TestPluginAppBackwardsInvocation:spy = mocker.patch(generator_path, return_value={"result": "ok"})result = PluginAppBackwardsInvocation.invoke_chat_app(+ execution_driver=WorkflowRunAgg.run,app=app,user=_end_user(),conversation_id="conv-1",@@ -302,6 +308,7 @@ class TestPluginAppBackwardsInvocation:session = self.sessionresult = PluginAppBackwardsInvocation.invoke_chat_app(+ execution_driver=WorkflowRunAgg.run,app=app,user=_end_user(),conversation_id="conv-1",@@ -324,6 +331,7 @@ class TestPluginAppBackwardsInvocation:mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=None)with pytest.raises(ValueError, match="unexpected app type"):PluginAppBackwardsInvocation.invoke_chat_app(+ execution_driver=WorkflowRunAgg.run,app=app,user=_end_user(),conversation_id="conv-1",@@ -338,6 +346,7 @@ class TestPluginAppBackwardsInvocation:app = _app(mode=cast(AppMode, "invalid"))with pytest.raises(ValueError, match="unexpected app type"):PluginAppBackwardsInvocation.invoke_chat_app(+ execution_driver=WorkflowRunAgg.run,app=app,user=_end_user(),conversation_id="conv-1",@@ -364,6 +373,7 @@ class TestPluginAppBackwardsInvocation:)result = PluginAppBackwardsInvocation.invoke_workflow_app(+ execution_driver=WorkflowRunAgg.run,app=app,workflow=workflow,user=_end_user(),@@ -385,6 +395,7 @@ class TestPluginAppBackwardsInvocation:mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=None)with pytest.raises(ValueError, match="unexpected app type"):PluginAppBackwardsInvocation.invoke_app(+ execution_driver=WorkflowRunAgg.run,app_id="app",user_id="user",tenant_id="tenant",@@ -535,6 +546,7 @@ class TestPluginAppBackwardsInvocation:route = mocker.patch.object(PluginAppBackwardsInvocation, "invoke_workflow_app", return_value={"ok": True})result = PluginAppBackwardsInvocation.invoke_app(+ execution_driver=WorkflowRunAgg.run,app_id="app",user_id="wecom-sender-1",tenant_id="tenant",diff --git a/api/tests/unit_tests/core/plugin/test_backwards_invocation_model.py b/api/tests/unit_tests/core/plugin/test_backwards_invocation_model.pyindex 83ff38e1bd5..5a1599ed994 100644--- a/api/tests/unit_tests/core/plugin/test_backwards_invocation_model.py+++ b/api/tests/unit_tests/core/plugin/test_backwards_invocation_model.py@@ -15,6 +15,7 @@ from graphon.model_runtime.entities.message_entities import UserPromptMessagefrom graphon.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelPropertyKey, ModelTypefrom graphon.model_runtime.entities.provider_entities import ProviderEntityfrom graphon.model_runtime.model_providers.base.tts_model import TTSModel+from graphon.model_runtime.protocols.tts_runtime import TTSChunkfrom models.account import Tenantfrom models.provider import ProviderType@@ -129,7 +130,7 @@ def test_invoke_tts_emits_the_verified_mime_type_for_backwards_invocation():patch.object(model_instance.model_type_instance,"invoke",- return_value=[b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00audio-data"],+ return_value=[TTSChunk(data=b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00audio-data", mime_type=None)],),patch.object(model_instance.model_type_instance, "get_model_schema", return_value=_tts_schema("wav")),):@@ -183,7 +184,12 @@ def test_backwards_event_stream_serializes_a_deferred_mime_error():patch.object(model_instance.model_type_instance,"invoke",- return_value=[TTSAudioChunk(b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00audio-data", "audio/mpeg")],+ return_value=[+ TTSChunk(+ data=TTSAudioChunk(b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00audio-data", "audio/mpeg"),+ mime_type="audio/mpeg",+ )+ ],),patch.object(model_instance.model_type_instance, "get_model_schema", return_value=_tts_schema("mp3")),):diff --git a/api/tests/unit_tests/core/plugin/test_model_runtime_adapter.py b/api/tests/unit_tests/core/plugin/test_model_runtime_adapter.pyindex ec55237688e..2e20eb941e6 100644--- a/api/tests/unit_tests/core/plugin/test_model_runtime_adapter.py+++ b/api/tests/unit_tests/core/plugin/test_model_runtime_adapter.py@@ -29,6 +29,7 @@ from graphon.model_runtime.entities.llm_entities import (from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessageToolfrom graphon.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelTypefrom graphon.model_runtime.entities.provider_entities import ConfigurateMethod, ProviderEntity+from graphon.model_runtime.protocols.tts_runtime import TTSChunkdef _recording_model_client(@@ -515,7 +516,7 @@ class TestPluginModelRuntime:voice="alloy",)- assert list(result) == [b"chunk"]+ assert list(result) == [TTSChunk(data=b"chunk", mime_type=None)]assert client_calls["invoke_tts"] == [call(tenant_id="tenant",diff --git a/api/tests/unit_tests/core/repositories/test_human_input_form_repository_impl.py b/api/tests/unit_tests/core/repositories/test_human_input_form_repository_impl.pyindex 15925ccdc14..39071a16b53 100644--- a/api/tests/unit_tests/core/repositories/test_human_input_form_repository_impl.py+++ b/api/tests/unit_tests/core/repositories/test_human_input_form_repository_impl.py@@ -2,7 +2,7 @@from __future__ import annotations-from datetime import datetime+from datetime import datetime, timedeltaimport pytestfrom sqlalchemy import Engine, event@@ -10,6 +10,7 @@ from sqlalchemy.exc import SQLAlchemyErrorfrom sqlalchemy.orm import Sessionfrom core.repositories.human_input_repository import (+ FormNotFoundError,HumanInputFormRecord,HumanInputFormRepositoryImpl,HumanInputFormSubmissionRepository,@@ -276,6 +277,39 @@ def _persist_form(class TestHumanInputFormRepositoryImplPublicMethods:+ @pytest.mark.parametrize("mismatched_scope", ["tenant_id", "app_id", "workflow_run_id", "node_id", "id"])+ def test_mark_timeout_rejects_form_outside_execution_scope(+ self, sqlite_session: Session, mismatched_scope: str+ ) -> None:+ form = _make_form(expiration_time=naive_utc_now() - timedelta(minutes=1))+ setattr(form, mismatched_scope, "other-value")+ _persist_form(sqlite_session, form)+ repo = HumanInputFormRepositoryImpl(tenant_id="tenant-id", app_id="app-id", workflow_execution_id="run-1")++ with pytest.raises(FormNotFoundError):+ repo.mark_timeout("node-1", form_id="form-1")++ sqlite_session.refresh(form)+ assert form.status == HumanInputFormStatus.WAITING++ @pytest.mark.parametrize("already_submitted", [False, True])+ def test_mark_timeout_preserves_waiting_form_that_is_not_eligible(+ self, sqlite_session: Session, already_submitted: bool+ ) -> None:+ form = _make_form(+ expiration_time=naive_utc_now() + timedelta(minutes=-1 if already_submitted else 1),+ submitted_at=naive_utc_now() if already_submitted else None,+ )+ _persist_form(sqlite_session, form)+ repo = HumanInputFormRepositoryImpl(tenant_id="tenant-id", app_id="app-id", workflow_execution_id="run-1")++ entity = repo.mark_timeout("node-1", form_id="form-1")++ assert entity.status == HumanInputFormStatus.WAITING+ assert entity.submitted == already_submitted+ sqlite_session.refresh(form)+ assert form.status == HumanInputFormStatus.WAITING+def test_get_form_returns_entity_and_recipients(self, sqlite_session: Session):form = _make_form()recipient = _make_recipient(form.id)diff --git a/api/tests/unit_tests/core/repositories/test_human_input_repository.py b/api/tests/unit_tests/core/repositories/test_human_input_repository.pyindex c53b5b88fac..97fddc97a8e 100644--- a/api/tests/unit_tests/core/repositories/test_human_input_repository.py+++ b/api/tests/unit_tests/core/repositories/test_human_input_repository.py@@ -6,7 +6,7 @@ from datetime import datetime, timedeltafrom typing import Anyimport pytest-from sqlalchemy import Engine, select+from sqlalchemy import Engine, event, selectfrom sqlalchemy.orm import Session, sessionmakerfrom core.repositories.human_input_repository import (@@ -40,6 +40,7 @@ from models.human_input import (HumanInputFormRecipient,RecipientType,)+from repositories.human_input_errors import FormAlreadyHandledError@pytest.fixture@@ -484,6 +485,39 @@ def test_submission_get_by_form_id_returns_none_on_missing(repository_session: Sassert repo.get_by_form_id("f") is None+def test_submission_batch_materializes_requested_forms_in_one_read(repository_session: Session) -> None:+ _persist_form(repository_session, form_id="first", status=HumanInputFormStatus.SUBMITTED)+ _persist_form(repository_session, form_id="second", status=HumanInputFormStatus.TIMEOUT)+ _persist_form(repository_session, form_id="unrequested")+ statements: list[str] = []++ def record_statement(_connection, _cursor, statement: str, _parameters, _context, _executemany) -> None:+ statements.append(statement)++ engine = repository_session.get_bind()+ event.listen(engine, "before_cursor_execute", record_statement)+ try:+ records = HumanInputFormSubmissionRepository().get_by_form_ids(+ ["second", "missing", "first", "first"], tenant_id="tenant", app_id="app", workflow_run_id="run"+ )+ assert len(statements) == 1+ assert set(records) == {"first", "second"}+ assert records["first"].status == HumanInputFormStatus.SUBMITTED+ assert records["second"].definition.user_actions[0].id == "submit"+ assert records["second"].status == HumanInputFormStatus.TIMEOUT+ # Materialized records remain usable after the repository's session closes.+ assert len(statements) == 1+ assert (+ HumanInputFormSubmissionRepository().get_by_form_ids(+ [], tenant_id="tenant", app_id="app", workflow_run_id="run"+ )+ == {}+ )+ assert len(statements) == 1+ finally:+ event.remove(engine, "before_cursor_execute", record_statement)++def test_mark_submitted_updates_and_raises_when_missing(repository_session: Session, monkeypatch: pytest.MonkeyPatch) -> None:@@ -526,6 +560,28 @@ def test_mark_submitted_updates_and_raises_when_missing(assert record.submitted_data == {"k": "v"}[email protected]("status", [HumanInputFormStatus.EXPIRED, HumanInputFormStatus.SUBMITTED])+def test_mark_submitted_does_not_overwrite_a_concurrent_stop_or_submission(+ repository_session: Session, status: HumanInputFormStatus+) -> None:+ form = _persist_form(repository_session, form_id="stopped-form", workflow_run_id=None)+ form.status = status+ repository_session.commit()+ with pytest.raises(FormAlreadyHandledError) as error:+ HumanInputFormSubmissionRepository().mark_submitted(+ form_id=form.id,+ recipient_id=None,+ selected_action_id="approve",+ form_data={"note": "late submission"},+ submission_user_id=None,+ submission_end_user_id=None,+ )+ assert error.value.status == status+ repository_session.expire_all()+ assert form.status == status+ assert form.submitted_data is None++def test_mark_submitted_serializes_select_and_file_payloads(repository_session: Session, monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_node_execution_repository.py b/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_node_execution_repository.pyindex 02e8990ebed..378e4cc3fbf 100644--- a/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_node_execution_repository.py+++ b/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_node_execution_repository.py@@ -18,6 +18,7 @@ from sqlalchemy.exc import IntegrityErrorfrom sqlalchemy.orm import Session, sessionmakerfrom configs import dify_config+from core.repositories.celery_workflow_node_execution_repository import CeleryWorkflowNodeExecutionRepositoryfrom core.repositories.factory import OrderConfigfrom core.repositories.sqlalchemy_workflow_node_execution_repository import (SQLAlchemyWorkflowNodeExecutionRepository,@@ -315,7 +316,9 @@ def test_save_execution_data_updates_existing_and_creates_missing(repo = _repository(monkeypatch, sqlite_session_factory)existing = _execution(inputs={"initial": True},- process_data={"workflow_agent_binding_id": "binding-1"},+ process_data={+ "workflow_agent_binding_id": "binding-1",+ },)repo.save(existing)existing.inputs = {"updated": True}@@ -450,6 +453,44 @@ def test_get_by_workflow_execution_maps_real_rows_to_domain(assert domains[0].outputs == {"output": 2}+ "repository_type", [SQLAlchemyWorkflowNodeExecutionRepository, CeleryWorkflowNodeExecutionRepository]+)+ "triggered_from",+ [WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN],+)+def test_resume_history_includes_paused_nodes_without_changing_default_reads(+ sqlite_session_factory: sessionmaker[Session],+ repository_type: type[SQLAlchemyWorkflowNodeExecutionRepository | CeleryWorkflowNodeExecutionRepository],+ triggered_from: WorkflowNodeExecutionTriggeredFrom,+) -> None:+ user = _account()+ writer = SQLAlchemyWorkflowNodeExecutionRepository(+ sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=triggered_from,+ )+ writer.save_synchronously(_execution(execution_id="finished", node_execution_id="finished"))+ paused = _execution(execution_id="paused", node_execution_id="paused", status=WorkflowNodeExecutionStatus.PAUSED)+ writer.save_synchronously(paused)+ reader = repository_type(+ sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=triggered_from,+ )+ assert {node.id for node in reader.get_by_workflow_execution("run-1")} == {"finished"}+ assert {node.id for node in reader.get_by_workflow_execution("run-1", include_paused=True)} == {+ "finished",+ "paused",+ }+ assert {node.id for node in reader.get_by_workflow_execution("run-1")} == {"finished"}++def test_to_domain_model_loads_offloaded_storage(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> None:diff --git a/api/tests/unit_tests/core/repositories/test_workflow_control_persistence.py b/api/tests/unit_tests/core/repositories/test_workflow_control_persistence.pynew file mode 100644index 00000000000..1d724abc6f4--- /dev/null+++ b/api/tests/unit_tests/core/repositories/test_workflow_control_persistence.py@@ -0,0 +1,328 @@+"""Configured history backends retain synchronous, monotonic task control."""++import pytest+from pytest_mock import MockerFixture+from sqlalchemy.orm import Session, sessionmaker++from core.repositories.celery_workflow_execution_repository import CeleryWorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from extensions.logstore.aliyun_logstore import AliyunLogStore+from extensions.logstore.repositories.logstore_api_workflow_node_execution_repository import (+ LogstoreAPIWorkflowNodeExecutionRepository,+)+from extensions.logstore.repositories.logstore_api_workflow_run_repository import LogstoreAPIWorkflowRunRepository+from extensions.logstore.repositories.logstore_workflow_execution_repository import LogstoreWorkflowExecutionRepository+from extensions.logstore.repositories.logstore_workflow_node_execution_repository import (+ LogstoreWorkflowNodeExecutionRepository,+)+from graphon.entities import WorkflowNodeExecution+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType+from libs.datetime_utils import naive_utc_now+from models import Account, WorkflowRun+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.workflow import WorkflowNodeExecutionTriggeredFrom+from repositories.entities.workflow_execution import WorkflowExecution+from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository+from repositories.workflow_run_control import apply_workflow_stop_to_node+from tasks.workflow_execution_tasks import _update_workflow_run_from_execution+++def logstore(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture) -> AliyunLogStore:+ monkeypatch.setattr(AliyunLogStore, "_instance", None)+ monkeypatch.setattr(AliyunLogStore, "_initialized", False)+ monkeypatch.setenv("ALIYUN_SLS_ENDPOINT", "https://example.com")+ mocker.patch.object(AliyunLogStore, "_check_endpoint_connectivity")+ store = AliyunLogStore()+ store._use_pg_protocol = True+ mocker.patch.object(store, "put_log")+ return store++[email protected]("backend", ["sql", "celery", "logstore"])+def test_task_owner_is_durable_before_start_and_delayed_writes_cannot_revive_stop(+ sqlite_session_factory: sessionmaker[Session], logstore: AliyunLogStore, mocker: MockerFixture, backend: str+) -> None:+ queued_save = mocker.patch(+ "core.repositories.celery_workflow_execution_repository.save_workflow_execution_task.delay"+ )+ get_logs = mocker.patch.object(logstore, "get_logs")+ repository_class = {+ "sql": SQLAlchemyWorkflowExecutionRepository,+ "celery": CeleryWorkflowExecutionRepository,+ "logstore": LogstoreWorkflowExecutionRepository,+ }[backend]+ user = Account(name="Test", email="[email protected]")+ user.id = "account-1"+ repository = repository_class(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ )+ if isinstance(repository, LogstoreWorkflowExecutionRepository):+ repository._enable_dual_write = False+ execution = WorkflowExecution(+ id_="run-1",+ workflow_id="workflow-1",+ workflow_type=WorkflowType.WORKFLOW,+ workflow_version="1",+ graph={"nodes": [], "edges": []},+ inputs={},+ started_at=naive_utc_now(),+ )+ repository.save_synchronously(execution)+ queued_save.assert_not_called()+ control = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ control.bind_workflow_task(tenant_id="tenant-1", app_id="app-1", workflow_run_id="run-1", task_id="task-1")+ with sqlite_session_factory() as session:+ run = session.get(WorkflowRun, "run-1")+ assert run is not None+ assert run.task_id == "task-1"+ assert (run.created_by_role, run.created_by) == (CreatorUserRole.ACCOUNT, "account-1")+ execution.status = WorkflowExecutionStatus.PAUSED+ repository.save_synchronously(execution)+ stopped = control.stop_paused_workflow_task(tenant_id="tenant-1", app_id="app-1", task_id="task-1", owner=None)+ assert stopped is not None+ assert stopped.status == WorkflowExecutionStatus.STOPPED++ # A writer already queued or running before the stop cannot restore RUNNING.+ execution.status = WorkflowExecutionStatus.RUNNING+ repository.save_synchronously(execution)+ with sqlite_session_factory.begin() as session:+ run = session.get(WorkflowRun, "run-1")+ assert run is not None+ _update_workflow_run_from_execution(run, execution)+ with sqlite_session_factory() as session:+ run = session.get(WorkflowRun, "run-1")+ assert run is not None+ assert run.status == WorkflowExecutionStatus.STOPPED+ assert run.stop_requested_at is not None+ if backend == "logstore":+ # LogStore implements the configured run-control subset of the history protocol.+ api = LogstoreAPIWorkflowRunRepository(sqlite_session_factory) # pyrefly: ignore[bad-instantiation]+ api._enable_dual_read = False+ found = api.get_workflow_run_by_id("tenant-1", "app-1", "run-1")+ assert found is not None+ assert found.status == WorkflowExecutionStatus.STOPPED+ unscoped = api.get_workflow_run_by_id_without_tenant("run-1")+ assert unscoped is not None+ assert unscoped.status == WorkflowExecutionStatus.STOPPED+ get_logs.assert_not_called()++[email protected]("logstore")+def test_logstore_pause_lifecycle_uses_sql_control_with_dual_write_disabled(+ sqlite_session_factory: sessionmaker[Session], mocker: MockerFixture+) -> None:+ mocker.patch("repositories.sqlalchemy_api_workflow_run_repository.storage.save")+ mocker.patch("repositories.sqlalchemy_api_workflow_run_repository.storage.delete")+ user = Account(name="Test", email="[email protected]")+ user.id = "account-1"+ sql = SQLAlchemyWorkflowExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ )+ sql.save_synchronously(+ WorkflowExecution(+ id_="run-1",+ workflow_id="workflow-1",+ workflow_type=WorkflowType.WORKFLOW,+ workflow_version="1",+ graph={},+ inputs={},+ started_at=naive_utc_now(),+ )+ )+ api = LogstoreAPIWorkflowRunRepository(sqlite_session_factory) # pyrefly: ignore[bad-instantiation]+ api.bind_workflow_task(tenant_id="tenant-1", app_id="app-1", workflow_run_id="run-1", task_id="task-1")+ pause = api.create_workflow_pause("run-1", "account-1", "{}", [])+ persisted_pause = api.get_workflow_pause("run-1")+ assert persisted_pause is not None+ assert persisted_pause.id == pause.id+ resume_callbacks: list[None] = []+ resumed = api.resume_workflow_pause("run-1", pause, before_resume=lambda: resume_callbacks.append(None))+ assert resume_callbacks == [None]+ api.delete_workflow_pause(resumed)+ assert api.get_workflow_pause("run-1") is None+++ ("record_type", "node_status"),+ [+ ("workflow", WorkflowNodeExecutionStatus.PAUSED),+ ("node", WorkflowNodeExecutionStatus.PAUSED),+ ("node", WorkflowNodeExecutionStatus.RETRY),+ ],+)+def test_logstore_stop_orders_before_delayed_upload_and_overrides_stale_indexes(+ sqlite_session_factory: sessionmaker[Session],+ logstore: AliyunLogStore,+ mocker: MockerFixture,+ record_type: str,+ node_status: WorkflowNodeExecutionStatus,+) -> None:+ user = Account(name="Test", email="[email protected]")+ user.id = "account-1"+ run = WorkflowExecution(+ accumulated_elapsed_time=3.5,+ id_="run-1",+ workflow_id="workflow-1",+ workflow_type=WorkflowType.WORKFLOW,+ workflow_version="1",+ graph={},+ inputs={},+ started_at=naive_utc_now(),+ )+ run.status = WorkflowExecutionStatus.PAUSED+ SQLAlchemyWorkflowExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ ).save(run)+ control = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ control.bind_workflow_task(tenant_id="tenant-1", app_id="app-1", workflow_run_id="run-1", task_id="task-1")+ node = WorkflowNodeExecution(+ id="node-1",+ node_execution_id="node-1",+ workflow_id="workflow-1",+ workflow_execution_id="run-1",+ index=1,+ node_id="review",+ node_type="human-input",+ title="Review",+ created_at=run.started_at,+ status=node_status,+ )+ SQLAlchemyWorkflowNodeExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ ).save_synchronously(node)+ records: list[dict[str, str]] = []+ interrupted = False++ def append_after_concurrent_stop(_store: str, fields: list[tuple[str, str]]) -> None:+ nonlocal interrupted+ if not interrupted:+ interrupted = True+ stopped = control.stop_paused_workflow_task(+ tenant_id="tenant-1", app_id="app-1", task_id="task-1", owner=None+ )+ assert stopped is not None+ if record_type == "workflow":+ run_writer.save(+ run.model_copy(+ update={+ "status": WorkflowExecutionStatus.STOPPED,+ "finished_at": stopped.finished_at,+ "error_message": stopped.error,+ }+ )+ )+ else:+ node_writer.save(+ node.model_copy(+ update={+ "status": WorkflowNodeExecutionStatus.FAILED,+ "finished_at": stopped.finished_at,+ "error": stopped.error,+ }+ )+ )+ records.append(dict(fields))++ mocker.patch.object(logstore, "put_log", side_effect=append_after_concurrent_stop)+ run_writer = LogstoreWorkflowExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ )+ node_writer = LogstoreWorkflowNodeExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=user,+ app_id="app-1",+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ )+ run_writer._enable_dual_write = node_writer._enable_dual_write = False+ if record_type == "workflow":+ run_writer.save(run)+ expected_status = "stopped"+ assert max(records, key=lambda row: int(row["log_version"]))["status"] == expected_status+ run.accumulated_elapsed_time = 999.0+ run_writer.save(run) # Another late RUNNING/PAUSED write also remains terminal.+ assert float(records[-1]["elapsed_time"]) == 3.5+ else:+ node_writer.save(node)+ expected_status = "failed"+ assert max(records, key=lambda row: int(row["log_version"]))["status"] == expected_status+ stale_node = next(row for row in records if row["status"] == node_status)+ mocker.patch.object(logstore, "execute_sql", return_value=[stale_node])+ api = LogstoreAPIWorkflowNodeExecutionRepository(sqlite_session_factory) # pyrefly: ignore[bad-instantiation]+ assert api.get_executions_by_workflow_run("tenant-1", "app-1", "run-1")[0].status == expected_status+ by_id = api.get_execution_by_id("node-1", "tenant-1")+ assert by_id is not None+ assert by_id.status == expected_status+ assert node_writer.get_by_workflow_execution("run-1")[0].status == expected_status+ node_writer.save(node)+ assert records[-1]["status"] == expected_status+++ "status",+ [+ WorkflowNodeExecutionStatus.SUCCEEDED,+ WorkflowNodeExecutionStatus.FAILED,+ WorkflowNodeExecutionStatus.EXCEPTION,+ WorkflowNodeExecutionStatus.STOPPED,+ ],+)+def test_stop_preserves_completed_node_history(status: WorkflowNodeExecutionStatus) -> None:+ finished_at = naive_utc_now()+ stopped = WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.STOPPED,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ error="User requested stop",+ finished_at=finished_at,+ stop_requested_at=finished_at,+ )+ node = WorkflowNodeExecution(+ id="node-1",+ workflow_id="workflow-1",+ workflow_execution_id="run-1",+ index=1,+ node_id="answer",+ node_type="llm",+ title="Answer",+ status=status,+ outputs={"answer": "preserved"},+ error=None if status == WorkflowNodeExecutionStatus.SUCCEEDED else "Original terminal error",+ elapsed_time=2.5,+ created_at=finished_at,+ finished_at=finished_at,+ )+ original = node.model_dump()++ apply_workflow_stop_to_node(node, stopped)++ assert node.model_dump() == originaldiff --git a/api/tests/unit_tests/core/test_model_manager.py b/api/tests/unit_tests/core/test_model_manager.pyindex fd275b71404..18e647a77b5 100644--- a/api/tests/unit_tests/core/test_model_manager.py+++ b/api/tests/unit_tests/core/test_model_manager.py@@ -17,10 +17,15 @@ from core.entities.provider_entities import ()from core.errors.error import ModelCurrentlyNotSupportErrorfrom core.model_manager import LBModelManager, ModelInstance, ModelManager, QuotaManagedModelInstance+from core.plugin.impl.model_runtime_factory import create_plugin_model_runtimefrom extensions.ext_redis import redis_client+from graphon.model_runtime.entities.common_entities import I18nObjectfrom graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsagefrom graphon.model_runtime.entities.message_entities import AssistantPromptMessagefrom graphon.model_runtime.entities.model_entities import ModelType+from graphon.model_runtime.entities.provider_entities import ProviderEntity+from graphon.model_runtime.model_providers.base.tts_model import TTSModel+from graphon.model_runtime.protocols.tts_runtime import TTSChunkfrom models.provider import ProviderType@@ -432,6 +437,30 @@ def test_quota_managed_tts_commits_before_first_chunk() -> None:reservation.release.assert_called_once_with()+def test_model_instance_adapts_tts_chunks_to_audio_bytes() -> None:+ manager, bundle = _build_model_manager_bundle(+ provider_type=ProviderType.CUSTOM,+ restrict_models=[],+ model_type=ModelType.TTS,+ )+ tts_model = TTSModel(+ provider_schema=ProviderEntity(+ provider="openai",+ label=I18nObject(en_US="OpenAI"),+ supported_model_types=[ModelType.TTS],+ configurate_methods=[],+ ),+ model_runtime=create_plugin_model_runtime(tenant_id="tenant-1"),+ )+ bundle.model_type_instance = tts_model+ model_instance = manager.get_model_instance("tenant-1", "openai", ModelType.TTS, "tts-model")++ with patch.object(+ tts_model.model_runtime, "invoke_tts", return_value=iter([TTSChunk(data=b"audio", mime_type="audio/wav")])+ ):+ assert list(model_instance.invoke_tts(content_text="hello", voice="voice")) == [b"audio"]++def test_quota_managed_tts_releases_when_provider_fails_before_first_chunk() -> None:manager, _ = _build_model_manager_bundle(provider_type=ProviderType.SYSTEM,diff --git a/api/tests/unit_tests/core/tools/test_tool_manager.py b/api/tests/unit_tests/core/tools/test_tool_manager.pyindex fb6b3f54f3b..6e2737eea2d 100644--- a/api/tests/unit_tests/core/tools/test_tool_manager.py+++ b/api/tests/unit_tests/core/tools/test_tool_manager.py@@ -23,14 +23,21 @@ from core.helper.provider_encryption import ProviderConfigEncrypterfrom core.plugin.entities.plugin_daemon import CredentialTypefrom core.plugin.impl.exc import PluginDaemonNotFoundError, PluginNotFoundErrorfrom core.tools.__base.tool_runtime import ToolRuntime+from core.tools.entities.common_entities import I18nObjectfrom core.tools.entities.tool_entities import (+ ToolEntity,+ ToolIdentity,ToolInvokeFrom,ToolParameter,+ ToolProviderEntity,+ ToolProviderIdentity,ToolProviderType,)from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundErrorfrom core.tools.plugin_tool.provider import PluginToolProviderControllerfrom core.tools.tool_manager import ToolManager+from core.tools.workflow_as_tool.provider import WorkflowToolProviderController+from core.tools.workflow_as_tool.tool import WorkflowToolfrom graphon.runtime import VariablePoolfrom models.base import TypeBasefrom models.tools import ApiToolProvider, BuiltinToolProvider, WorkflowToolProvider@@ -598,23 +605,50 @@ def test_get_tool_runtime_workflow_path(monkeypatch: pytest.MonkeyPatch, tool_datool_database.session.add(workflow_provider)tool_database.session.commit()monkeypatch.setattr("core.tools.tool_manager.db", tool_database)- workflow_tool = Mock()- workflow_tool.fork_tool_runtime.return_value = "wf-runtime"- workflow_controller = Mock()- workflow_controller.get_tools.return_value = [workflow_tool]+ workflow_tool = WorkflowTool(+ workflow_app_id=workflow_provider.app_id,+ workflow_as_tool_id=provider_id,+ version=workflow_provider.version,+ workflow_entities={},+ workflow_call_depth=0,+ entity=ToolEntity(+ identity=ToolIdentity(author="test", name="wf", label=I18nObject(en_US="Workflow"), provider=provider_id),+ ),+ runtime=ToolRuntime(tenant_id=tenant_id),+ )+ workflow_controller = WorkflowToolProviderController(+ entity=ToolProviderEntity(+ identity=ToolProviderIdentity(+ author="test",+ name=workflow_provider.name,+ label=I18nObject(en_US=workflow_provider.label),+ description=I18nObject(en_US=workflow_provider.description),+ icon=workflow_provider.icon,+ ),+ ),+ provider_id=provider_id,+ )+ workflow_controller.tools = [workflow_tool]with patch("core.tools.tool_manager.ToolTransformService.workflow_provider_to_controller",return_value=workflow_controller,):- assert (- ToolManager.get_tool_runtime(- provider_type=ToolProviderType.WORKFLOW,- provider_id=provider_id,- tool_name="wf",- tenant_id=tenant_id,- )- == "wf-runtime"+ runtime_tool = ToolManager.get_tool_runtime(+ provider_type=ToolProviderType.WORKFLOW,+ provider_id=provider_id,+ tool_name="wf",+ tenant_id=tenant_id,)+ assert isinstance(runtime_tool, WorkflowTool)+ assert runtime_tool is not workflow_tool+ assert runtime_tool.entity == workflow_tool.entity+ assert runtime_tool.workflow_as_tool_id == provider_id+ assert runtime_tool.runtime == ToolRuntime(+ tenant_id=tenant_id,+ invoke_from=InvokeFrom.DEBUGGER,+ tool_invoke_from=ToolInvokeFrom.AGENT,+ )+ assert runtime_tool.execution_driver is Nonedef test_get_tool_runtime_plugin_path():@@ -705,6 +739,7 @@ def test_get_agent_runtime_apply_runtime_parameters():invoke_from=InvokeFrom.DEBUGGER,tool_invoke_from=ToolInvokeFrom.AGENT,credential_id=None,+ execution_driver=None,)@@ -752,6 +787,7 @@ def test_get_workflow_runtime_apply_runtime_parameters():invoke_from=InvokeFrom.DEBUGGER,tool_invoke_from=ToolInvokeFrom.WORKFLOW,credential_id=None,+ execution_driver=None,)@@ -815,6 +851,7 @@ def test_get_tool_runtime_from_plugin_only_uses_form_parameters():invoke_from=InvokeFrom.SERVICE_API,tool_invoke_from=ToolInvokeFrom.PLUGIN,credential_id=None,+ execution_driver=None,)diff --git a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.pyindex ba4fb38fbd0..ca8c5192bd2 100644--- a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py+++ b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py@@ -8,7 +8,8 @@ from typing import Anyfrom unittest.mock import patchimport pytest-from sqlalchemy import Engine, inspect+from flask import Flask+from sqlalchemy import Engine, inspect, selectfrom sqlalchemy.orm import Session, sessionmakerfrom core.app.entities.app_invoke_entities import InvokeFrom@@ -22,14 +23,19 @@ from core.tools.entities.tool_entities import (ToolProviderType,)from core.tools.errors import ToolInvokeError+from core.tools.tool_manager import ToolManagerfrom core.tools.workflow_as_tool import tool as workflow_tool_modulefrom core.tools.workflow_as_tool.tool import WorkflowToolfrom graphon.file import FILE_MODEL_IDENTITY, File, FileTransferMethod, FileTypefrom models.account import Account, Tenant, TenantAccountJoin, TenantAccountRolefrom models.base import TypeBase+from models.engine import dbfrom models.enums import EndUserTypefrom models.model import App, AppMode, EndUser-from models.workflow import Workflow, WorkflowType+from models.tools import WorkflowToolProvider+from models.workflow import Workflow, WorkflowRun, WorkflowType+from services.workflow_run_agg import WorkflowRunAgg+from tests.unit_tests.model_factories import make_accountTENANT_ID = "00000000-0000-0000-0000-000000000001"OTHER_TENANT_ID = "00000000-0000-0000-0000-000000000002"@@ -170,19 +176,86 @@ def _build_tool(*, tenant_id: str = "test_tool", workflow_app_id: str = "app-1",)runtime = ToolRuntime(tenant_id=tenant_id, invoke_from=InvokeFrom.EXPLORE)return WorkflowTool(+ execution_driver=WorkflowRunAgg.run,workflow_app_id=workflow_app_id,+ workflow_entities={},workflow_as_tool_id="wf-tool-1",version=version,- workflow_entities={},workflow_call_depth=1,entity=entity,runtime=runtime,)+def test_legacy_workflow_tool_executes_with_the_upper_owned_driver(+ sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], sqlite_session: Session+) -> None:+ tool_db = SqliteToolDb(sqlite_engine, sqlite_session_factory, sqlite_session)+ _persist_tenant(tool_db)+ user = _persist_end_user(tool_db)+ app_model = _persist_app(tool_db)+ workflow = _persist_workflow(tool_db, version="1")+ workflow.graph = json.dumps(+ {+ "nodes": [+ {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}},+ {+ "id": "end",+ "data": {+ "type": "end",+ "title": "End",+ "outputs": [{"variable": "caller", "value_selector": ["sys", "user_id"]}],+ },+ },+ ],+ "edges": [{"id": "start-end", "source": "start", "target": "end"}],+ }+ )+ app_model.workflow_id = workflow.id+ provider = WorkflowToolProvider(+ name="legacy_workflow",+ label="Legacy workflow",+ icon='{"background":"#222","content":"W"}',+ app_id=app_model.id,+ version=workflow.version,+ user_id=CREATOR_ID,+ tenant_id=TENANT_ID,+ description="Workflow tool",+ parameter_configuration="[]",+ )+ sqlite_session.add(provider)+ sqlite_session.commit()+ flask_app = Flask(__name__)+ flask_app.config["SQLALCHEMY_DATABASE_URI"] = str(sqlite_engine.url)+ db.init_app(flask_app)++ with flask_app.app_context():+ tool = ToolManager.get_tool_runtime(+ provider_type=ToolProviderType.WORKFLOW,+ provider_id=provider.id,+ tool_name=provider.name,+ tenant_id=TENANT_ID,+ user_id=user.id,+ invoke_from=InvokeFrom.SERVICE_API,+ execution_driver=WorkflowRunAgg.run,+ )+ messages = list(tool.invoke(session=sqlite_session, user_id=user.id, tool_parameters={}))++ assert any(+ isinstance(message.message, ToolInvokeMessage.VariableMessage)+ and message.message.variable_name == "caller"+ and message.message.variable_value == user.session_id+ for message in messages+ )+ run = sqlite_session.scalar(select(WorkflowRun).where(WorkflowRun.app_id == app_model.id))+ assert run is not None+ assert run.status == "succeeded"+ assert run.outputs_dict == {"caller": user.session_id}++def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_field(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure that WorkflowTool will throw a `ToolInvokeError` exception when`WorkflowAppGenerator.generate` returns a result with `error` key inside@@ -194,8 +267,7 @@ def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_fielmonkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- # Resolve a persisted account without exercising the lookup in this behavior test.- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)# replace `WorkflowAppGenerator.generate` 's return value.@@ -207,13 +279,13 @@ def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_fielwith pytest.raises(ToolInvokeError) as exc_info:# WorkflowTool always returns a generator, so we need to iterate to# actually `run` the tool.- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ list(tool.invoke(sqlite_session, "test_user", {}))assert exc_info.value.args == ("oops",)def test_workflow_tool_does_not_use_pause_state_config(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure pause_state_config is passed as None."""tool = _build_tool()@@ -221,14 +293,13 @@ def test_workflow_tool_does_not_use_pause_state_config(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ list(tool.invoke(sqlite_session, "test_user", {}))call_kwargs = generate_calls[-1]assert "pause_state_config" in call_kwargs@@ -237,7 +308,7 @@ def test_workflow_tool_does_not_use_pause_state_config(def test_workflow_tool_passes_parent_trace_context_from_runtime(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure nested workflow runtime metadata is forwarded as parent trace context."""tool = _build_tool()@@ -249,14 +320,13 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ list(tool.invoke(sqlite_session, "test_user", {}))call_kwargs = generate_calls[-1]assert call_kwargs["args"]["parent_trace_context"].model_dump() == {@@ -267,7 +337,7 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime(def test_workflow_tool_passes_parent_trace_session_id(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure nested workflows inherit the parent observability session ID."""tool = _build_tool()@@ -284,14 +354,13 @@ def test_workflow_tool_passes_parent_trace_session_id(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {"trace_session_id": "user-input-session"}))+ list(tool.invoke(sqlite_session, "test_user", {"trace_session_id": "user-input-session"}))call_kwargs = generate_calls[-1]assert call_kwargs["args"]["inputs"]["trace_session_id"] == "user-input-session"@@ -300,7 +369,7 @@ def test_workflow_tool_passes_parent_trace_session_id(def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure private trace context does not overwrite same-named workflow inputs."""tool = _build_tool()@@ -326,16 +395,15 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)list(tool.invoke(- sqlite_tool_db.caller_session,+ sqlite_session,"test_user",{"outer_workflow_run_id": "user-workflow-input",@@ -355,7 +423,7 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys(def test_workflow_tool_can_clear_parent_trace_context(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure reused WorkflowTool instances do not keep stale parent trace context."""tool = _build_tool()@@ -368,14 +436,13 @@ def test_workflow_tool_can_clear_parent_trace_context(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ list(tool.invoke(sqlite_session, "test_user", {}))call_kwargs = generate_calls[-1]assert "parent_trace_context" not in call_kwargs["args"]@@ -383,7 +450,7 @@ def test_workflow_tool_can_clear_parent_trace_context(def test_workflow_tool_can_clear_trace_session_id(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure reused WorkflowTool instances do not keep stale trace session IDs."""tool = _build_tool()@@ -393,14 +460,13 @@ def test_workflow_tool_can_clear_trace_session_id(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ list(tool.invoke(sqlite_session, "test_user", {}))call_kwargs = generate_calls[-1]assert "trace_session_id" not in call_kwargs["args"]@@ -417,8 +483,8 @@ def test_workflow_tool_can_clear_trace_session_id()def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete(monkeypatch: pytest.MonkeyPatch,+ sqlite_session: Session,runtime_parameters: dict[str, Any],- sqlite_tool_db: SqliteToolDb,):"""Ensure incomplete runtime metadata does not leak parent trace context into generator args."""tool = _build_tool()@@ -427,14 +493,13 @@ def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ list(tool.invoke(sqlite_session, "test_user", {}))call_kwargs = generate_calls[-1]assert "parent_trace_context" not in call_kwargs["args"]@@ -442,7 +507,7 @@ def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete(def test_workflow_tool_should_generate_variable_messages_for_outputs(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Test that WorkflowTool should generate variable messages when there are outputs"""tool = _build_tool()@@ -454,8 +519,7 @@ def test_workflow_tool_should_generate_variable_messages_for_outputs(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- # Resolve a persisted account without exercising the lookup in this behavior test.- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)# replace `WorkflowAppGenerator.generate` 's return value.@@ -463,10 +527,9 @@ def test_workflow_tool_should_generate_variable_messages_for_outputs("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate",lambda *args, **kwargs: {"data": {"outputs": mock_outputs}},)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)# Execute tool invocation- messages = list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ messages = list(tool.invoke(sqlite_session, "test_user", {}))# Verify variable messagesvariable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE]@@ -491,7 +554,7 @@ def test_workflow_tool_should_generate_variable_messages_for_outputs(def test_workflow_tool_should_handle_empty_outputs(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Test that WorkflowTool should handle empty outputs correctly"""tool = _build_tool()@@ -500,8 +563,7 @@ def test_workflow_tool_should_handle_empty_outputs(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- # Resolve a persisted account without exercising the lookup in this behavior test.- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)# replace `WorkflowAppGenerator.generate` 's return value.@@ -509,10 +571,9 @@ def test_workflow_tool_should_handle_empty_outputs("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate",lambda *args, **kwargs: {"data": {}},)- monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None)# Execute tool invocation- messages = list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {}))+ messages = list(tool.invoke(sqlite_session, "test_user", {}))# Verify generated messages# Should contain: 0 variable messages + 1 text message + 1 JSON message = 2 messages@@ -647,7 +708,7 @@ def test_extract_usage_from_nested():def test_invoke_raises_when_user_not_found(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Raise ToolInvokeError when user resolution fails."""tool = _build_tool()@@ -656,7 +717,7 @@ def test_invoke_raises_when_user_not_found(monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: None)with pytest.raises(ToolInvokeError, match="User not found"):- list(tool.invoke(sqlite_tool_db.caller_session, "missing", {}))+ list(tool.invoke(sqlite_session, "missing", {}))def test_resolve_user_from_database_returns_account(sqlite_tool_db: SqliteToolDb):@@ -812,7 +873,7 @@ def test_transform_args_normalizes_optional_files_parameter(def test_workflow_tool_invocation_normalizes_optional_files_parameter(monkeypatch: pytest.MonkeyPatch,- sqlite_tool_db: SqliteToolDb,+ sqlite_session: Session,):"""Ensure casted empty FILES values do not reach workflow input validation as [None]."""tool = _build_tool()@@ -827,13 +888,13 @@ def test_workflow_tool_invocation_normalizes_optional_files_parameter(monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None)monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None)- user = _persist_account(sqlite_tool_db)+ user = make_account(account_id="test_user")monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user)generate, generate_calls = _record_calls(return_value={"data": {}})monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate)- list(tool.invoke(sqlite_tool_db.caller_session, "test_user", {"images": None}))+ list(tool.invoke(sqlite_session, "test_user", {"images": None}))call_kwargs = generate_calls[-1]assert call_kwargs["args"]["inputs"]["images"] == []diff --git a/api/tests/unit_tests/core/workflow/context/test_execution_context.py b/api/tests/unit_tests/core/workflow/context/test_execution_context.pyindex 1359209083b..272f1fb5b5c 100644--- a/api/tests/unit_tests/core/workflow/context/test_execution_context.py+++ b/api/tests/unit_tests/core/workflow/context/test_execution_context.py@@ -2,7 +2,7 @@import contextvarsimport threading-from contextlib import contextmanager, suppress+from contextlib import contextmanager, nullcontext, suppressfrom typing import Anyfrom unittest.mock import MagicMock@@ -88,27 +88,47 @@ class TestExecutionContext:assert ctx.context_vars is Noneassert ctx.user is None- def test_enter_with_context_vars(self):- """Test enter restores context variables."""+ @pytest.mark.parametrize("use_enter", [False, True])+ @pytest.mark.parametrize("caller_value", [None, "caller"])+ @pytest.mark.parametrize("raise_error", [False, True])+ def test_enter_with_context_vars(self, use_enter, caller_value, raise_error):+ """Restore the caller's binding, including an unset variable, on every exit."""test_var = contextvars.ContextVar("test_var")- test_var.set("original_value")-- # Copy context with the variable- context_vars = contextvars.copy_context()-- # Change the variable- test_var.set("new_value")-- # Create execution context and enter it+ context_vars = contextvars.Context()+ context_vars.run(test_var.set, "captured")ctx = ExecutionContext(context_vars=context_vars)-- with ctx.enter():- # Variable should be restored to original value- assert test_var.get() == "original_value"-- # After exiting, variable stays at the value from within the context- # (this is expected Python contextvars behavior)- assert test_var.get() == "original_value"+ token = test_var.set(caller_value) if caller_value is not None else None++ try:+ with pytest.raises(ValueError, match="body failed") if raise_error else nullcontext():+ with ctx.enter() if use_enter else ctx:+ assert test_var.get() == "captured"+ test_var.set("body")+ if raise_error:+ raise ValueError("body failed")++ assert test_var.get(None) == caller_value+ assert (test_var in contextvars.copy_context()) == (caller_value is not None)+ finally:+ if token is not None:+ test_var.reset(token)++ @pytest.mark.parametrize(("outer_enter", "inner_enter"), [(False, False), (False, True), (True, False)])+ def test_nested_context_restores_each_scope(self, outer_enter, inner_enter):+ test_var = contextvars.ContextVar("nested_var")+ captured = contextvars.Context()+ captured.run(test_var.set, "captured")+ ctx = ExecutionContext(context_vars=captured)++ with ctx.enter() if outer_enter else ctx:+ assert test_var.get() == "captured"+ test_var.set("outer")+ with ctx.enter() if inner_enter else ctx:+ assert test_var.get() == "captured"+ test_var.set("inner")+ assert test_var.get() == "outer"++ assert test_var not in contextvars.copy_context()def test_enter_with_app_context(self):"""Test enter enters app context if available."""diff --git a/api/tests/unit_tests/core/workflow/context/test_flask_app_context.py b/api/tests/unit_tests/core/workflow/context/test_flask_app_context.pyindex 503f1f9c338..d8ee03e3fef 100644--- a/api/tests/unit_tests/core/workflow/context/test_flask_app_context.py+++ b/api/tests/unit_tests/core/workflow/context/test_flask_app_context.py@@ -1,6 +1,7 @@"""Tests for Flask app context module."""import contextvars+from contextlib import nullcontextimport pytestfrom flask import Flask, current_app, g@@ -130,7 +131,8 @@ class TestCaptureFlaskContext:with flask_app.app_context():ctx = capture_flask_context()- assert ctx._flask_app is flask_app+ with ctx:+ assert current_app._get_current_object() is flask_appdef test_capture_flask_context_captures_user_from_g(self, flask_app):from context.flask_app_context import capture_flask_context@@ -168,33 +170,87 @@ class TestCaptureFlaskContext:class TestFlaskExecutionContextIntegration:"""Integration tests for FlaskExecutionContext."""- def test_enter_restores_context_vars(self, flask_app):- """Test that enter restores captured context variables."""- # Create a context variable and set a value+ @pytest.mark.parametrize("use_enter", [False, True])+ @pytest.mark.parametrize("caller_value", [None, "caller"])+ @pytest.mark.parametrize("raise_error", [False, True])+ def test_enter_restores_context_vars(self, flask_app, use_enter, caller_value, raise_error):+ """Restore the caller's binding on normal and exceptional exits."""test_var = contextvars.ContextVar("integration_test_var")- test_var.set("original_value")-- # Capture the context- context_vars = contextvars.copy_context()-- # Change the value- test_var.set("new_value")-- # Create FlaskExecutionContext and enter it+ context_vars = contextvars.Context()+ context_vars.run(test_var.set, "captured")from context.flask_app_context import FlaskExecutionContextctx = FlaskExecutionContext(flask_app=flask_app,context_vars=context_vars,)+ token = test_var.set(caller_value) if caller_value is not None else None- with ctx:- # Value should be restored to original- assert test_var.get() == "original_value"+ try:+ with pytest.raises(ValueError, match="body failed") if raise_error else nullcontext():+ with ctx.enter() if use_enter else ctx:+ assert test_var.get() == "captured"+ test_var.set("body")+ if raise_error:+ raise ValueError("body failed")++ assert test_var.get(None) == caller_value+ assert (test_var in contextvars.copy_context()) == (caller_value is not None)+ finally:+ if token is not None:+ test_var.reset(token)- # After exiting, variable stays at the value from within the context- # (this is expected Python contextvars behavior)- assert test_var.get() == "original_value"+ @pytest.mark.parametrize(("outer_enter", "inner_enter"), [(False, False), (False, True), (True, False)])+ def test_nested_context_restores_each_scope(self, outer_enter, inner_enter):+ from context.flask_app_context import FlaskExecutionContext++ flask_app = Flask(__name__)+ caller_app = current_app._get_current_object()+ caller_g = g._get_current_object()+ test_var = contextvars.ContextVar("nested_var")+ captured = contextvars.copy_context()+ captured.run(test_var.set, "captured")+ ctx = FlaskExecutionContext(flask_app=flask_app, context_vars=captured)++ with ctx.enter() if outer_enter else ctx:+ assert current_app == flask_app+ test_var.set("outer")+ outer_g = g._get_current_object()+ with ctx.enter() if inner_enter else ctx:+ assert current_app == flask_app+ assert g._get_current_object() is not outer_g+ assert test_var.get() == "captured"+ test_var.set("inner")+ assert g._get_current_object() is outer_g+ assert test_var.get() == "outer"++ assert test_var not in contextvars.copy_context()+ assert current_app == caller_app+ assert g._get_current_object() is caller_g++ @pytest.mark.parametrize("use_enter", [False, True])+ @pytest.mark.parametrize("failure_stage", ["enter", "teardown"])+ def test_context_vars_restored_when_flask_context_fails(self, flask_app, monkeypatch, use_enter, failure_stage):+ from context.flask_app_context import FlaskExecutionContext++ test_var = contextvars.ContextVar("failure_var")+ captured = contextvars.Context()+ captured.run(test_var.set, "captured")+ ctx = FlaskExecutionContext(flask_app=flask_app, context_vars=captured)++ def fail_context(_exception=None):+ raise ValueError("context failed")++ with monkeypatch.context() as patch:+ method = "app_context" if failure_stage == "enter" else "do_teardown_appcontext"+ patch.setattr(flask_app, method, fail_context)+ with pytest.raises(ValueError, match="context failed"), ctx.enter() if use_enter else ctx:+ assert test_var.get() == "captured"++ assert test_var not in contextvars.copy_context()+ with ctx:+ assert test_var.get() == "captured"+ assert test_var not in contextvars.copy_context()def test_enter_enters_flask_app_context(self, flask_app):"""Test that enter enters Flask app context."""diff --git a/api/tests/unit_tests/core/workflow/graph_engine/layers/conftest.py b/api/tests/unit_tests/core/workflow/graph_engine/layers/conftest.pyindex 025d79b25df..6459379ddc2 100644--- a/api/tests/unit_tests/core/workflow/graph_engine/layers/conftest.py+++ b/api/tests/unit_tests/core/workflow/graph_engine/layers/conftest.py@@ -8,7 +8,6 @@ import pytestfrom opentelemetry.sdk.trace import TracerProviderfrom opentelemetry.sdk.trace.export import SimpleSpanProcessorfrom opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter-from opentelemetry.trace import set_tracer_providerfrom graphon.enums import BuiltinNodeTypes@@ -20,21 +19,16 @@ def memory_span_exporter():@pytest.fixture-def tracer_provider_with_memory_exporter(memory_span_exporter):+def tracer_provider_with_memory_exporter(memory_span_exporter, monkeypatch):"""Provide a TracerProvider configured with memory exporter."""- import opentelemetry.trace as trace_api-- trace_api._TRACER_PROVIDER = None- trace_api._TRACER_PROVIDER_SET_ONCE._done = False-provider = TracerProvider()processor = SimpleSpanProcessor(memory_span_exporter)provider.add_span_processor(processor)- set_tracer_provider(provider)+ monkeypatch.setattr("core.app.workflow.layers.observability.get_tracer", provider.get_tracer)yield provider- provider.force_flush()+ provider.shutdown()@pytest.fixture@@ -117,7 +111,7 @@ def mock_result_event():"""Create a mock result event with NodeRunResult."""from datetime import datetime- from graphon.graph_events import NodeRunSucceededEvent+ from graphon.engine_events import NodeRunSucceededEventfrom graphon.node_events import NodeRunResultnode_run_result = NodeRunResult(diff --git a/api/tests/unit_tests/core/workflow/graph_engine/layers/test_observability.py b/api/tests/unit_tests/core/workflow/graph_engine/layers/test_observability.pyindex ccb0cdff7a0..94acff901bc 100644--- a/api/tests/unit_tests/core/workflow/graph_engine/layers/test_observability.py+++ b/api/tests/unit_tests/core/workflow/graph_engine/layers/test_observability.py@@ -10,13 +10,22 @@ Test coverage:- Disabled mode behavior"""+from concurrent.futures import ThreadPoolExecutor+from threading import Event+import pytest-from opentelemetry.trace import StatusCode+from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor+from opentelemetry.trace import StatusCode, get_current_spanfrom core.app.workflow.layers.observability import ObservabilityLayer+from core.workflow.workflow_entry import WorkflowEntryfrom extensions.otel.semconv import DifySpanAttributes-from graphon.enums import BuiltinNodeTypes-from graphon.graph_events import GraphRunAbortedEvent+from graphon.engine_events import GraphRunAbortedEvent, GraphRunSucceededEvent+from graphon.graph import Graph+from graphon.nodes.start import StartNode+from graphon.nodes.start.entities import StartNodeData+from graphon.runtime import RuntimeState, VariablePool+from tests.workflow_test_utils import build_test_graph_init_params@pytest.fixture(autouse=True)@@ -24,27 +33,52 @@ def _otel_config(config_overrides) -> None:config_overrides(ENABLE_OTEL=True)-class TestObservabilityLayerInitialization:- """Test ObservabilityLayer initialization logic."""-- @pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")- def test_initialization_when_otel_enabled(self, tracer_provider_with_memory_exporter):- """Test that layer initializes correctly when OTel is enabled."""- layer = ObservabilityLayer()- assert not layer._is_disabled- assert layer._tracer is not None- assert BuiltinNodeTypes.TOOL in layer._parsers- assert layer._default_parser is not None-- @pytest.mark.usefixtures("mock_is_instrument_flag_enabled_true")- def test_initialization_when_instrument_flag_enabled(self, tracer_provider_with_memory_exporter, config_overrides):- """Test that layer enables when instrument flag is enabled."""- config_overrides(ENABLE_OTEL=False)- layer = ObservabilityLayer()- assert not layer._is_disabled- assert layer._tracer is not None- assert BuiltinNodeTypes.TOOL in layer._parsers- assert layer._default_parser is not None[email protected]("tracing_enabled", [False, True])+def test_unbound_node_still_runs_and_preserves_body_errors(config_overrides, tracing_enabled):+ config_overrides(ENABLE_OTEL=tracing_enabled)+ node = StartNode(+ node_id="start",+ data=StartNodeData(title="Start", variables=[]),+ init_params=build_test_graph_init_params(),+ runtime_state=RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=0),+ )+ layer = ObservabilityLayer()+ parent = get_current_span()+ executed = []++ with layer.node_run_context(node):+ executed.append("completed")++ with pytest.raises(ValueError, match="node failed"), layer.node_run_context(node):+ raise ValueError("node failed")++ assert executed == ["completed"]+ assert get_current_span() is parent++[email protected]("mock_is_instrument_flag_enabled_true", "tracer_provider_with_memory_exporter")+def test_instrument_flag_enables_workflow_node_spans(memory_span_exporter, config_overrides, monkeypatch):+ config_overrides(ENABLE_OTEL=False, GRAPH_ENGINE_MAX_WORKERS=1)+ monkeypatch.setattr("core.workflow.workflow_entry.is_instrument_flag_enabled", lambda: True)+ state = RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=0)+ node = StartNode(+ node_id="start",+ data=StartNodeData(title="Start", variables=[]),+ init_params=build_test_graph_init_params(),+ runtime_state=state,+ )+ entry = WorkflowEntry(+ graph_config={},+ graph=Graph.new().add_root(node).build(),+ call_depth=0,+ graph_runtime_state=state,+ )++ events = list(entry.run())++ assert isinstance(events[-1], GraphRunSucceededEvent)+ spans = memory_span_exporter.get_finished_spans()+ assert [span.name for span in spans] == [node.title]class TestObservabilityLayerNodeSpanLifecycle:@@ -58,8 +92,8 @@ class TestObservabilityLayerNodeSpanLifecycle:layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_llm_node)- layer.on_node_run_end(mock_llm_node, None)+ with layer.node_run_context(mock_llm_node):+ layer.on_node_run_end(mock_llm_node, None)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -75,8 +109,8 @@ class TestObservabilityLayerNodeSpanLifecycle:layer.on_graph_start()error = ValueError("Test error")- layer.on_node_run_start(mock_llm_node)- layer.on_node_run_end(mock_llm_node, error)+ with layer.node_run_context(mock_llm_node):+ layer.on_node_run_end(mock_llm_node, error)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -97,6 +131,64 @@ class TestObservabilityLayerNodeSpanLifecycle:spans = memory_span_exporter.get_finished_spans()assert len(spans) == 0+ @pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")+ def test_suspension_deactivates_span_until_resume(+ self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_tool_node+ ):+ layer = ObservabilityLayer()+ parent = get_current_span()++ with layer.node_run_context(mock_tool_node):+ tool_span = get_current_span()+ assert tool_span.is_recording()++ assert get_current_span() is parent+ assert memory_span_exporter.get_finished_spans() == ()++ with layer.node_run_context(mock_tool_node):+ assert get_current_span() is tool_span+ layer.on_node_run_end(mock_tool_node, None)++ assert get_current_span() is parent+ spans = memory_span_exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].context == tool_span.get_span_context()++ @pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")+ def test_activation_does_not_swallow_node_errors(+ self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_llm_node+ ):+ layer = ObservabilityLayer()+ parent = get_current_span()+ error = ValueError("node failed")++ with pytest.raises(ValueError, match="node failed"):+ with layer.node_run_context(mock_llm_node):+ raise error++ assert get_current_span() is parent+ layer.on_node_run_end(mock_llm_node, error)+ spans = memory_span_exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.ERROR++ @pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")+ def test_parser_failure_still_ends_span(+ self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_start_node, monkeypatch, caplog+ ):+ layer = ObservabilityLayer()++ def fail_parse(**_kwargs):+ raise ValueError("invalid node attributes")++ monkeypatch.setattr(layer._default_parser, "parse", fail_parse)++ with layer.node_run_context(mock_start_node):+ layer.on_node_run_end(mock_start_node, None)++ assert len(memory_span_exporter.get_finished_spans()) == 1+ assert "invalid node attributes" in caplog.text+class TestObservabilityLayerParserIntegration:"""Test parser integration for different node types."""@@ -109,8 +201,8 @@ class TestObservabilityLayerParserIntegration:layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_start_node)- layer.on_node_run_end(mock_start_node, None)+ with layer.node_run_context(mock_start_node):+ layer.on_node_run_end(mock_start_node, None)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -127,8 +219,8 @@ class TestObservabilityLayerParserIntegration:layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_tool_node)- layer.on_node_run_end(mock_tool_node, None)+ with layer.node_run_context(mock_tool_node):+ layer.on_node_run_end(mock_tool_node, None)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -159,8 +251,8 @@ class TestObservabilityLayerParserIntegration:layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_llm_node)- layer.on_node_run_end(mock_llm_node, None, mock_result_event)+ with layer.node_run_context(mock_llm_node):+ layer.on_node_run_end(mock_llm_node, None, mock_result_event)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -191,8 +283,8 @@ class TestObservabilityLayerParserIntegration:layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_retrieval_node)- layer.on_node_run_end(mock_retrieval_node, None, mock_result_event)+ with layer.node_run_context(mock_retrieval_node):+ layer.on_node_run_end(mock_retrieval_node, None, mock_result_event)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -218,8 +310,8 @@ class TestObservabilityLayerParserIntegration:layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_start_node)- layer.on_node_run_end(mock_start_node, None, mock_result_event)+ with layer.node_run_context(mock_start_node):+ layer.on_node_run_end(mock_start_node, None, mock_result_event)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -232,47 +324,78 @@ class TestObservabilityLayerGraphLifecycle:"""Test graph lifecycle management."""@pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")- def test_on_graph_start_clears_contexts(self, tracer_provider_with_memory_exporter, mock_llm_node):- """Test that on_graph_start clears node contexts."""+ def test_on_graph_end_with_no_unfinished_spans(+ self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_llm_node+ ):+ """Test that on_graph_end handles normal completion."""layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_llm_node)- assert len(layer._node_contexts) == 1+ with layer.node_run_context(mock_llm_node):+ layer.on_node_run_end(mock_llm_node, None)+ layer.on_graph_end(None)- layer.on_graph_start()- assert len(layer._node_contexts) == 0+ spans = memory_span_exporter.get_finished_spans()+ assert len(spans) == 1@pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")- def test_on_graph_end_with_no_unfinished_spans(+ def test_on_graph_end_ends_suspended_spans(self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_llm_node):- """Test that on_graph_end handles normal completion."""+ """Pause and abort cleanup must export spans that never reached node end."""layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_llm_node)- layer.on_node_run_end(mock_llm_node, None)+ with layer.node_run_context(mock_llm_node):+ span = get_current_span()++ layer.on_graph_end(None)layer.on_graph_end(None)+ assert not span.is_recording()spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1+ assert spans[0].name == mock_llm_node.title@pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")- def test_on_graph_end_with_unfinished_spans_logs_warning(- self, tracer_provider_with_memory_exporter, mock_llm_node, caplog+ def test_graph_end_and_late_worker_each_end_spans_once(+ self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_start_node, mock_llm_node, caplog):- """Test that on_graph_end logs warning for unfinished spans."""- layer = ObservabilityLayer()- layer.on_graph_start()+ """An aborted graph can finish while a worker still owns another span."""+ graph_ending = Event()+ worker_finished = Event()- layer.on_node_run_start(mock_llm_node)- assert len(layer._node_contexts) == 1+ class EndBarrier(SpanProcessor):+ def on_end(self, span: ReadableSpan) -> None:+ if span.name == mock_start_node.title:+ graph_ending.set()+ assert worker_finished.wait(5)- layer.on_graph_end(None)+ tracer_provider_with_memory_exporter.add_span_processor(EndBarrier())+ layer = ObservabilityLayer()+ for node in (mock_start_node, mock_llm_node):+ with layer.node_run_context(node):+ pass++ def finish_worker() -> None:+ assert graph_ending.wait(5)+ try:+ layer.on_node_run_end(mock_llm_node, None)+ finally:+ worker_finished.set()++ with ThreadPoolExecutor(max_workers=1) as executor:+ worker = executor.submit(finish_worker)+ try:+ layer.on_graph_end(None)+ finally:+ worker.result(timeout=5)- assert len(layer._node_contexts) == 0- assert "node spans were not properly ended" in caplog.text+ layer.on_graph_end(None)+ spans = memory_span_exporter.get_finished_spans()+ assert len(spans) == 2+ assert {span.name for span in spans} == {mock_start_node.title, mock_llm_node.title}+ assert "Calling end() on an ended span" not in caplog.text@pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")def test_graph_aborted_event_records_reason_on_current_span(@@ -280,10 +403,9 @@ class TestObservabilityLayerGraphLifecycle:):layer = ObservabilityLayer()layer.on_graph_start()- layer.on_node_run_start(mock_start_node)-- layer.on_event(GraphRunAbortedEvent(reason="worker shutdown", outputs={}))- layer.on_node_run_end(mock_start_node, None)+ with layer.node_run_context(mock_start_node):+ layer.on_event(GraphRunAbortedEvent(reason="worker shutdown", outputs={}))+ layer.on_node_run_end(mock_start_node, None)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 1@@ -299,25 +421,27 @@ class TestObservabilityLayerDisabledMode:"""Test behavior when layer is disabled."""@pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")- def test_disabled_mode_skips_node_start(self, memory_span_exporter, mock_start_node, config_overrides):+ def test_disabled_mode_skips_node_start(+ self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_start_node, config_overrides+ ):"""Test that disabled layer doesn't create spans on node start."""config_overrides(ENABLE_OTEL=False)layer = ObservabilityLayer()- assert layer._is_disabledlayer.on_graph_start()- layer.on_node_run_start(mock_start_node)- layer.on_node_run_end(mock_start_node, None)+ with layer.node_run_context(mock_start_node):+ layer.on_node_run_end(mock_start_node, None)spans = memory_span_exporter.get_finished_spans()assert len(spans) == 0@pytest.mark.usefixtures("mock_is_instrument_flag_enabled_false")- def test_disabled_mode_skips_node_end(self, memory_span_exporter, mock_llm_node, config_overrides):+ def test_disabled_mode_skips_node_end(+ self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_llm_node, config_overrides+ ):"""Test that disabled layer doesn't process node end."""config_overrides(ENABLE_OTEL=False)layer = ObservabilityLayer()- assert layer._is_disabledlayer.on_node_run_end(mock_llm_node, None)diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.py b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.pyindex 9c507bb1619..c3114fb47b9 100644--- a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.py+++ b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_factory.py@@ -29,8 +29,7 @@ from .test_mock_nodes import ()if TYPE_CHECKING:- from graphon.entities import GraphInitParams- from graphon.runtime import GraphRuntimeState+ from graphon.runtime import InitParams, RuntimeStatefrom .test_mock_config import MockConfig@@ -46,18 +45,18 @@ class MockNodeFactory(DifyNodeFactory):def __init__(self,- graph_init_params: "GraphInitParams",- graph_runtime_state: "GraphRuntimeState",+ init_params: "InitParams",+ runtime_state: "RuntimeState",mock_config: "MockConfig | None" = None,) -> None:"""Initialize the mock node factory.- :param graph_init_params: Graph initialization parameters- :param graph_runtime_state: Graph runtime state+ :param init_params: Graph initialization parameters+ :param runtime_state: Graph runtime state:param mock_config: Optional mock configuration for customizing mock behavior"""- super().__init__(graph_init_params, graph_runtime_state)+ super().__init__(init_params, runtime_state)self.mock_config = mock_config# Map of node types that should be mocked@@ -77,10 +76,10 @@ class MockNodeFactory(DifyNodeFactory):}@override- def with_runtime_state(self, graph_runtime_state: "GraphRuntimeState") -> "MockNodeFactory":+ def with_runtime_state(self, runtime_state: "RuntimeState") -> "MockNodeFactory":return MockNodeFactory(- graph_init_params=self.graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=self.init_params,+ runtime_state=runtime_state,mock_config=self.mock_config,)@@ -105,8 +104,8 @@ class MockNodeFactory(DifyNodeFactory):mock_instance = mock_class(node_id=node_id,data=resolved_node_data,- graph_init_params=self.graph_init_params,- graph_runtime_state=self.graph_runtime_state,+ init_params=self.init_params,+ runtime_state=self.runtime_state,mock_config=self.mock_config,code_executor=self._code_executor,code_limits=self._code_limits,@@ -115,8 +114,8 @@ class MockNodeFactory(DifyNodeFactory):mock_instance = mock_class(node_id=node_id,data=resolved_node_data,- graph_init_params=self.graph_init_params,- graph_runtime_state=self.graph_runtime_state,+ init_params=self.init_params,+ runtime_state=self.runtime_state,mock_config=self.mock_config,http_request_config=self._http_request_config,http_client=self._http_request_http_client,@@ -131,8 +130,8 @@ class MockNodeFactory(DifyNodeFactory):mock_instance = mock_class(node_id=node_id,data=resolved_node_data,- graph_init_params=self.graph_init_params,- graph_runtime_state=self.graph_runtime_state,+ init_params=self.init_params,+ runtime_state=self.runtime_state,mock_config=self.mock_config,credentials_provider=self._llm_credentials_provider,model_factory=self._llm_model_factory,@@ -141,8 +140,8 @@ class MockNodeFactory(DifyNodeFactory):mock_instance = mock_class(node_id=node_id,data=resolved_node_data,- graph_init_params=self.graph_init_params,- graph_runtime_state=self.graph_runtime_state,+ init_params=self.init_params,+ runtime_state=self.runtime_state,mock_config=self.mock_config,)diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes.py b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes.pyindex 542411c5656..8e77e23d8f6 100644--- a/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes.py+++ b/api/tests/unit_tests/core/workflow/graph_engine/test_mock_nodes.py@@ -40,8 +40,7 @@ from graphon.template_rendering import Jinja2TemplateRenderer, TemplateRenderErrfrom tests.unit_tests.core.model_fixtures import make_model_instanceif TYPE_CHECKING:- from graphon.entities import GraphInitParams- from graphon.runtime import GraphRuntimeState+ from graphon.runtime import InitParams, RuntimeStatefrom .test_mock_config import MockConfig@@ -66,12 +65,12 @@ class MockNodeMixin:node_id: str,data: Any,*,- graph_init_params: "GraphInitParams",- graph_runtime_state: "GraphRuntimeState",+ init_params: "InitParams",+ runtime_state: "RuntimeState",mock_config: Optional["MockConfig"] = None,**kwargs: Any,) -> None:- run_context = resolve_dify_run_context(graph_init_params.run_context)+ run_context = resolve_dify_run_context(init_params.run_context)if isinstance(self, (LLMNode, QuestionClassifierNode, ParameterExtractorNode)):kwargs.setdefault("credentials_provider", DifyCredentialsProvider(run_context=run_context))kwargs.setdefault("model_factory", DifyModelFactory(run_context=run_context))@@ -99,7 +98,7 @@ class MockNodeMixin:if isinstance(self, _ToolNode):kwargs.setdefault("tool_file_manager", DifyToolFileManager(run_context))- kwargs.setdefault("runtime", DifyToolNodeRuntime(graph_init_params.run_context))+ kwargs.setdefault("runtime", DifyToolNodeRuntime(init_params.run_context))if isinstance(self, AgentNode):presentation_provider = MagicMock()@@ -112,8 +111,8 @@ class MockNodeMixin:super().__init__(node_id=node_id,data=data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,**kwargs,)self.mock_config = mock_config@@ -659,7 +658,7 @@ class MockTemplateTransformNode(MockNodeMixin, TemplateTransformNode):if hasattr(self._node_data, "variables"):for variable_selector in self._node_data.variables:variable_name = variable_selector.variable- value = self.graph_runtime_state.variable_pool.get(variable_selector.value_selector)+ value = self.runtime_state.variable_pool.get(variable_selector.value_selector)variables[variable_name] = value.to_object() if value else None# Check if we have custom mock outputs configureddiff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_parallel_human_input_join_resume.py b/api/tests/unit_tests/core/workflow/graph_engine/test_parallel_human_input_join_resume.pyindex 86960db44c1..14fac8cdbdd 100644--- a/api/tests/unit_tests/core/workflow/graph_engine/test_parallel_human_input_join_resume.py+++ b/api/tests/unit_tests/core/workflow/graph_engine/test_parallel_human_input_join_resume.py@@ -23,44 +23,44 @@ from core.workflow.nodes.human_input.entities import ()from core.workflow.nodes.human_input.enums import HumanInputFormStatus, ValueSourceTypefrom core.workflow.system_variables import build_system_variables-from graphon.entities import WorkflowStartReason-from graphon.file import File, FileTransferMethod, FileType-from graphon.graph import Graph-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import (+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import (GraphRunPausedEvent,GraphRunStartedEvent,GraphRunSucceededEvent,NodeRunSucceededEvent,)+from graphon.entities import WorkflowStartReason+from graphon.file import File, FileTransferMethod, FileType+from graphon.graph import Graphfrom graphon.nodes.base.entities import OutputVariableEntityfrom graphon.nodes.end.end_node import EndNodefrom graphon.nodes.end.entities import EndNodeDatafrom graphon.nodes.human_input.human_input_node import HumanInputNodefrom graphon.nodes.start.entities import StartNodeDatafrom graphon.nodes.start.start_node import StartNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom libs.datetime_utils import naive_utc_nowfrom tests.workflow_test_utils import build_test_graph_init_paramsclass PauseStateStore(Protocol):- def save(self, runtime_state: GraphRuntimeState) -> None: ...+ def save(self, runtime_state: RuntimeState) -> None: ...- def load(self) -> GraphRuntimeState: ...+ def load(self) -> RuntimeState: ...class InMemoryPauseStore:def __init__(self) -> None:self._snapshot: str | None = None- def save(self, runtime_state: GraphRuntimeState) -> None:+ def save(self, runtime_state: RuntimeState) -> None:self._snapshot = runtime_state.dumps()- def load(self) -> GraphRuntimeState:+ def load(self) -> RuntimeState:assert self._snapshot is not None- return GraphRuntimeState.from_snapshot(self._snapshot)+ return RuntimeState.from_snapshot(self._snapshot)class _TestFileReferenceFactory:@@ -143,8 +143,11 @@ class StaticRepo(HumanInputFormRepository):def create_form(self, params: FormCreateParams) -> HumanInputFormEntity:raise AssertionError("create_form should not be called in resume scenario")+ def mark_timeout(self, node_id: str, *, form_id: str) -> HumanInputFormEntity:+ raise AssertionError("these resume scenarios do not advance the form deadline")+-def _build_runtime_state() -> GraphRuntimeState:+def _build_runtime_state() -> RuntimeState:variable_pool = VariablePool.from_bootstrap(system_variables=build_system_variables(user_id="user",@@ -155,10 +158,10 @@ def _build_runtime_state() -> GraphRuntimeState:user_inputs={},conversation_variables=[],)- return GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter())-def _build_graph(runtime_state: GraphRuntimeState, repo: HumanInputFormRepository) -> Graph:+def _build_graph(runtime_state: RuntimeState, repo: HumanInputFormRepository) -> Graph:graph_config: dict[str, object] = {"nodes": [], "edges": []}graph_init_params = build_test_graph_init_params(workflow_id="workflow",@@ -175,8 +178,8 @@ def _build_graph(runtime_state: GraphRuntimeState, repo: HumanInputFormRepositorstart_node = StartNode(node_id=start_config["id"],data=StartNodeData(title="Start", variables=[]),- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,)human_data = HumanInputNodeData(@@ -204,8 +207,8 @@ def _build_graph(runtime_state: GraphRuntimeState, repo: HumanInputFormRepositorhuman_a = HumanInputNode(node_id=human_a_config["id"],data=human_data,- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,hitl_callback=human_a_callback,)@@ -220,8 +223,8 @@ def _build_graph(runtime_state: GraphRuntimeState, repo: HumanInputFormRepositorhuman_b = HumanInputNode(node_id=human_b_config["id"],data=human_data,- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,hitl_callback=human_b_callback,)@@ -241,8 +244,8 @@ def _build_graph(runtime_state: GraphRuntimeState, repo: HumanInputFormRepositorend_node = EndNode(node_id=end_config["id"],data=end_data,- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,)builder = (@@ -255,18 +258,12 @@ def _build_graph(runtime_state: GraphRuntimeState, repo: HumanInputFormRepositorreturn builder.connect(tail="human_b", head="end", source_handle="approve").build()-def _run_graph(graph: Graph, runtime_state: GraphRuntimeState) -> list[object]:- engine = GraphEngine(- workflow_id="workflow",+def _run_graph(graph: Graph, runtime_state: RuntimeState) -> list[object]:+ engine = Engine(graph=graph,- graph_runtime_state=runtime_state,+ runtime_state=runtime_state,command_channel=InMemoryChannel(),- config=GraphEngineConfig(- min_workers=2,- max_workers=2,- scale_up_threshold=1,- scale_down_idle_time=30.0,- ),+ workers=2,)return list(engine.run())diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_table_runner.py b/api/tests/unit_tests/core/workflow/graph_engine/test_table_runner.pyindex fdece324442..cd2b061f792 100644--- a/api/tests/unit_tests/core/workflow/graph_engine/test_table_runner.py+++ b/api/tests/unit_tests/core/workflow/graph_engine/test_table_runner.py@@ -1,5 +1,5 @@"""-Table-driven test framework for GraphEngine workflows.+Table-driven test framework for Engine workflows.This module provides a robust table-driven testing framework with support for:- Parallel test execution@@ -25,16 +25,15 @@ from core.workflow.node_factory import DifyNodeFactory, get_default_root_node_idfrom core.workflow.system_variables import build_bootstrap_variables, build_system_variablesfrom core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_poolfrom core.workflow.workflow_entry import iter_dify_graph_engine_events-from graphon.entities import GraphInitParams-from graphon.graph import Graph-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import (- GraphEngineEvent,+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import (+ EngineEvent,GraphRunStartedEvent,GraphRunSucceededEvent,)-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.graph import Graph+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom graphon.variables import (ArrayNumberVariable,ArrayObjectVariable,@@ -63,8 +62,8 @@ class WorkflowTestCase:timeout: float = 30.0mock_config: MockConfig | None = Noneuse_auto_mock: bool = False- expected_event_sequence: Sequence[type[GraphEngineEvent]] | None = None- graph_factory: Callable[[], tuple[Graph, GraphRuntimeState]] | None = None+ expected_event_sequence: Sequence[type[EngineEvent]] | None = None+ graph_factory: Callable[[], tuple[Graph, RuntimeState]] | None = None@dataclass@@ -78,9 +77,9 @@ class WorkflowTestResult:execution_time: float = 0.0event_sequence_match: bool | None = Noneevent_mismatch_details: str | None = None- events: list[GraphEngineEvent] = field(default_factory=list)+ events: list[EngineEvent] = field(default_factory=list)graph: Graph | None = None- graph_runtime_state: GraphRuntimeState | None = None+ graph_runtime_state: RuntimeState | None = Nonevalidation_details: str | None = None@@ -143,7 +142,7 @@ class WorkflowRunner:inputs: dict[str, Any] | None = None,use_mock_factory: bool = False,mock_config: MockConfig | None = None,- ) -> tuple[Graph, GraphRuntimeState]:+ ) -> tuple[Graph, RuntimeState]:"""Create a Graph instance from fixture data."""workflow_config = fixture_data.get("workflow", {})graph_config = workflow_config.get("graph", {})@@ -151,7 +150,7 @@ class WorkflowRunner:if not graph_config:raise ValueError("Fixture missing workflow.graph configuration")- graph_init_params = GraphInitParams(+ graph_init_params = InitParams(workflow_id="test_workflow",graph_config=graph_config,run_context={@@ -214,14 +213,16 @@ class WorkflowRunner:)add_node_inputs_to_pool(variable_pool, node_id=root_node_id, inputs=root_node_inputs)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )if use_mock_factory:node_factory = MockNodeFactory(- graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state, mock_config=mock_config+ init_params=graph_init_params, runtime_state=graph_runtime_state, mock_config=mock_config)else:- node_factory = DifyNodeFactory(graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state)+ node_factory = DifyNodeFactory(init_params=graph_init_params, runtime_state=graph_runtime_state)graph = Graph.init(graph_config=graph_config,@@ -251,10 +252,7 @@ class TableTestRunner:max_workers: int = 4,enable_logging: bool = False,log_level: str = "INFO",- graph_engine_min_workers: int = 3,graph_engine_max_workers: int = 1,- graph_engine_scale_up_threshold: int = 5,- graph_engine_scale_down_idle_time: float = 30.0,):"""Initialize the table test runner.@@ -264,19 +262,12 @@ class TableTestRunner:max_workers: Maximum number of parallel workers for test executionenable_logging: Enable detailed logginglog_level: Logging level (DEBUG, INFO, WARNING, ERROR)- graph_engine_min_workers: Minimum workers for GraphEngine (default: 3)- graph_engine_max_workers: Maximum workers for GraphEngine (default: 1)- graph_engine_scale_up_threshold: Queue depth to trigger scale up- graph_engine_scale_down_idle_time: Idle time before scaling down+ graph_engine_max_workers: Maximum workers for Engine (default: 1)"""self.workflow_runner = WorkflowRunner(fixtures_dir)self.max_workers = max_workers- # Store GraphEngine worker configuration- self.graph_engine_min_workers = graph_engine_min_workersself.graph_engine_max_workers = graph_engine_max_workers- self.graph_engine_scale_up_threshold = graph_engine_scale_up_threshold- self.graph_engine_scale_down_idle_time = graph_engine_scale_down_idle_timeif enable_logging:logging.basicConfig(@@ -321,21 +312,15 @@ class TableTestRunner:graph, graph_runtime_state = self._create_graph_runtime_state(test_case)# Create and run the engine with configured worker settings- engine = GraphEngine(- workflow_id="test_workflow",+ engine = Engine(graph=graph,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,command_channel=InMemoryChannel(),- config=GraphEngineConfig(- min_workers=self.graph_engine_min_workers,- max_workers=self.graph_engine_max_workers,- scale_up_threshold=self.graph_engine_scale_up_threshold,- scale_down_idle_time=self.graph_engine_scale_down_idle_time,- ),+ workers=self.graph_engine_max_workers,)# Execute and collect events- events: list[GraphEngineEvent] = []+ events: list[EngineEvent] = []for event in iter_dify_graph_engine_events(engine):events.append(event)@@ -406,7 +391,7 @@ class TableTestRunner:graph_runtime_state=graph_runtime_state if "graph_runtime_state" in locals() else None,)- def _create_graph_runtime_state(self, test_case: WorkflowTestCase) -> tuple[Graph, GraphRuntimeState]:+ def _create_graph_runtime_state(self, test_case: WorkflowTestCase) -> tuple[Graph, RuntimeState]:"""Create or retrieve graph/runtime state according to test configuration."""if test_case.graph_factory is not None:@@ -469,7 +454,7 @@ class TableTestRunner:return True, Nonedef _validate_event_sequence(- self, expected_sequence: list[type[GraphEngineEvent]], actual_events: list[GraphEngineEvent]+ self, expected_sequence: list[type[EngineEvent]], actual_events: list[EngineEvent]) -> tuple[bool, str | None]:"""Validate that actual events match the expected event sequence.diff --git a/api/tests/unit_tests/core/workflow/graph_engine/test_tool_in_chatflow.py b/api/tests/unit_tests/core/workflow/graph_engine/test_tool_in_chatflow.pyindex 33e8f869a5a..00910502df8 100644--- a/api/tests/unit_tests/core/workflow/graph_engine/test_tool_in_chatflow.py+++ b/api/tests/unit_tests/core/workflow/graph_engine/test_tool_in_chatflow.py@@ -1,7 +1,7 @@from core.workflow.workflow_entry import iter_dify_graph_engine_events-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import (+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import (GraphRunSucceededEvent,NodeRunStreamChunkEvent,)@@ -24,12 +24,10 @@ def test_tool_in_chatflow():)# Create and run the engine- engine = GraphEngine(- workflow_id="test_workflow",+ engine = Engine(graph=graph,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,command_channel=InMemoryChannel(),- config=GraphEngineConfig(),)events = list(iter_dify_graph_engine_events(engine))@@ -78,12 +76,10 @@ def test_answer_can_render_llm_structured_output_in_chatflow():mock_config=mock_config,)- engine = GraphEngine(- workflow_id="test_workflow",+ engine = Engine(graph=graph,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,command_channel=InMemoryChannel(),- config=GraphEngineConfig(),)events = list(iter_dify_graph_engine_events(engine))diff --git a/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_entities.py b/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_entities.pyindex e94a728b68c..8257bbba161 100644--- a/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_entities.py+++ b/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_entities.py@@ -1,11 +1,10 @@from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_contextfrom core.workflow.node_factory import DifyNodeFactory-from graphon.entities import GraphInitParams+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import GraphRunSucceededEvent, NodeEventfrom graphon.graph import Graph-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import GraphNodeEventBase, GraphRunSucceededEvent-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePooldef test_unconfigured_disconnected_agent_does_not_block_workflow() -> None:@@ -16,8 +15,8 @@ def test_unconfigured_disconnected_agent_does_not_block_workflow() -> None:],"edges": [],}- graph_runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0)- graph_init_params = GraphInitParams(+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0)+ graph_init_params = InitParams(workflow_id="workflow",graph_config=graph_config,run_context=build_dify_run_context(@@ -34,15 +33,14 @@ def test_unconfigured_disconnected_agent_does_not_block_workflow() -> None:node_factory=DifyNodeFactory(graph_init_params, graph_runtime_state),root_node_id="start",)- engine = GraphEngine(- workflow_id="workflow",+ engine = Engine(graph=graph,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,command_channel=InMemoryChannel(),- config=GraphEngineConfig(min_workers=1, max_workers=1),+ workers=1,)events = list(engine.run())assert isinstance(events[-1], GraphRunSucceededEvent)- assert not any(isinstance(event, GraphNodeEventBase) and event.node_id == "agent" for event in events)+ assert not any(isinstance(event, NodeEvent) and event.node_id == "agent" for event in events)diff --git a/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_node.py b/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_node.pyindex b1596fecd1d..2ed54e5fadc 100644--- a/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/agent/test_agent_node.py@@ -3,24 +3,23 @@ from unittest.mock import MagicMockfrom core.workflow.nodes.agent.agent_node import AgentNodefrom core.workflow.nodes.agent.entities import AgentNodeDatafrom core.workflow.nodes.agent.events import AgentLogEvent, NodeRunAgentLogEvent-from graphon.entities import GraphInitParams+from graphon.engine_events import NodeRunStreamChunkEventfrom graphon.enums import BuiltinNodeTypes-from graphon.graph_events import NodeRunStreamChunkEventfrom graphon.node_events import StreamChunkEvent-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePooldef test_dispatch_converts_agent_events_and_delegates_other_events() -> None:node = AgentNode(node_id="node-id",data=AgentNodeData(title="Agent"),- graph_init_params=GraphInitParams(+ init_params=InitParams(workflow_id="workflow-id",graph_config={},run_context={},call_depth=0,),- graph_runtime_state=GraphRuntimeState(variable_pool=VariablePool(), start_at=0),+ runtime_state=RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0),strategy_resolver=MagicMock(),presentation_provider=MagicMock(),runtime_support=MagicMock(),diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.pyindex 9b94d9da4c0..10750315ccf 100644--- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py@@ -47,7 +47,6 @@ from core.workflow.nodes.agent_v2.session_store import (WorkflowAgentSessionScope,WorkflowAgentWorkspaceStore,)-from graphon.entities import GraphInitParamsfrom graphon.enums import (BuiltinNodeTypes,ErrorStrategy,@@ -57,7 +56,7 @@ from graphon.enums import ()from graphon.file import File, FileTransferMethod, FileTypefrom graphon.node_events import StreamChunkEvent, StreamCompletedEvent-from graphon.runtime import GraphRuntimeState+from graphon.runtime import InitParams, RuntimeStatefrom graphon.variables.segments import ArrayFileSegment, FileSegment, StringSegmentfrom models.agent import Agent, AgentConfigSnapshot, WorkflowAgentNodeBindingfrom models.agent_config_entities import (@@ -166,17 +165,8 @@ class FakeSessionStore:self.backend_binding_ref = backend_binding_refself.resolved_scopes: list[WorkflowAgentSessionScope] = []self.existing_scope_lookups: list[dict[str, object]] = []- # ENG-638: set to simulate resume after a submitted/timed-out form.self.loaded_session: StoredWorkflowAgentSession | None = None- self.saved: list[- tuple[- WorkflowAgentSessionScope,- str,- SessionSnapshot | None,- str | None,- str | None,- ]- ] = []+ self.saved: list[tuple[WorkflowAgentSessionScope, str, SessionSnapshot | None]] = []def load_existing_node_execution_scope(self, **kwargs: object) -> WorkflowAgentSessionScope | None:self.existing_scope_lookups.append(kwargs)@@ -418,7 +408,7 @@ def _node(runtime_request_builder: WorkflowAgentRuntimeRequestBuilder | None = None,error_strategy: ErrorStrategy | None = None,) -> DifyAgentNode:- graph_init_params = GraphInitParams(+ graph_init_params = InitParams(workflow_id="workflow-1",graph_config={"nodes": [], "edges": []},run_context={@@ -464,9 +454,9 @@ def _node(else [],}),- graph_init_params=graph_init_params,- graph_runtime_state=cast(- GraphRuntimeState,+ init_params=graph_init_params,+ runtime_state=cast(+ RuntimeState,SimpleNamespace(variable_pool=FakeVariablePool(),graph_execution=SimpleNamespace(aborted=False),@@ -955,7 +945,7 @@ def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error():def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(caplog):client = CancelFailingStreamBackendClient()node = _node(agent_backend_client=client)- node.graph_runtime_state.graph_execution = SimpleNamespace(aborted=True)+ node.runtime_state.graph_execution = SimpleNamespace(aborted=True)terminal, failure = _drain_event_stream(node)@@ -1049,7 +1039,7 @@ def test_agent_routes_initialize_through_workflow_node_factory(config_overrides(AGENT_BACKEND_USE_FAKE=True)template = _node()- factory = DifyNodeFactory(template.graph_init_params, template.graph_runtime_state)+ factory = DifyNodeFactory(template.init_params, template.runtime_state)node_data = template.node_data.model_dump(mode="python", by_alias=True)node_data.update(agent_output_routes={@@ -1058,7 +1048,7 @@ def test_agent_routes_initialize_through_workflow_node_factory(},error_strategy=error_strategy,)- # Exercise the production factory, which serializes data before construction.+ # Exercise the production factory's concrete node-data validation.node_config = {"id": "agent-node", "data": node_data}if expected_execution_type is None:with pytest.raises(ValueError, match="default-value"):@@ -1072,13 +1062,11 @@ def test_agent_routes_initialize_through_workflow_node_factory(def test_agent_routes_use_the_successful_retry_to_fan_out_and_skip_other_branches():+ from graphon.engine.scheduler import Scheduler+ from graphon.engine_events.traversal import GraphEdgeSkippedEvent, GraphEdgeTakenEventfrom graphon.enums import NodeStatefrom graphon.graph.edge import Edgefrom graphon.graph.graph import Graph- from graphon.graph_engine.graph_state_manager import GraphStateManager- from graphon.graph_engine.graph_traversal.edge_processor import EdgeProcessor- from graphon.graph_engine.graph_traversal.skip_propagator import SkipPropagator- from graphon.graph_events.traversal import GraphEdgeSkippedEvent, GraphEdgeTakenEventfrom graphon.nodes.end.end_node import EndNodefrom graphon.nodes.end.entities import EndNodeData@@ -1118,8 +1106,8 @@ def test_agent_routes_use_the_successful_retry_to_fan_out_and_skip_other_branchetarget: EndNode(node_id=target,data=EndNodeData(title=target, outputs=[]),- graph_init_params=node.graph_init_params,- graph_runtime_state=node.graph_runtime_state,+ init_params=node.init_params,+ runtime_state=node.runtime_state,)for target in edges}@@ -1130,8 +1118,7 @@ def test_agent_routes_use_the_successful_retry_to_fan_out_and_skip_other_brancheout_edges={"agent-node": list(edges)},in_edges={target: [target] for target in edges},)- state = GraphStateManager(graph, node.graph_runtime_state, "root")- ready, traversed = EdgeProcessor(graph, state, SkipPropagator(graph, state)).process_node_success(+ ready, traversed = Scheduler(graph, node.runtime_state, "root").process_node_success("agent-node", result.edge_source_handle)assert ready == ["b-end-1", "b-end-2"]@@ -1186,12 +1173,13 @@ def test_agent_route_conditions_load_variables_only_when_routing_is_enabled(enabdef test_disabled_routes_continue_through_graphon_default_values():+ from graphon.engine.event.node_failure import NodeFailureHandler+ from graphon.engine.scheduler import Scheduler+ from graphon.engine_events.node import NodeRunExceptionEvent, NodeRunFailedEventfrom graphon.graph.edge import Edgefrom graphon.graph.graph import Graph- from graphon.graph_engine.error_handler import ErrorHandler- from graphon.graph_engine.graph_traversal.edge_processor import EdgeProcessor- from graphon.graph_events.node import NodeRunExceptionEvent, NodeRunFailedEventfrom graphon.node_events import NodeRunResult+ from graphon.runtime.execution import GraphExecutionnode = _node(error_strategy=ErrorStrategy.DEFAULT_VALUE)edges = {target: Edge(id=target, tail="agent-node", head=target) for target in ["next-a", "next-b"]}@@ -1205,10 +1193,10 @@ def test_disabled_routes_continue_through_graphon_default_values():error="backend failed",node_run_result=NodeRunResult(status=WorkflowNodeExecutionStatus.FAILED, error="backend failed"),)- result = ErrorHandler(graph, MagicMock()).handle_node_failure(frame_id="root", event=failure)+ result = NodeFailureHandler(graph, GraphExecution(workflow_id="workflow-1")).handle(frame_id="root", event=failure)assert isinstance(result, NodeRunExceptionEvent)assert result.node_run_result.outputs["text"] == "fallback"- ready, traversed = EdgeProcessor(graph, MagicMock(), MagicMock()).process_node_success(+ ready, traversed = Scheduler(graph, node.runtime_state, "root").process_node_success("agent-node", result.node_run_result.edge_source_handle)assert ready == ["next-a", "next-b"]diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_workspace_retirement_layer.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_workspace_retirement_layer.pyindex 889ad056f37..0ddd5f85cb7 100644--- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_workspace_retirement_layer.py+++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_workspace_retirement_layer.py@@ -6,10 +6,10 @@ from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, Usfrom core.workflow.nodes.agent_v2 import workspace_retirement_layer as layer_modulefrom core.workflow.nodes.agent_v2.workspace_retirement_layer import WorkflowAgentWorkspaceRetirementLayerfrom core.workflow.system_variables import build_system_variables+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import GraphRunSucceededEvent, NodeRunStartedEventfrom graphon.enums import BuiltinNodeTypes-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import GraphRunSucceededEvent, NodeRunStartedEvent-from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool+from graphon.runtime import ReadOnlyRuntimeStateWrapper, RuntimeState, VariablePoolfrom libs.datetime_utils import naive_utc_now@@ -24,9 +24,10 @@ def _run_context() -> DifyRunContext:@pytest.fixture-def runtime_state() -> ReadOnlyGraphRuntimeStateWrapper:- return ReadOnlyGraphRuntimeStateWrapper(- GraphRuntimeState(+def runtime_state() -> ReadOnlyRuntimeStateWrapper:+ return ReadOnlyRuntimeStateWrapper(+ RuntimeState(+ workflow_id="workflow-1",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="workflow-run-1")),@@ -36,7 +37,7 @@ def runtime_state() -> ReadOnlyGraphRuntimeStateWrapper:def test_terminal_event_retires_workflow_workspace(- monkeypatch: pytest.MonkeyPatch, runtime_state: ReadOnlyGraphRuntimeStateWrapper+ monkeypatch: pytest.MonkeyPatch, runtime_state: ReadOnlyRuntimeStateWrapper) -> None:store = MagicMock()events: list[str] = []@@ -77,7 +78,7 @@ def test_non_terminal_event_does_not_retire_workspace(monkeypatch: pytest.Monkeydef test_terminal_retirement_failure_does_not_replace_terminal_event(- monkeypatch: pytest.MonkeyPatch, runtime_state: ReadOnlyGraphRuntimeStateWrapper+ monkeypatch: pytest.MonkeyPatch, runtime_state: ReadOnlyRuntimeStateWrapper) -> None:store = MagicMock()store.retire_workflow_run.side_effect = RuntimeError("database unavailable")diff --git a/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.py b/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.pyindex 2603e29be6d..1e6248fba5c 100644--- a/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.py+++ b/api/tests/unit_tests/core/workflow/nodes/answer/test_answer.py@@ -6,7 +6,7 @@ from core.workflow.system_variables import build_system_variablesfrom graphon.enums import WorkflowNodeExecutionStatusfrom graphon.nodes.answer.answer_node import AnswerNodefrom graphon.nodes.answer.entities import AnswerNodeData-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_params@@ -41,14 +41,15 @@ def _build_answer_node(*, answer: str, variable_pool: VariablePool) -> AnswerNodinvoke_from=InvokeFrom.DEBUGGER,call_depth=0,)- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=variable_pool,start_at=time.perf_counter(),)return AnswerNode(node_id=str(uuid.uuid4()),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,data=AnswerNodeData(title="Answer",type="answer",diff --git a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.py b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.pyindex d6987f0d49a..595b81ab397 100644--- a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node.py@@ -90,8 +90,8 @@ def test_datasource_node_delegates_to_manager_stream(mocker: MockerFixture):plugin_id="plug",datasource_name="ds",),- graph_init_params=gp,- graph_runtime_state=gs,+ init_params=gp,+ runtime_state=gs,datasource_credentials=lambda _tenant, _provider, _plugin, _credential: {},)diff --git a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.py b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.pyindex 08acd9ab747..18ab7ae577f 100644--- a/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.py+++ b/api/tests/unit_tests/core/workflow/nodes/datasource/test_datasource_node_stream.py@@ -17,7 +17,7 @@ from core.workflow.nodes.datasource.entities import DatasourceNodeDatafrom core.workflow.system_variables import build_system_variablesfrom graphon.enums import WorkflowNodeExecutionStatusfrom graphon.node_events import NodeRunResult, StreamCompletedEvent-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_params@@ -41,7 +41,11 @@ def test_node_integration_minimal_stream(mocker: MockerFixture, sqlite_engine: Euser_from=UserFrom.ACCOUNT,invoke_from=InvokeFrom.DEBUGGER,)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ variable_pool=variable_pool,+ start_at=time.perf_counter(),+ workflow_id=graph_init_params.workflow_id,+ )class _Mgr:@classmethod@@ -78,7 +82,7 @@ def test_node_integration_minimal_stream(mocker: MockerFixture, sqlite_engine: E"core.workflow.node_factory.build_data_source_credentials",return_value=SimpleNamespace(providers=SimpleNamespace(get_datasource_credentials=load_credentials)),)- factory = DifyNodeFactory(graph_init_params=graph_init_params, graph_runtime_state=graph_runtime_state)+ factory = DifyNodeFactory(init_params=graph_init_params, runtime_state=graph_runtime_state)node = factory.create_node({"id": "n", "data": node_data.model_dump()})build_credentials.assert_called_once_with(database_client=database_client)assert isinstance(node, DatasourceNode)@@ -86,8 +90,8 @@ def test_node_integration_minimal_stream(mocker: MockerFixture, sqlite_engine: Enode = DatasourceNode(node_id="n",data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,datasource_credentials=load_credentials,)diff --git a/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_node.py b/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_node.pyindex afde541beb3..8a3b8419bcd 100644--- a/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/http_request/test_http_request_node.py@@ -13,7 +13,7 @@ from graphon.enums import WorkflowNodeExecutionStatusfrom graphon.file.file_manager import file_managerfrom graphon.nodes.http_request import HTTP_REQUEST_CONFIG_FILTER_KEY, HttpRequestNode, HttpRequestNodeConfigfrom graphon.nodes.http_request.entities import HttpRequestNodeData, HttpRequestNodeTimeout, Response-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_paramsHTTP_REQUEST_CONFIG = HttpRequestNodeConfig(@@ -109,7 +109,8 @@ def _build_http_node(invoke_from=InvokeFrom.DEBUGGER,call_depth=0,)- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(user_id="user", files=[]),user_inputs={},@@ -119,8 +120,8 @@ def _build_http_node(return HttpRequestNode(node_id="http-node",data=HttpRequestNodeData.model_validate(node_data),- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,http_request_config=HTTP_REQUEST_CONFIG,http_client=ssrf_proxy,tool_file_manager_factory=ToolFileManager,diff --git a/api/tests/unit_tests/core/workflow/nodes/human_input/test_entities.py b/api/tests/unit_tests/core/workflow/nodes/human_input/test_entities.pyindex a3d555c0d60..d492fd8f270 100644--- a/api/tests/unit_tests/core/workflow/nodes/human_input/test_entities.py+++ b/api/tests/unit_tests/core/workflow/nodes/human_input/test_entities.py@@ -53,13 +53,12 @@ from core.workflow.nodes.human_input.enums import (ValueSourceType,)from core.workflow.system_variables import build_system_variables-from graphon.entities import GraphInitParamsfrom graphon.file import File, FileTransferMethod, FileTypefrom graphon.node_events import PauseRequestedEventfrom graphon.node_events.node import StreamCompletedEventfrom graphon.nodes.human_input.human_input_node import HumanInputNodefrom graphon.nodes.protocols import FileReferenceFactoryProtocol-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom graphon.variables.segments import ArrayFileSegment, FileSegment, StringSegmentfrom libs.datetime_utils import naive_utc_nowfrom models.account import TenantAccountJoin, TenantAccountRole@@ -144,6 +143,9 @@ class InMemoryHumanInputFormRepository(HumanInputFormRepository):def get_form(self, node_id: str) -> HumanInputFormEntity | None:return self._forms_by_node_id.get(node_id)+ def mark_timeout(self, node_id: str, *, form_id: str) -> HumanInputFormEntity:+ raise AssertionError(f"unexpected timeout in submission scenario: node={node_id}, form={form_id}")+def set_submission(self, *, action_id: str, form_data: Mapping[str, Any] | None = None) -> None:if not self.created_forms:raise AssertionError("no form has been created to attach submission data")@@ -175,8 +177,8 @@ def _build_human_input_node(*,node_id: str,node_data: HumanInputNodeData | Mapping[str, Any],- graph_init_params: GraphInitParams,- graph_runtime_state: GraphRuntimeState,+ init_params: InitParams,+ runtime_state: RuntimeState,runtime: DifyHumanInputNodeRuntime,) -> HumanInputNode:typed_node_data = (@@ -193,8 +195,8 @@ def _build_human_input_node(return HumanInputNode(node_id=node_id,data=typed_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,hitl_callback=callback,)@@ -497,8 +499,8 @@ class TestHumanInputNodeVariableResolution:conversation_variables=[],)variable_pool.add(("start", "name"), "Jane Doe")- runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)- graph_init_params = GraphInitParams(+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -540,8 +542,8 @@ class TestHumanInputNodeVariableResolution:node = _build_human_input_node(node_id=config["id"],node_data=config["data"],- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,runtime=runtime,)@@ -567,8 +569,8 @@ class TestHumanInputNodeVariableResolution:user_inputs={},conversation_variables=[],)- runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)- graph_init_params = GraphInitParams(+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -601,8 +603,8 @@ class TestHumanInputNodeVariableResolution:node = _build_human_input_node(node_id=config["id"],node_data=config["data"],- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,runtime=runtime,)@@ -623,8 +625,8 @@ class TestHumanInputNodeVariableResolution:user_inputs={},conversation_variables=[],)- runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)- graph_init_params = GraphInitParams(+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -661,8 +663,8 @@ class TestHumanInputNodeVariableResolution:node = _build_human_input_node(node_id=config["id"],node_data=config["data"],- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,runtime=runtime,)@@ -684,8 +686,8 @@ class TestHumanInputNodeVariableResolution:user_inputs={},conversation_variables=[],)- runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)- graph_init_params = GraphInitParams(+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -732,8 +734,8 @@ class TestHumanInputNodeVariableResolution:node = _build_human_input_node(node_id=config["id"],node_data=config["data"],- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,runtime=runtime,)@@ -798,8 +800,8 @@ class TestHumanInputNodeRenderedContent:user_inputs={},conversation_variables=[],)- runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)- graph_init_params = GraphInitParams(+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -828,8 +830,8 @@ class TestHumanInputNodeRenderedContent:node = _build_human_input_node(node_id=config["id"],node_data=config["data"],- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,runtime=runtime,)@@ -860,8 +862,8 @@ class TestHumanInputNodeRenderedContent:user_inputs={},conversation_variables=[],)- runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)- graph_init_params = GraphInitParams(+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -901,8 +903,8 @@ class TestHumanInputNodeRenderedContent:node = _build_human_input_node(node_id=config["id"],node_data=config["data"],- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,runtime=runtime,)diff --git a/api/tests/unit_tests/core/workflow/nodes/human_input/test_human_input_form_filled_event.py b/api/tests/unit_tests/core/workflow/nodes/human_input/test_human_input_form_filled_event.pyindex 11857203f88..d5e203aa12a 100644--- a/api/tests/unit_tests/core/workflow/nodes/human_input/test_human_input_form_filled_event.py+++ b/api/tests/unit_tests/core/workflow/nodes/human_input/test_human_input_form_filled_event.py@@ -56,18 +56,19 @@ from core.workflow.nodes.human_input.entities import (from core.workflow.nodes.human_input.enums import HumanInputFormStatusfrom core.workflow.system_variables import build_system_variablesfrom core.workflow.workflow_entry import WorkflowEntry, iter_dify_graph_engine_events-from graphon.entities import GraphInitParams, WorkflowStartReason-from graphon.enums import BuiltinNodeTypes-from graphon.file import File, FileTransferMethod, FileType-from graphon.filters import GraphEventFilterContext, filter_graph_events-from graphon.graph import Graph-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import (+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine.filter import EngineEventFilterContext, filter_engine_events+from graphon.engine_events import (+ EngineEvent,GraphEdgeSkippedEvent,GraphEdgeTakenEvent,- GraphEngineEvent,+ NodeRunHumanInputFormFilledEvent,)+from graphon.entities import WorkflowStartReason+from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKey+from graphon.file import File, FileTransferMethod, FileType+from graphon.graph import Graphfrom graphon.nodes.answer.answer_node import AnswerNodefrom graphon.nodes.answer.entities import AnswerNodeDatafrom graphon.nodes.end.end_node import EndNode@@ -76,7 +77,8 @@ from graphon.nodes.human_input.human_input_node import HumanInputNodefrom graphon.nodes.protocols import FileReferenceFactoryProtocolfrom graphon.nodes.start.entities import StartNodeDatafrom graphon.nodes.start.start_node import StartNode-from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool+from graphon.runtime import InitParams, ReadOnlyRuntimeStateWrapper, RuntimeState, VariablePool+from graphon.runtime.execution import ROOT_FRAME_IDfrom graphon.variables.segments import StringSegmentfrom libs.datetime_utils import naive_utc_nowfrom libs.helper import compact_generate_response@@ -119,6 +121,11 @@ class _FakeFormRepository:def get_form(self, *_args, **_kwargs):return self._form+ def mark_timeout(self, _node_id: str, *, form_id: str):+ assert form_id == self._form.id+ self._form.status = HumanInputFormStatus.TIMEOUT+ return self._form+class _TestFileReferenceFactory(FileReferenceFactoryProtocol):def build_from_mapping(self, *, mapping: Mapping[str, Any]):@@ -138,8 +145,8 @@ class _TestFileReferenceFactory(FileReferenceFactoryProtocol):def _create_human_input_node(*,config: dict,- graph_init_params: GraphInitParams,- graph_runtime_state: GraphRuntimeState,+ init_params: InitParams,+ runtime_state: RuntimeState,repo: _FakeFormRepository,) -> HumanInputNode:node_data = (@@ -155,8 +162,8 @@ def _create_human_input_node(node = HumanInputNode(node_id=config["id"],data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,hitl_callback=callback,)node.bind_execution_id("00000000-0000-4000-8000-000000000001")@@ -175,7 +182,8 @@ def _build_node(node_id: str = "node-1",) -> HumanInputNode:system_variables = build_system_variables(app_id="app", workflow_execution_id="run-1")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="workflow",variable_pool=VariablePool.from_bootstrap(system_variables=system_variables,user_inputs={},@@ -183,7 +191,7 @@ def _build_node(),start_at=0.0,)- graph_init_params = GraphInitParams(+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -255,8 +263,8 @@ def _build_node(repo = _FakeFormRepository(fake_form)return _create_human_input_node(config=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,repo=repo,)@@ -268,7 +276,8 @@ def _build_timeout_node(node_id: str = "node-1",) -> HumanInputNode:system_variables = build_system_variables(app_id="app", workflow_execution_id="run-1")- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="workflow",variable_pool=VariablePool.from_bootstrap(system_variables=system_variables,user_inputs={},@@ -276,7 +285,7 @@ def _build_timeout_node(),start_at=0.0,)- graph_init_params = GraphInitParams(+ graph_init_params = InitParams(workflow_id="workflow",graph_config={"nodes": [], "edges": []},run_context={@@ -316,23 +325,27 @@ def _build_timeout_node(repo = _FakeFormRepository(fake_form)return _create_human_input_node(config=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,repo=repo,)def _publish_node_events(node: HumanInputNode) -> list[AppQueueEvent]:- return _publish_graph_events(_filter_human_input_events(node.run(), node=node))+ return _publish_graph_events(+ _filter_human_input_events(node.run(), node=node),+ graph=Graph(root_node=node),+ runtime_state=node.runtime_state,+ )-def _filter_human_input_events(events: Iterable[GraphEngineEvent], *, node: HumanInputNode | None = None):+def _filter_human_input_events(events: Iterable[EngineEvent], *, node: HumanInputNode | None = None):node = node or _build_node()- return filter_graph_events(+ return filter_engine_events(events,- context=GraphEventFilterContext(+ context=EngineEventFilterContext(graph=Graph(root_node=node),- runtime_state=ReadOnlyGraphRuntimeStateWrapper(node.graph_runtime_state),+ runtime_state=ReadOnlyRuntimeStateWrapper(node.runtime_state),),filters=[HumanInputFormEventFilter(form_repository=HumanInputFormSubmissionRepository())],)@@ -341,10 +354,10 @@ def _filter_human_input_events(events: Iterable[GraphEngineEvent], *, node: Huma@pytest.mark.parametrize("event_type", [GraphEdgeTakenEvent, GraphEdgeSkippedEvent])def test_human_input_filter_forwards_traversals_without_waiting_for_completion(event_type):started, _ = list(_build_node().run())- edge = event_type(edge_id="human-answer", source_node_id="node-1", target_node_id="answer")+ edge = event_type(frame_id=ROOT_FRAME_ID, edge_id="human-answer", source_node_id="node-1", target_node_id="answer")edge_forwarded = False- def source() -> Generator[GraphEngineEvent]:+ def source() -> Generator[EngineEvent]:yield startedyield edgeassert edge_forwarded, "The traversal must be forwarded before consuming more upstream events"@@ -361,15 +374,35 @@ def test_form_events_keep_titles_for_interleaved_executions_of_one_node():started, succeeded = list(node.run())first_start = started.model_copy(update={"id": "first", "node_title": "First approval"})second_start = started.model_copy(update={"id": "second", "node_title": "Second approval"})- first_result = succeeded.model_copy(update={"id": "first", "in_loop_id": "loop-1"})- second_result = succeeded.model_copy(update={"id": "second", "in_iteration_id": "iteration-2"})-- events = _publish_graph_events(- _filter_human_input_events([first_start, second_start, second_result, first_result], node=node)+ first_result = succeeded.model_copy(+ update={+ "id": "first",+ "container_id": "loop-1",+ "node_run_result": succeeded.node_run_result.model_copy(+ update={"metadata": {WorkflowNodeExecutionMetadataKey.LOOP_ID: "loop-1"}}+ ),+ }+ )+ second_result = succeeded.model_copy(+ update={+ "id": "second",+ "container_id": "iteration-2",+ "node_run_result": succeeded.node_run_result.model_copy(+ update={"metadata": {WorkflowNodeExecutionMetadataKey.ITERATION_ID: "iteration-2"}}+ ),+ })+ filtered = list(_filter_human_input_events([first_start, second_start, second_result, first_result], node=node))+ form_events = [event for event in filtered if isinstance(event, NodeRunHumanInputFormFilledEvent)]+ assert [(event.id, event.container_id, event.node_run_result.metadata) for event in form_events] == [+ ("second", "iteration-2", second_result.node_run_result.metadata),+ ("first", "loop-1", first_result.node_run_result.metadata),+ ]+ events = _publish_graph_events(filtered, graph=Graph(root_node=node), runtime_state=node.runtime_state)+filled = [event for event in events if isinstance(event, QueueHumanInputFormFilledEvent)]- assert [(event.node_execution_id, event.node_title) for event in filled] == [+ assert [(event.form_id, event.node_title) for event in filled] == [("second", "Second approval"),("first", "First approval"),]@@ -383,7 +416,7 @@ def test_form_events_keep_titles_for_interleaved_executions_of_one_node():class _EventQueue(AppQueueManager):"""Record events and replay a finite stream without starting a watchdog."""- def __init__(self, invoke_from: InvokeFrom, runtime_state: GraphRuntimeState) -> None:+ def __init__(self, invoke_from: InvokeFrom, runtime_state: RuntimeState) -> None:self.invoke_from = invoke_fromself.graph_runtime_state = runtime_stateself.events: list[AppQueueEvent] = []@@ -397,26 +430,19 @@ class _EventQueue(AppQueueManager):yield WorkflowQueueMessage(task_id="task-1", app_mode=AppMode.ADVANCED_CHAT, event=event)-def _publish_graph_events(events: Iterable[GraphEngineEvent]) -> list[AppQueueEvent]:- node = _build_node()- runtime_state = node.graph_runtime_state+def _publish_graph_events(+ events: Iterable[EngineEvent], *, graph: Graph, runtime_state: RuntimeState+) -> list[AppQueueEvent]:queue_manager = _EventQueue(InvokeFrom.WEB_APP, runtime_state)runner = WorkflowBasedAppRunner(queue_manager=queue_manager, app_id="app")workflow_entry = WorkflowEntry(- tenant_id="tenant",- app_id="app",- workflow_id="workflow",graph_config={},- graph=Graph(root_node=node),- user_id="user",- user_from=UserFrom.ACCOUNT,- invoke_from=InvokeFrom.WEB_APP,+ graph=graph,call_depth=0,- variable_pool=runtime_state.variable_pool,graph_runtime_state=runtime_state,)for event in events:- runner._handle_event(workflow_entry, event)+ runner.handle_event(workflow_entry, event)return queue_manager.events@@ -425,7 +451,7 @@ def _sse_payloads(invoke_from: InvokeFrom,app: Flask,sessions: sessionmaker[Session],- runtime_state: GraphRuntimeState | None = None,+ runtime_state: RuntimeState | None = None,) -> list[dict[str, Any]]:generate_entity = AdvancedChatAppGenerateEntity(task_id="task-1",@@ -444,7 +470,8 @@ def _sse_payloads(workflow_run_id="run-1",)if runtime_state is None:- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run-1")),@@ -474,7 +501,7 @@ def _sse_payloads(if isinstance(event, QueueHumanInputFormFilledEvent):_persist_form(sessions,- form_id=event.node_execution_id,+ form_id=event.form_id,app_id="app",node_id=event.node_id,expiration_time=naive_utc_now() + datetime.timedelta(days=1),@@ -530,6 +557,7 @@ def test_submitted_human_input_reaches_response_stream(]payload = payloads[1]assert payload["workflow_run_id"] == "run-1"+ assert payload["data"]["form_id"] == "00000000-0000-4000-8000-000000000001"assert payload["data"]["node_id"] == "node-1"assert payload["data"]["node_title"] == "Human Input"assert payload["data"]["action_id"] == "Accept"@@ -589,6 +617,7 @@ def test_timed_out_human_input_reaches_response_stream("node_finished",]assert payloads[1]["data"] == {+ "form_id": "00000000-0000-4000-8000-000000000001","node_id": "node-1","node_title": "Human Input","expiration_time": 1735689600,@@ -606,28 +635,28 @@ def test_human_input_completion_and_referenced_answer_reach_response_stream(if timed_outelse _build_node("Approve?", with_inputs=False, node_id="human"))- runtime_state = node.graph_runtime_state+ runtime_state = node.runtime_stateruntime_state.variable_pool.add(["sys", "workflow_execution_id"], StringSegment(value="run-1"))- init_params = GraphInitParams(workflow_id="workflow", graph_config={}, run_context={}, call_depth=0)+ init_params = InitParams(workflow_id="workflow", graph_config={}, run_context={}, call_depth=0)start = StartNode(node_id="start",data=StartNodeData(title="Start", variables=[]),- graph_init_params=init_params,- graph_runtime_state=runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)if terminal == "answer":terminal_node = AnswerNode(node_id="answer",data=AnswerNodeData(title="Answer", answer="Action: {{#human.__action_id#}}"),- graph_init_params=init_params,- graph_runtime_state=runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)else:terminal_node = EndNode(node_id="end",data=EndNodeData(title="End", outputs=[]),- graph_init_params=init_params,- graph_runtime_state=runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)graph = (Graph.new()@@ -636,12 +665,11 @@ def test_human_input_completion_and_referenced_answer_reach_response_stream(.add_node(terminal_node, from_node_id=node.id, source_handle="__timeout" if timed_out else "Accept").build())- engine = GraphEngine(- workflow_id="workflow",+ engine = Engine(graph=graph,- graph_runtime_state=runtime_state,+ runtime_state=runtime_state,command_channel=InMemoryChannel(),- config=GraphEngineConfig(min_workers=1, max_workers=1),+ workers=1,)bind_execution_id = node.bind_execution_id@@ -658,7 +686,7 @@ def test_human_input_completion_and_referenced_answer_reach_response_stream(mocker.patch.object(node, "bind_execution_id", side_effect=bind_form)- events = _publish_graph_events(iter_dify_graph_engine_events(engine))+ events = _publish_graph_events(iter_dify_graph_engine_events(engine), graph=graph, runtime_state=runtime_state)payloads = _sse_payloads(events, InvokeFrom.WEB_APP, app, sqlite_session_factory, runtime_state)form_event = "human_input_form_timeout" if timed_out else "human_input_form_filled"diff --git a/api/tests/unit_tests/core/workflow/nodes/knowledge_index/test_knowledge_index_node.py b/api/tests/unit_tests/core/workflow/nodes/knowledge_index/test_knowledge_index_node.pyindex 41d9af344c2..f279c25b7b9 100644--- a/api/tests/unit_tests/core/workflow/nodes/knowledge_index/test_knowledge_index_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/knowledge_index/test_knowledge_index_node.py@@ -19,14 +19,14 @@ from core.workflow.nodes.knowledge_index.protocols import ()from core.workflow.system_variables import SystemVariableKey, build_system_variablesfrom graphon.enums import WorkflowNodeExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variables.segments import StringSegmentfrom tests.workflow_test_utils import build_test_graph_init_params@pytest.fixturedef mock_graph_init_params():- """Create mock GraphInitParams."""+ """Create mock InitParams."""return build_test_graph_init_params(workflow_id=str(uuid.uuid4()),graph_config={},@@ -41,14 +41,14 @@ def mock_graph_init_params():@pytest.fixturedef mock_graph_runtime_state():- """Create mock GraphRuntimeState."""+ """Create mock RuntimeState."""variable_pool = VariablePool.from_bootstrap(system_variables=build_system_variables(user_id=str(uuid.uuid4()), files=[]),user_inputs={},environment_variables=[],conversation_variables=[],)- return GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter())@pytest.fixture@@ -102,8 +102,8 @@ def _build_node(*,node_id: str,node_data: KnowledgeIndexNodeData | dict[str, object],- graph_init_params,- graph_runtime_state,+ init_params,+ runtime_state,) -> KnowledgeIndexNode:return KnowledgeIndexNode(node_id=node_id,@@ -112,8 +112,8 @@ def _build_node(if isinstance(node_data, KnowledgeIndexNodeData)else KnowledgeIndexNodeData.model_validate(node_data)),- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)@@ -141,8 +141,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act & Assert@@ -174,8 +174,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act & Assert@@ -210,8 +210,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -266,8 +266,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -329,8 +329,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -380,8 +380,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -437,8 +437,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -495,8 +495,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -534,8 +534,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -578,8 +578,8 @@ class TestKnowledgeIndexNode:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -619,8 +619,8 @@ class TestInvokeKnowledgeIndex:node = _build_node(node_id=node_id,node_data=config["data"],- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Actdiff --git a/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval/test_knowledge_retrieval_node.py b/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval/test_knowledge_retrieval_node.pyindex a5ceeb46459..c016fb269fb 100644--- a/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval/test_knowledge_retrieval_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval/test_knowledge_retrieval_node.py@@ -25,7 +25,7 @@ from core.workflow.nodes.knowledge_retrieval.retrieval import Source, SourceMetafrom core.workflow.system_variables import build_system_variablesfrom graphon.enums import WorkflowNodeExecutionStatusfrom graphon.model_runtime.entities.llm_entities import LLMUsage-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variables import ArrayObjectSegment, StringSegmentfrom tests.workflow_test_utils import build_test_graph_init_params@@ -46,7 +46,7 @@ def _source() -> Source:@pytest.fixturedef mock_graph_init_params():- """Create mock GraphInitParams."""+ """Create mock InitParams."""return build_test_graph_init_params(workflow_id=str(uuid.uuid4()),graph_config={},@@ -61,14 +61,14 @@ def mock_graph_init_params():@pytest.fixturedef mock_graph_runtime_state():- """Create mock GraphRuntimeState."""+ """Create mock RuntimeState."""variable_pool = VariablePool.from_bootstrap(system_variables=build_system_variables(user_id=str(uuid.uuid4()), files=[]),user_inputs={},environment_variables=[],conversation_variables=[],)- return GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter())@pytest.fixture@@ -136,8 +136,8 @@ class TestKnowledgeRetrievalNode:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -193,8 +193,8 @@ class TestKnowledgeRetrievalNode:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -235,8 +235,8 @@ class TestKnowledgeRetrievalNode:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -271,8 +271,8 @@ class TestKnowledgeRetrievalNode:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -306,8 +306,8 @@ class TestKnowledgeRetrievalNode:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -346,8 +346,8 @@ class TestKnowledgeRetrievalNode:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -384,8 +384,8 @@ class TestKnowledgeRetrievalNode:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -464,8 +464,8 @@ class TestFetchDatasetRetriever:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -500,8 +500,8 @@ class TestFetchDatasetRetriever:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -554,8 +554,8 @@ class TestFetchDatasetRetriever:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)# Act@@ -594,8 +594,8 @@ class TestFetchDatasetRetriever:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)conditions = MetadataFilteringCondition(@@ -655,8 +655,8 @@ class TestFetchDatasetRetriever:node = KnowledgeRetrievalNode(node_id=node_id,data=KnowledgeRetrievalNodeData.model_validate(config["data"]),- graph_init_params=mock_graph_init_params,- graph_runtime_state=mock_graph_runtime_state,+ init_params=mock_graph_init_params,+ runtime_state=mock_graph_runtime_state,)mock_rag_retrieval.knowledge_retrieval.return_value = []diff --git a/api/tests/unit_tests/core/workflow/nodes/list_operator/node_spec.py b/api/tests/unit_tests/core/workflow/nodes/list_operator/node_spec.pyindex 198ee553d9a..771c0a13c78 100644--- a/api/tests/unit_tests/core/workflow/nodes/list_operator/node_spec.py+++ b/api/tests/unit_tests/core/workflow/nodes/list_operator/node_spec.py@@ -1,11 +1,10 @@import pytestfrom core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY-from graphon.entities import GraphInitParamsfrom graphon.enums import WorkflowNodeExecutionStatusfrom graphon.nodes.list_operator.entities import ListOperatorNodeDatafrom graphon.nodes.list_operator.node import ListOperatorNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom graphon.variables import ArrayNumberSegment, ArrayStringSegment@@ -13,12 +12,12 @@ class TestListOperatorNode:"""Comprehensive tests for ListOperatorNode."""@staticmethod- def _build_node(*, data, graph_init_params, graph_runtime_state):+ def _build_node(*, data, init_params, runtime_state):return ListOperatorNode(node_id="test",data=data if isinstance(data, ListOperatorNodeData) else ListOperatorNodeData.model_validate(data),- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)@staticmethod@@ -31,12 +30,12 @@ class TestListOperatorNode:@pytest.fixturedef graph_runtime_state(self):"""Create graph state backed by the real variable pool."""- return GraphRuntimeState(variable_pool=VariablePool(), start_at=0)+ return RuntimeState(workflow_id="test", variable_pool=VariablePool(), start_at=0)@pytest.fixturedef graph_init_params(self):- """Create GraphInitParams fixture."""- return GraphInitParams(+ """Create InitParams fixture."""+ return InitParams(workflow_id="test",graph_config={},run_context={@@ -59,8 +58,8 @@ class TestListOperatorNode:graph_runtime_state.variable_pool.add(config["variable"], variable)return self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)return _create_node@@ -98,8 +97,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -124,8 +123,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -148,8 +147,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -172,8 +171,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -199,8 +198,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -226,8 +225,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -253,8 +252,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -283,8 +282,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -304,8 +303,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -328,8 +327,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -353,8 +352,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -377,8 +376,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -401,8 +400,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -425,8 +424,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()@@ -452,8 +451,8 @@ class TestListOperatorNode:node = self._build_node(data=config,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,)result = node._run()diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_node_execution_behavior.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_node_execution_behavior.pyindex 32334e561fd..5270eebc718 100644--- a/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_node_execution_behavior.py+++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_node_execution_behavior.py@@ -25,7 +25,7 @@ from graphon.model_runtime.entities.message_entities import PromptMessagefrom graphon.node_events import StreamCompletedEventfrom graphon.nodes.llm.entities import LLMNodeDatafrom graphon.nodes.llm.node import LLMNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.unit_tests.core.model_fixtures import make_model_config, make_model_instancefrom tests.workflow_test_utils import build_test_graph_init_params@@ -77,14 +77,16 @@ def init_llm_node(config: dict[str, object]) -> LLMNode:)variable_pool.add(["abc", "output"], "sunny")- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id=workflow_id, variable_pool=variable_pool, start_at=time.perf_counter()+ )run_context = resolve_dify_run_context(init_params.run_context)node = LLMNode(node_id=str(uuid.uuid4()),data=LLMNodeData.model_validate(config["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,credentials_provider=DifyCredentialsProvider(run_context=run_context),model_factory=DifyModelFactory(run_context=run_context),model_instance=DifyPreparedLLM(make_model_instance(provider="openai", model="gpt-3.5-turbo")),@@ -174,7 +176,7 @@ def test_execute_llm(monkeypatch: pytest.MonkeyPatch) -> None:node._model_instance = build_model_instance()- with patch.object(LLMNode, "fetch_prompt_messages", mock_fetch_prompt_messages_1):+ with patch("graphon.nodes.llm.llm_utils.fetch_prompt_messages", mock_fetch_prompt_messages_1):# execute noderesult = node._run()assert isinstance(result, Generator)@@ -275,7 +277,7 @@ def test_execute_llm_with_jinja2(monkeypatch: pytest.MonkeyPatch) -> None:node._model_instance = build_model_instance()- with patch.object(LLMNode, "fetch_prompt_messages", mock_fetch_prompt_messages_2):+ with patch("graphon.nodes.llm.llm_utils.fetch_prompt_messages", mock_fetch_prompt_messages_2):# execute noderesult = node._run()diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_utils.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_utils.pyindex 3d6f14fbb11..141053d24d5 100644--- a/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_utils.py+++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_llm_utils.py@@ -34,6 +34,7 @@ from graphon.nodes.llm.exc import (TemplateTypeNotSupportError,)from graphon.runtime import VariablePool+from graphon.template_rendering import TemplateRenderErrorfrom graphon.variables import ArrayAnySegment, ArrayFileSegment, NoneSegmentfrom tests.unit_tests.core.model_fixtures import make_model_instance@@ -135,7 +136,7 @@ def _fetch_prompt_messages_with_mocked_content(content):vision_detail=ImagePromptMessageContent.DETAIL.HIGH,variable_pool=variable_pool,jinja2_variables=[],- template_renderer=None,+ jinja2_template_renderer=None,)@@ -433,7 +434,7 @@ def test_handle_list_messages_renders_jinja2_messages():jinja2_variables=[VariableSelector(variable="name", value_selector=["input", "name"])],variable_pool=variable_pool,vision_detail_config=ImagePromptMessageContent.DETAIL.HIGH,- template_renderer=renderer,+ jinja2_template_renderer=renderer,)assert prompt_messages == [SystemPromptMessage(content=[TextPromptMessageContent(data="Hello Dify")])]@@ -542,17 +543,17 @@ def test_render_jinja2_message_handles_empty_template_success_and_missing_rendertemplate="",jinja2_variables=variables,variable_pool=variable_pool,- template_renderer=None,+ jinja2_template_renderer=None,)== "")- with pytest.raises(ValueError, match="template_renderer is required"):+ with pytest.raises(TemplateRenderError, match="LLM prompts require an injected jinja2_template_renderer"):llm_utils.render_jinja2_message(template="Hello {{ name }}",jinja2_variables=variables,variable_pool=variable_pool,- template_renderer=None,+ jinja2_template_renderer=None,)renderer = mock.MagicMock()@@ -562,7 +563,7 @@ def test_render_jinja2_message_handles_empty_template_success_and_missing_rendertemplate="Hello {{ name }}",jinja2_variables=variables,variable_pool=variable_pool,- template_renderer=renderer,+ jinja2_template_renderer=renderer,)== "Hello Dify")@@ -592,7 +593,7 @@ def test_handle_completion_template_supports_basic_and_jinja2_templates():context="",jinja2_variables=[VariableSelector(variable="name", value_selector=["input", "name"])],variable_pool=variable_pool,- template_renderer=renderer,+ jinja2_template_renderer=renderer,)assert basic_messages == [diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.pyindex 090155bc2da..8b8119c2d47 100644--- a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py@@ -21,7 +21,6 @@ from core.plugin.impl.model_runtime_factory import create_plugin_model_assemblyfrom core.prompt.entities.advanced_prompt_entities import MemoryConfigfrom core.workflow.node_runtime import DifyFileReferenceFactory, DifyPromptMessageSerializer, DifyToolFileManagerfrom core.workflow.system_variables import default_system_variables-from graphon.entities import GraphInitParamsfrom graphon.file import File, FileTransferMethod, FileTypefrom graphon.model_runtime.entities.common_entities import I18nObjectfrom graphon.model_runtime.entities.llm_entities import (@@ -75,16 +74,17 @@ from graphon.nodes.llm.exc import (VariableNotFoundError,)from graphon.nodes.llm.file_saver import FileSaverImpl, LLMFileSaver-from graphon.nodes.llm.node import (- LLMNode,- _calculate_rest_token,- _handle_completion_template,- _handle_memory_chat_mode,- _handle_memory_completion_mode,- _render_jinja2_message,+from graphon.nodes.llm.llm_utils import (+ calculate_rest_token,+ handle_completion_template,+ handle_list_messages,+ handle_memory_chat_mode,+ handle_memory_completion_mode,+ render_jinja2_message,)+from graphon.nodes.llm.node import LLMNodefrom graphon.nodes.llm.reasoning import split_reasoning-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom graphon.template_rendering import TemplateRenderErrorfrom graphon.variables import ArrayAnySegment, ArrayFileSegment, NoneSegmentfrom models.provider import ProviderType@@ -219,7 +219,7 @@ def llm_node_data() -> LLMNodeData:@pytest.fixture-def graph_init_params() -> GraphInitParams:+def graph_init_params() -> InitParams:return build_test_graph_init_params(workflow_id="1",graph_config={},@@ -233,21 +233,20 @@ def graph_init_params() -> GraphInitParams:@pytest.fixture-def graph_runtime_state() -> GraphRuntimeState:+def graph_runtime_state() -> RuntimeState:variable_pool = VariablePool.from_bootstrap(system_variables=default_system_variables(),user_inputs={},)- return GraphRuntimeState(+ return RuntimeState(+ workflow_id="test-workflow",variable_pool=variable_pool,start_at=0,)@pytest.fixture-def llm_node(- llm_node_data: LLMNodeData, graph_init_params: GraphInitParams, graph_runtime_state: GraphRuntimeState-) -> LLMNode:+def llm_node(llm_node_data: LLMNodeData, graph_init_params: InitParams, graph_runtime_state: RuntimeState) -> LLMNode:file_saver = _build_file_saver()credentials_provider = DifyCredentialsProvider(run_context=_run_context())model_factory = DifyModelFactory(run_context=_run_context())@@ -256,8 +255,8 @@ def llm_node(node = LLMNode(node_id="1",data=llm_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,credentials_provider=credentials_provider,model_factory=model_factory,model_instance=_build_prepared_llm_mock(),@@ -626,7 +625,7 @@ def test_fetch_files_with_non_existent_variable():# memory_config=None,# vision_enabled=False,# vision_detail=fake_vision_detail,-# variable_pool=llm_node.graph_runtime_state.variable_pool,+# variable_pool=llm_node.runtime_state.variable_pool,# jinja2_variables=[],# )@@ -802,7 +801,7 @@ def test_fetch_files_with_non_existent_variable():# for k, v in scenario.file_variables.items():# selector = k.split(".")-# llm_node.graph_runtime_state.variable_pool.add(selector, v)+# llm_node.runtime_state.variable_pool.add(selector, v)# # Call the method under test# prompt_messages, _ = llm_node._fetch_prompt_messages(@@ -815,7 +814,7 @@ def test_fetch_files_with_non_existent_variable():# memory_config=memory_config,# vision_enabled=scenario.vision_enabled,# vision_detail=scenario.vision_detail,-# variable_pool=llm_node.graph_runtime_state.variable_pool,+# variable_pool=llm_node.runtime_state.variable_pool,# jinja2_variables=[],# )@@ -836,10 +835,10 @@ def test_handle_list_messages_basic(llm_node):]context = "world"jinja2_variables = []- variable_pool = llm_node.graph_runtime_state.variable_pool+ variable_pool = llm_node.runtime_state.variable_poolvision_detail_config = ImagePromptMessageContent.DETAIL.HIGH- result = llm_node.handle_list_messages(+ result = handle_list_messages(messages=messages,context=context,jinja2_variables=jinja2_variables,@@ -862,11 +861,11 @@ def test_handle_list_messages_replaces_double_brace_context_placeholder(llm_node]context = "## Overview\nSends a JSON request."- result = llm_node.handle_list_messages(+ result = handle_list_messages(messages=messages,context=context,jinja2_variables=[],- variable_pool=llm_node.graph_runtime_state.variable_pool,+ variable_pool=llm_node.runtime_state.variable_pool,vision_detail_config=ImagePromptMessageContent.DETAIL.HIGH,)@@ -880,11 +879,11 @@ def test_handle_list_messages_replaces_double_brace_context_placeholder(llm_nodedef test_handle_list_messages_renders_jinja2_messages(llm_node):- llm_node.graph_runtime_state.variable_pool.add(["input", "name"], "Dify")+ llm_node.runtime_state.variable_pool.add(["input", "name"], "Dify")renderer = mock.MagicMock()renderer.render_template.return_value = "Hello Dify"- prompt_messages = llm_node.handle_list_messages(+ prompt_messages = handle_list_messages(messages=[LLMNodeChatModelMessage(text="ignored",@@ -895,7 +894,7 @@ def test_handle_list_messages_renders_jinja2_messages(llm_node):],context="",jinja2_variables=[VariableSelector(variable="name", value_selector=["input", "name"])],- variable_pool=llm_node.graph_runtime_state.variable_pool,+ variable_pool=llm_node.runtime_state.variable_pool,vision_detail_config=ImagePromptMessageContent.DETAIL.HIGH,jinja2_template_renderer=renderer,)@@ -935,15 +934,15 @@ def test_transform_chat_messages_prefers_jinja2_text(llm_node):def test_fetch_jinja_inputs_serializes_supported_segment_types(llm_node):- llm_node.graph_runtime_state.variable_pool.add(+ llm_node.runtime_state.variable_pool.add(["input", "items"],["alpha", {"metadata": {"_source": "knowledge"}, "content": "beta"}, 3],)- llm_node.graph_runtime_state.variable_pool.add(+ llm_node.runtime_state.variable_pool.add(["input", "context_doc"],{"metadata": {"_source": "knowledge"}, "content": "context body"},)- llm_node.graph_runtime_state.variable_pool.add(["input", "payload"], {"a": 1})+ llm_node.runtime_state.variable_pool.add(["input", "payload"], {"a": 1})node_data = llm_node.node_data.model_copy(update={@@ -978,8 +977,8 @@ def test_fetch_jinja_inputs_raises_for_missing_variable(llm_node):def test_fetch_inputs_collects_prompt_and_memory_variables(llm_node):- llm_node.graph_runtime_state.variable_pool.add(["input", "name"], "Dify")- llm_node.graph_runtime_state.variable_pool.add(["input", "payload"], {"active": True})+ llm_node.runtime_state.variable_pool.add(["input", "name"], "Dify")+ llm_node.runtime_state.variable_pool.add(["input", "payload"], {"active": True})node_data = llm_node.node_data.model_copy(update={@@ -1005,7 +1004,7 @@ def test_fetch_inputs_collects_prompt_and_memory_variables(llm_node):def test_fetch_context_emits_string_context_event(llm_node):- llm_node.graph_runtime_state.variable_pool.add(["context", "value"], "retrieved context")+ llm_node.runtime_state.variable_pool.add(["context", "value"], "retrieved context")node_data = llm_node.node_data.model_copy(update={"context": ContextConfig(enabled=True, variable_selector=["context", "value"])})@@ -1026,7 +1025,7 @@ def test_fetch_context_collects_retriever_resources_and_attachments(llm_node):llm_node._retriever_attachment_loader = mock.MagicMock()llm_node._retriever_attachment_loader.load.return_value = [attachment]- llm_node.graph_runtime_state.variable_pool.add(+ llm_node.runtime_state.variable_pool.add(["context", "value"],[{@@ -1079,7 +1078,7 @@ def test_fetch_context_collects_retriever_resources_and_attachments(llm_node):def test_fetch_context_rejects_invalid_context_structure(llm_node):- llm_node.graph_runtime_state.variable_pool.add(["context", "value"], [{"summary": "missing content"}])+ llm_node.runtime_state.variable_pool.add(["context", "value"], [{"summary": "missing content"}])node_data = llm_node.node_data.model_copy(update={"context": ContextConfig(enabled=True, variable_selector=["context", "value"])})@@ -1117,9 +1116,9 @@ def test_fetch_prompt_messages_chat_mode_appends_memory_query_and_files(mocker:),]- with mock.patch("graphon.nodes.llm.node.file_manager.to_prompt_message_content") as mock_to_prompt:+ with mock.patch("graphon.nodes.llm.llm_utils.file_manager.to_prompt_message_content") as mock_to_prompt:mock_to_prompt.side_effect = prompt_content_side_effect- prompt_messages, stop = LLMNode.fetch_prompt_messages(+ prompt_messages, stop = llm_utils.fetch_prompt_messages(sys_query="current question",sys_files=[sys_file],context="",@@ -1168,7 +1167,7 @@ def test_fetch_prompt_messages_completion_mode_injects_histories_and_query():])- prompt_messages, stop = LLMNode.fetch_prompt_messages(+ prompt_messages, stop = llm_utils.fetch_prompt_messages(sys_query="latest question",sys_files=[],context="",@@ -1209,7 +1208,7 @@ def test_fetch_prompt_messages_raises_when_only_unsupported_content_remains():with (mock.patch(- "graphon.nodes.llm.node.file_manager.to_prompt_message_content",+ "graphon.nodes.llm.llm_utils.file_manager.to_prompt_message_content",return_value=ImagePromptMessageContent(url="https://example.com/file.png",format="png",@@ -1219,7 +1218,7 @@ def test_fetch_prompt_messages_raises_when_only_unsupported_content_remains():),pytest.raises(NoPromptFoundError, match="No prompt found"),):- LLMNode.fetch_prompt_messages(+ llm_utils.fetch_prompt_messages(sys_query=None,sys_files=[],context="",@@ -1242,14 +1241,14 @@ def test_fetch_prompt_messages_raises_when_only_unsupported_content_remains():def test_handle_completion_template_replaces_double_brace_context_placeholder(llm_node):- prompt_messages = _handle_completion_template(+ prompt_messages = handle_completion_template(template=LLMNodeCompletionModelPromptTemplate(text="Summarize the following context:\n{{#context#}}",edition_type="basic",),context="## Overview\nSends a JSON request.",jinja2_variables=[],- variable_pool=llm_node.graph_runtime_state.variable_pool,+ variable_pool=llm_node.runtime_state.variable_pool,jinja2_template_renderer=None,)@@ -1287,8 +1286,8 @@ def test_handle_memory_completion_mode_uses_prompt_message_interface(mocker: Mocwindow=MemoryConfig.WindowConfig(enabled=True, size=3),)- with mock.patch("graphon.nodes.llm.node._calculate_rest_token", return_value=2000) as mock_rest_token:- memory_text = _handle_memory_completion_mode(+ with mock.patch("graphon.nodes.llm.llm_utils.calculate_rest_token", return_value=2000) as mock_rest_token:+ memory_text = handle_memory_completion_mode(memory=memory,memory_config=memory_config,model_instance=model_instance,@@ -1311,8 +1310,8 @@ def llm_node_for_multimodal(node = LLMNode(node_id="1",data=llm_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,credentials_provider=credentials_provider,model_factory=model_factory,model_instance=_build_prepared_llm_mock(),@@ -1346,9 +1345,6 @@ class TestLLMNodeSaveMultiModalImageOutput:content=content,file_saver=file_saver,)- # Manually append to _file_outputs since the static method doesn't do it- llm_node._file_outputs.append(file)- assert llm_node._file_outputs == [mock_file]assert file == mock_fileassert file_saver.binary_calls == [(b"test-data", "image/png", FileType.IMAGE, ".png")]@@ -1374,9 +1370,6 @@ class TestLLMNodeSaveMultiModalImageOutput:content=content,file_saver=file_saver,)- # Manually append to _file_outputs since the static method doesn't do it- llm_node._file_outputs.append(file)- assert llm_node._file_outputs == [mock_file]assert file == mock_fileassert file_saver.remote_calls == [(content.url, FileType.IMAGE)]@@ -1430,6 +1423,7 @@ class TestSaveMultimodalOutputAndConvertResultToMarkdown:storage_key="test_storage_key",)file_saver.saved_file = mock_saved_file+ file_outputs: list[File] = []gen = llm_node._save_multimodal_output_and_convert_result_to_markdown(contents=[ImagePromptMessageContent(@@ -1439,7 +1433,7 @@ class TestSaveMultimodalOutputAndConvertResultToMarkdown:)],file_saver=file_saver,- file_outputs=llm_node._file_outputs,+ file_outputs=file_outputs,)yielded_strs = list(gen)assert len(yielded_strs) == 1@@ -1454,7 +1448,7 @@ class TestSaveMultimodalOutputAndConvertResultToMarkdown:assert expected_file_url_path in yielded_strs[0]assert yielded_strs[0].endswith(")")assert file_saver.binary_calls == [(image_raw_data, "image/png", FileType.IMAGE, ".png")]- assert mock_saved_file in llm_node._file_outputs+ assert mock_saved_file in file_outputsdef test_unknown_content_type(self, llm_node_for_multimodal):llm_node, file_saver = llm_node_for_multimodal@@ -1671,7 +1665,7 @@ def test_handle_invoke_result_wraps_structured_output_parse_errors():raise ValueError("bad json")yield- with pytest.raises(LLMNodeError, match="Failed to parse structured output: bad json"):+ with pytest.raises(LLMNodeError, match=r"Failed to parse structured output \(stage=result, path=\$\): bad json"):list(LLMNode.handle_invoke_result(invoke_result=broken_stream(),@@ -1711,10 +1705,10 @@ def test_fetch_structured_output_schema_validates_payload():"type": "object"}- with pytest.raises(LLMNodeError, match="Please provide a valid structured output schema"):+ with pytest.raises(LLMNodeError, match=r"Invalid structured output schema \(stage=schema, path=\$\.schema\)"):LLMNode.fetch_structured_output_schema(structured_output={})- with pytest.raises(LLMNodeError, match="structured_output_schema must be a JSON object"):+ with pytest.raises(LLMNodeError, match=r"Invalid structured output schema \(stage=schema, path=\$\.schema\)"):LLMNode.fetch_structured_output_schema(structured_output={"schema": ["not", "an", "object"]})@@ -1768,9 +1762,9 @@ def test_render_jinja2_message_requires_renderer_and_passes_inputs():with pytest.raises(TemplateRenderError,- match="LLMNode requires an injected jinja2_template_renderer for jinja2 prompts",+ match="LLM prompts require an injected jinja2_template_renderer",):- _render_jinja2_message(+ render_jinja2_message(template="Hello {{ name }}",jinja2_variables=variables,variable_pool=variable_pool,@@ -1781,7 +1775,7 @@ def test_render_jinja2_message_requires_renderer_and_passes_inputs():renderer.render_template.return_value = "Hello Dify"assert (- _render_jinja2_message(+ render_jinja2_message(template="Hello {{ name }}",jinja2_variables=variables,variable_pool=variable_pool,@@ -1808,7 +1802,7 @@ def test_calculate_rest_token_uses_context_size_and_max_tokens():model_instance.get_llm_num_tokens.return_value = 1000assert (- _calculate_rest_token(+ calculate_rest_token(prompt_messages=[UserPromptMessage(content="hello")],model_instance=model_instance,)@@ -1821,8 +1815,8 @@ def test_handle_memory_chat_mode_uses_calculated_token_budget(mocker: MockerFixtmemory = MockTokenBufferMemory(history)history_lookup = mocker.spy(memory, "get_history_prompt_messages")- with mock.patch("graphon.nodes.llm.node._calculate_rest_token", return_value=321) as mock_rest_token:- result = _handle_memory_chat_mode(+ with mock.patch("graphon.nodes.llm.llm_utils.calculate_rest_token", return_value=321) as mock_rest_token:+ result = handle_memory_chat_mode(memory=memory,memory_config=MemoryConfig(window=MemoryConfig.WindowConfig(enabled=True, size=2)),model_instance=_build_prepared_llm_mock(),diff --git a/api/tests/unit_tests/core/workflow/nodes/parameter_extractor/test_parameter_extractor_execution_behavior.py b/api/tests/unit_tests/core/workflow/nodes/parameter_extractor/test_parameter_extractor_execution_behavior.pyindex 75fb10d6102..eb3a8e84c89 100644--- a/api/tests/unit_tests/core/workflow/nodes/parameter_extractor/test_parameter_extractor_execution_behavior.py+++ b/api/tests/unit_tests/core/workflow/nodes/parameter_extractor/test_parameter_extractor_execution_behavior.py@@ -19,7 +19,7 @@ from graphon.model_runtime.entities.message_entities import PromptMessagefrom graphon.model_runtime.memory.prompt_message_memory import PromptMessageMemoryfrom graphon.nodes.parameter_extractor.entities import ParameterExtractorNodeDatafrom graphon.nodes.parameter_extractor.parameter_extractor_node import ParameterExtractorNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.unit_tests.core.model_fixtures import make_model_instancefrom tests.unit_tests.core.workflow.nodes.parameter_extractor.fixtures.model import get_mocked_fetch_model_instancefrom tests.workflow_test_utils import build_test_graph_init_params@@ -77,14 +77,16 @@ def init_parameter_extractor_node(variable_pool.add(["a", "args1"], 1)variable_pool.add(["a", "args2"], 2)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )run_context = resolve_dify_run_context(init_params.run_context)node = ParameterExtractorNode(node_id=str(uuid.uuid4()),data=ParameterExtractorNodeData.model_validate(config["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,credentials_provider=DifyCredentialsProvider(run_context=run_context),model_factory=DifyModelFactory(run_context=run_context),model_instance=DifyPreparedLLM(make_model_instance(provider="openai", model="gpt-3.5-turbo")),diff --git a/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py b/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.pyindex dbb4ec86950..77c0e602ae8 100644--- a/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py+++ b/api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py@@ -7,7 +7,7 @@ from graphon.enums import WorkflowNodeExecutionStatusfrom graphon.nodes.base.entities import VariableSelectorfrom graphon.nodes.template_transform.entities import TemplateTransformNodeDatafrom graphon.nodes.template_transform.template_transform_node import TemplateTransformNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.template_rendering import TemplateRenderErrorfrom tests.workflow_test_utils import build_test_graph_init_params@@ -15,8 +15,8 @@ from tests.workflow_test_utils import build_test_graph_init_paramsdef _build_template_transform_node(*,node_data,- graph_init_params,- graph_runtime_state,+ init_params,+ runtime_state,node_id: str = "test_node",**kwargs,) -> TemplateTransformNode:@@ -28,8 +28,8 @@ def _build_template_transform_node(return TemplateTransformNode(node_id=node_id,data=typed_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,**kwargs,)@@ -38,9 +38,9 @@ class TestTemplateTransformNode:"""Comprehensive test suite for TemplateTransformNode."""@pytest.fixture- def graph_runtime_state(self) -> GraphRuntimeState:+ def graph_runtime_state(self) -> RuntimeState:"""Create graph state with real variable storage and conversion."""- return GraphRuntimeState(variable_pool=VariablePool(), start_at=0)+ return RuntimeState(workflow_id="test_workflow", variable_pool=VariablePool(), start_at=0)@pytest.fixturedef graph_init_params(self):@@ -82,8 +82,8 @@ class TestTemplateTransformNode:with pytest.raises(ValueError, match="max_output_length must be a positive integer"):_build_template_transform_node(node_data=basic_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,max_output_length=max_output_length,)@@ -106,8 +106,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=basic_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)@@ -131,8 +131,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)@@ -151,8 +151,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=basic_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)@@ -171,8 +171,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=basic_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,max_output_length=10,)@@ -191,8 +191,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=basic_node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,max_output_length=10,)@@ -231,8 +231,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)@@ -337,8 +337,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)@@ -374,8 +374,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)@@ -401,8 +401,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)@@ -429,8 +429,8 @@ class TestTemplateTransformNode:node = _build_template_transform_node(node_data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=graph_runtime_state,+ init_params=graph_init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=mock_renderer,)diff --git a/api/tests/unit_tests/core/workflow/nodes/template_transform/test_template_transform_execution.py b/api/tests/unit_tests/core/workflow/nodes/template_transform/test_template_transform_execution.pyindex 086ca44ee99..8069971e434 100644--- a/api/tests/unit_tests/core/workflow/nodes/template_transform/test_template_transform_execution.py+++ b/api/tests/unit_tests/core/workflow/nodes/template_transform/test_template_transform_execution.py@@ -11,7 +11,7 @@ from graphon.enums import WorkflowNodeExecutionStatusfrom graphon.graph import Graphfrom graphon.nodes.template_transform.entities import TemplateTransformNodeDatafrom graphon.nodes.template_transform.template_transform_node import TemplateTransformNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.template_rendering import Jinja2TemplateRenderer, TemplateRenderErrorfrom tests.workflow_test_utils import build_test_graph_init_params@@ -79,12 +79,14 @@ def test_execute_template_transform() -> None:variable_pool.add(["1", "args1"], 1)variable_pool.add(["1", "args2"], 3)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )# Create node factory (graph init path still works regardless of renderer choice below)node_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)graph = Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -93,8 +95,8 @@ def test_execute_template_transform() -> None:node = TemplateTransformNode(node_id=str(uuid.uuid4()),data=TemplateTransformNodeData.model_validate(config["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,jinja2_template_renderer=cast(Jinja2TemplateRenderer, _SimpleJinja2Renderer()),)diff --git a/api/tests/unit_tests/core/workflow/nodes/test_base_node.py b/api/tests/unit_tests/core/workflow/nodes/test_base_node.pyindex 45aa7470618..fdf206ba83d 100644--- a/api/tests/unit_tests/core/workflow/nodes/test_base_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/test_base_node.py@@ -5,11 +5,10 @@ import pytestfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFromfrom core.workflow.node_runtime import resolve_dify_run_contextfrom core.workflow.system_variables import build_system_variables-from graphon.entities import GraphInitParamsfrom graphon.entities.base_node_data import BaseNodeDatafrom graphon.enums import BuiltinNodeTypesfrom graphon.nodes.base.node import Node-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_params@@ -28,13 +27,14 @@ class _SampleNode(Node[_SampleNodeData]):raise NotImplementedError-def _build_context(graph_config: Mapping[str, object]) -> tuple[GraphInitParams, GraphRuntimeState]:+def _build_context(graph_config: Mapping[str, object]) -> tuple[InitParams, RuntimeState]:init_params = build_test_graph_init_params(graph_config=graph_config,user_from="account",invoke_from="debugger",)- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(user_id="user", files=[]),user_inputs={},@@ -66,8 +66,8 @@ def test_node_hydrates_data_during_initialization():node = _SampleNode(node_id="node-1",data=_build_node_data(),- graph_init_params=init_params,- graph_runtime_state=runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)assert node.node_data.foo == "bar"@@ -84,7 +84,8 @@ def test_node_accepts_invoke_from_enum():user_from=UserFrom.ACCOUNT,invoke_from=InvokeFrom.DEBUGGER,)- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(user_id="user", files=[]),user_inputs={},@@ -95,8 +96,8 @@ def test_node_accepts_invoke_from_enum():node = _SampleNode(node_id="node-1",data=_build_node_data(),- graph_init_params=init_params,- graph_runtime_state=runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)dify_ctx = resolve_dify_run_context(node.run_context)@@ -145,8 +146,8 @@ def test_node_hydration_preserves_compatibility_extra_fields():node = _SampleNode(node_id="node-1",data=node_config["data"],- graph_init_params=init_params,- graph_runtime_state=runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)assert node.node_data.foo == "bar"diff --git a/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py b/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.pyindex d8bc859dae6..b745c15e0e1 100644--- a/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py@@ -4,9 +4,10 @@ from unittest.mock import Mock, patchimport pandas as pdimport pytestfrom docx import Document+from odfdo import Document as OdfDocument+from odfdo import Paragraphfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFrom-from graphon.entities import GraphInitParamsfrom graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatusfrom graphon.file import File, FileTransferMethod, FileTypefrom graphon.node_events import NodeRunResult@@ -20,6 +21,7 @@ from graphon.nodes.document_extractor.node import (_extract_text_from_plain_text,_normalize_docx_zip,)+from graphon.runtime import InitParamsfrom graphon.variables import ArrayFileSegment, FileSegmentfrom graphon.variables.segments import ArrayStringSegmentfrom graphon.variables.variables import StringVariable@@ -38,7 +40,7 @@ def make_file(extension: str | None, mime_type: str | None) -> File:@pytest.fixture-def graph_init_params() -> GraphInitParams:+def graph_init_params() -> InitParams:return build_test_graph_init_params(workflow_id="test_workflow",graph_config={},@@ -61,8 +63,8 @@ def document_extractor_node(graph_init_params):node = DocumentExtractorNode(node_id="test_node_id",data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=Mock(),+ init_params=graph_init_params,+ runtime_state=Mock(),http_client=http_client,)return node@@ -74,7 +76,7 @@ def mock_graph_runtime_state():def test_run_variable_not_found(document_extractor_node, mock_graph_runtime_state):- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_statemock_graph_runtime_state.variable_pool.get.return_value = Noneresult = document_extractor_node._run()@@ -86,7 +88,7 @@ def test_run_variable_not_found(document_extractor_node, mock_graph_runtime_statdef test_run_invalid_variable_type(document_extractor_node, mock_graph_runtime_state):- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_statemock_graph_runtime_state.variable_pool.get.return_value = StringVariable(value="Not an ArrayFileSegment", name="test")@@ -101,7 +103,7 @@ def test_run_invalid_variable_type(document_extractor_node, mock_graph_runtime_sdef test_run_empty_file_list_returns_succeeded(document_extractor_node, mock_graph_runtime_state):"""Empty file list should return SUCCEEDED with empty documents and ArrayStringSegment([])."""- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_state# Provide an actual ArrayFileSegment with an empty listmock_graph_runtime_state.variable_pool.get.return_value = ArrayFileSegment(value=[])@@ -116,7 +118,7 @@ def test_run_empty_file_list_returns_succeeded(document_extractor_node, mock_gradef test_run_none_only_file_list_returns_succeeded(document_extractor_node, mock_graph_runtime_state):"""A file list containing only None (e.g., [None]) should be filtered to [] and succeed."""- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_state# Preserve the legacy null-entry payload handled by the node filtering branch.afs = ArrayFileSegment.model_construct(value=[None])@@ -173,7 +175,7 @@ def test_run_extract_text(extension,monkeypatch,):- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_statefile = File(file_type=FileType.DOCUMENT,@@ -460,28 +462,21 @@ def test_extract_text_from_file_routes_odt_inputs_to_graphon_odt_extractor():file = make_file(extension, mime_type)- def fake_partition(file_content, *, suffix, unstructured_api_config, load_local_partition, render_element):- assert file_content == b"odt content"- assert suffix == ".odt"- assert unstructured_api_config == document_extractor_node._unstructured_api_config- assert load_local_partition.__name__ == "_load_partition_odt"- assert render_element is not None- return f"extracted through {route_label}"-- with (- patch(- "graphon.nodes.document_extractor.node._download_file_content",- return_value=b"odt content",- ) as mock_download,- patch("graphon.nodes.document_extractor.node._partition_unstructured_file", side_effect=fake_partition),- ):+ document = OdfDocument()+ document.body.append(Paragraph(f"extracted through {route_label}"))+ content = io.BytesIO()+ document.save(content)++ with patch(+ "graphon.nodes.document_extractor.node._download_file_content", return_value=content.getvalue()+ ) as mock_download:text = _extract_text_from_file(document_extractor_node.http_client,file,unstructured_api_config=document_extractor_node._unstructured_api_config,)- assert text == f"extracted through {route_label}"+ assert text.strip() == f"extracted through {route_label}"mock_download.assert_called_once_with(document_extractor_node.http_client, file)@@ -549,7 +544,7 @@ def test_extract_text_from_file_rejects_missing_extension_and_mime_type(documentdef test_run_list_file_extraction_error_returns_failed(document_extractor_node, mock_graph_runtime_state):- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_statefile_list = ArrayFileSegment(value=[make_file(".txt", "text/plain")])mock_graph_runtime_state.variable_pool.get.return_value = file_list@@ -564,7 +559,7 @@ def test_run_list_file_extraction_error_returns_failed(document_extractor_node,def test_run_single_file_segment_extraction_error_returns_failed(document_extractor_node, mock_graph_runtime_state):- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_statefile_segment = FileSegment(value=make_file(".txt", "text/plain"))mock_graph_runtime_state.variable_pool.get.return_value = file_segment@@ -579,7 +574,7 @@ def test_run_single_file_segment_extraction_error_returns_failed(document_extracdef test_run_single_file_segment_returns_string_output(document_extractor_node, mock_graph_runtime_state):- document_extractor_node.graph_runtime_state = mock_graph_runtime_state+ document_extractor_node.runtime_state = mock_graph_runtime_statefile_segment = FileSegment(value=make_file(".txt", "text/plain"))mock_graph_runtime_state.variable_pool.get.return_value = file_segmentdiff --git a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py b/api/tests/unit_tests/core/workflow/nodes/test_if_else.pyindex 20d4e40cf23..d9c774c2ddb 100644--- a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py+++ b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py@@ -12,7 +12,7 @@ from graphon.file import File, FileTransferMethod, FileTypefrom graphon.graph import Graphfrom graphon.nodes.if_else.entities import IfElseNodeDatafrom graphon.nodes.if_else.if_else_node import IfElseNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.utils.condition.entities import Condition, SubCondition, SubVariableConditionfrom graphon.variables import ArrayFileSegmentfrom tests.workflow_test_utils import build_test_graph_init_params@@ -22,12 +22,12 @@ def _build_if_else_node(*,node_data: IfElseNodeData | dict[str, object],init_params,- graph_runtime_state,+ runtime_state,) -> IfElseNode:return IfElseNode(node_id=str(uuid.uuid4()),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,data=node_data if isinstance(node_data, IfElseNodeData) else IfElseNodeData.model_validate(node_data),)@@ -70,10 +70,10 @@ def test_execute_if_else_result_true():pool.add(["start", "null"], None)pool.add(["start", "not_null"], "1212")- graph_runtime_state = GraphRuntimeState(variable_pool=pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=pool, start_at=time.perf_counter())node_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -120,7 +120,7 @@ def test_execute_if_else_result_true():],},init_params=init_params,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)# execute node@@ -155,10 +155,10 @@ def test_execute_if_else_result_false():pool.add(["start", "array_contains"], ["1ab", "def"])pool.add(["start", "array_not_contains"], ["ab", "def"])- graph_runtime_state = GraphRuntimeState(variable_pool=pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=pool, start_at=time.perf_counter())node_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -181,7 +181,7 @@ def test_execute_if_else_result_false():],},init_params=init_params,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)# execute node@@ -235,9 +235,9 @@ def test_array_file_contains_file_name():}}- node = _build_if_else_node(node_data=node_data, init_params=graph_init_params, graph_runtime_state=Mock())+ node = _build_if_else_node(node_data=node_data, init_params=graph_init_params, runtime_state=Mock())- node.graph_runtime_state.variable_pool.get.return_value = ArrayFileSegment(+ node.runtime_state.variable_pool.get.return_value = ArrayFileSegment(value=[File(file_type=FileType.IMAGE,@@ -309,10 +309,10 @@ def test_execute_if_else_boolean_conditions(condition: Condition):pool.add(["start", "bool_array"], [True, False, True])pool.add(["start", "mixed_array"], [True, "false", 1, 0])- graph_runtime_state = GraphRuntimeState(variable_pool=pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=pool, start_at=time.perf_counter())node_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -325,7 +325,7 @@ def test_execute_if_else_boolean_conditions(condition: Condition):node = _build_if_else_node(node_data=node_data,init_params=init_params,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)# execute node@@ -359,10 +359,10 @@ def test_execute_if_else_boolean_false_conditions():pool.add(["start", "bool_false"], False)pool.add(["start", "bool_array"], [True, False, True])- graph_runtime_state = GraphRuntimeState(variable_pool=pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=pool, start_at=time.perf_counter())node_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -387,7 +387,7 @@ def test_execute_if_else_boolean_false_conditions():node = _build_if_else_node(node_data=node_data,init_params=init_params,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)# execute node@@ -420,10 +420,10 @@ def test_execute_if_else_boolean_cases_structure():pool.add(["start", "bool_true"], True)pool.add(["start", "bool_false"], False)- graph_runtime_state = GraphRuntimeState(variable_pool=pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=pool, start_at=time.perf_counter())node_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -452,7 +452,7 @@ def test_execute_if_else_boolean_cases_structure():node = _build_if_else_node(node_data=node_data,init_params=init_params,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)# execute nodediff --git a/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py b/api/tests/unit_tests/core/workflow/nodes/test_list_operator.pyindex aa050cffd0a..46a2bb605a1 100644--- a/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py+++ b/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py@@ -23,8 +23,8 @@ def _build_list_operator_node(node_data: ListOperatorNodeData, graph_init_paramsreturn ListOperatorNode(node_id="test_node_id",data=node_data,- graph_init_params=graph_init_params,- graph_runtime_state=MagicMock(),+ init_params=graph_init_params,+ runtime_state=MagicMock(),)@@ -60,8 +60,8 @@ def list_operator_node():}node = _build_list_operator_node(node_data, graph_init_params)- node.graph_runtime_state = MagicMock()- node.graph_runtime_state.variable_pool = MagicMock()+ node.runtime_state = MagicMock()+ node.runtime_state.variable_pool = MagicMock()return node@@ -98,7 +98,7 @@ def test_filter_files_by_type(list_operator_node):),]variable = ArrayFileSegment(value=files)- list_operator_node.graph_runtime_state.variable_pool.get.return_value = variable+ list_operator_node.runtime_state.variable_pool.get.return_value = variable# Run the noderesult = list_operator_node._run()diff --git a/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.py b/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.pyindex f890f795118..aa1d10a6898 100644--- a/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.py+++ b/api/tests/unit_tests/core/workflow/nodes/test_start_node_json_object.py@@ -8,7 +8,7 @@ from core.workflow.system_variables import build_system_variablesfrom core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_IDfrom graphon.nodes.start.entities import StartNodeDatafrom graphon.nodes.start.start_node import StartNode-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom graphon.variables import build_segment, segment_to_variablefrom graphon.variables.input_entities import VariableEntity, VariableEntityTypefrom graphon.variables.variables import Variable@@ -24,7 +24,8 @@ def make_start_node(user_inputs, variables):node_data = StartNodeData(title="Start", variables=variables)- graph_runtime_state = GraphRuntimeState(+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=variable_pool,start_at=time.perf_counter(),)@@ -32,7 +33,7 @@ def make_start_node(user_inputs, variables):return StartNode(node_id="start",data=node_data,- graph_init_params=build_test_graph_init_params(+ init_params=build_test_graph_init_params(workflow_id="wf",graph_config={},tenant_id="tenant",@@ -42,7 +43,7 @@ def make_start_node(user_inputs, variables):invoke_from="debugger",call_depth=0,),- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)@@ -257,11 +258,13 @@ def test_start_node_outputs_full_variable_pool_snapshot():],)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )node = StartNode(node_id="start",data=node_data,- graph_init_params=build_test_graph_init_params(+ init_params=build_test_graph_init_params(workflow_id="wf",graph_config={},tenant_id="tenant",@@ -271,7 +274,7 @@ def test_start_node_outputs_full_variable_pool_snapshot():invoke_from="debugger",call_depth=0,),- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,)result = node._run()diff --git a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.pyindex 2e316a6e436..c8ea5a542f5 100644--- a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py@@ -18,7 +18,7 @@ from graphon.model_runtime.entities.llm_entities import LLMUsagefrom graphon.node_events import StreamChunkEvent, StreamCompletedEventfrom graphon.nodes.tool.entities import ToolNodeDatafrom graphon.nodes.tool_runtime_entities import ToolRuntimeHandle, ToolRuntimeMessage-from graphon.runtime import GraphRuntimeState+from graphon.runtime import RuntimeStatefrom graphon.variables.segments import ArrayFileSegmentfrom tests.workflow_test_utils import build_test_graph_init_params, build_test_variable_pool@@ -109,7 +109,7 @@ def tool_node(monkeypatch) -> ToolNode:)variable_pool = build_test_variable_pool(variables=build_system_variables(user_id="user-id"))- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)config = graph_config["nodes"][0]@@ -119,8 +119,8 @@ def tool_node(monkeypatch) -> ToolNode:node = ToolNode(node_id="node-instance",data=ToolNodeData.model_validate(config["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,tool_file_manager=tool_file_manager,runtime=runtime,)@@ -243,7 +243,7 @@ def test_tool_node_passes_node_execution_id_when_runtime_accepts_it(tool_node: Ttool_node._runtime.get_runtime = MagicMock(return_value=runtime_handle)result = tool_node._get_tool_runtime(- variable_pool=tool_node.graph_runtime_state.variable_pool,+ variable_pool=tool_node.runtime_state.variable_pool,node_execution_id="node-execution-id",)@@ -251,6 +251,6 @@ def test_tool_node_passes_node_execution_id_when_runtime_accepts_it(tool_node: Ttool_node._runtime.get_runtime.assert_called_once_with(node_id="node-instance",node_data=tool_node.node_data,- variable_pool=tool_node.graph_runtime_state.variable_pool,+ variable_pool=tool_node.runtime_state.variable_pool,node_execution_id="node-execution-id",)diff --git a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.pyindex 987ef88696d..eb642540748 100644--- a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py+++ b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_invocation_behavior.py@@ -14,7 +14,7 @@ from graphon.graph import Graphfrom graphon.node_events import StreamCompletedEventfrom graphon.nodes.tool.entities import ToolNodeDatafrom graphon.nodes.tool.tool_node import ToolNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_graph_init_params@@ -49,12 +49,14 @@ def init_tool_node(config: dict[str, object]) -> ToolNode:conversation_variables=[],)- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())+ graph_runtime_state = RuntimeState(+ workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter()+ )# Create node factorynode_factory = DifyNodeFactory(- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,)Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start")@@ -64,8 +66,8 @@ def init_tool_node(config: dict[str, object]) -> ToolNode:node = ToolNode(node_id=str(uuid.uuid4()),data=ToolNodeData.model_validate(config["data"]),- graph_init_params=init_params,- graph_runtime_state=graph_runtime_state,+ init_params=init_params,+ runtime_state=graph_runtime_state,tool_file_manager=tool_file_manager,runtime=DifyToolNodeRuntime(init_params.run_context),)@@ -97,7 +99,7 @@ def test_tool_variable_invoke() -> None:"decrypt_tool_parameters",return_value={"format": "%Y-%m-%d %H:%M:%S"},):- node.graph_runtime_state.variable_pool.add(["1", "args1"], "1+1")+ node.runtime_state.variable_pool.add(["1", "args1"], "1+1")# execute noderesult = node._run()diff --git a/api/tests/unit_tests/core/workflow/nodes/trigger_plugin/test_trigger_event_node.py b/api/tests/unit_tests/core/workflow/nodes/trigger_plugin/test_trigger_event_node.pyindex c5ac8d2ce2d..68f7f30b58e 100644--- a/api/tests/unit_tests/core/workflow/nodes/trigger_plugin/test_trigger_event_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/trigger_plugin/test_trigger_event_node.py@@ -4,19 +4,19 @@ from core.trigger.constants import TRIGGER_PLUGIN_NODE_TYPEfrom core.workflow.nodes.trigger_plugin.entities import TriggerEventNodeDatafrom core.workflow.nodes.trigger_plugin.trigger_event_node import TriggerEventNodefrom core.workflow.system_variables import build_system_variables-from graphon.entities import GraphInitParamsfrom graphon.enums import WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus-from graphon.runtime import GraphRuntimeState+from graphon.runtime import InitParams, RuntimeStatefrom tests.workflow_test_utils import build_test_graph_init_params, build_test_variable_pool-def _build_context(graph_config: Mapping[str, object]) -> tuple[GraphInitParams, GraphRuntimeState]:+def _build_context(graph_config: Mapping[str, object]) -> tuple[InitParams, RuntimeState]:init_params = build_test_graph_init_params(graph_config=graph_config,user_from="account",invoke_from="debugger",)- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=build_test_variable_pool(variables=build_system_variables(user_id="user", files=[]),node_id="node-1",@@ -45,8 +45,8 @@ def test_trigger_event_node_run_populates_trigger_info_metadata() -> None:node = TriggerEventNode(node_id="node-1",data=_build_node_data(),- graph_init_params=init_params,- graph_runtime_state=runtime_state,+ init_params=init_params,+ runtime_state=runtime_state,)result = node._run()diff --git a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_file_conversion.py b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_file_conversion.pyindex 8a504f7d1f8..758b5f0098c 100644--- a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_file_conversion.py+++ b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_file_conversion.py@@ -18,9 +18,8 @@ from core.workflow.nodes.trigger_webhook.entities import ()from core.workflow.nodes.trigger_webhook.node import TriggerWebhookNodefrom core.workflow.system_variables import default_system_variables-from graphon.entities import GraphInitParamsfrom graphon.enums import WorkflowNodeExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom tests.workflow_test_utils import build_test_variable_pool@@ -30,7 +29,7 @@ def create_webhook_node(tenant_id: str = "test-tenant",) -> TriggerWebhookNode:"""Helper function to create a webhook node with proper initialization."""- graph_init_params = GraphInitParams(+ graph_init_params = InitParams(workflow_id="test-workflow",graph_config={},run_context={@@ -45,7 +44,8 @@ def create_webhook_node(call_depth=0,)- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=variable_pool,start_at=0,)@@ -53,8 +53,8 @@ def create_webhook_node(node = TriggerWebhookNode(node_id="webhook-node-1",data=webhook_data,- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,)# Attach a lightweight app_config onto runtime state for tenant lookupsdiff --git a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.pyindex d99189c446c..8606f5631ca 100644--- a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py+++ b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py@@ -13,17 +13,16 @@ from core.workflow.nodes.trigger_webhook.entities import ()from core.workflow.nodes.trigger_webhook.node import TriggerWebhookNodefrom core.workflow.system_variables import default_system_variables-from graphon.entities import GraphInitParamsfrom graphon.enums import WorkflowNodeExecutionStatusfrom graphon.file import File, FileTransferMethod, FileType-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom graphon.variables import FileVariable, StringVariablefrom tests.workflow_test_utils import build_test_variable_pooldef create_webhook_node(webhook_data: WebhookData, variable_pool: VariablePool) -> TriggerWebhookNode:"""Helper function to create a webhook node with proper initialization."""- graph_init_params = GraphInitParams(+ graph_init_params = InitParams(workflow_id="1",graph_config={},run_context={@@ -37,15 +36,16 @@ def create_webhook_node(webhook_data: WebhookData, variable_pool: VariablePool)},call_depth=0,)- runtime_state = GraphRuntimeState(+ runtime_state = RuntimeState(+ workflow_id="test-workflow",variable_pool=variable_pool,start_at=0,)node = TriggerWebhookNode(node_id="1",data=webhook_data,- graph_init_params=graph_init_params,- graph_runtime_state=runtime_state,+ init_params=graph_init_params,+ runtime_state=runtime_state,)# Provide tenant_id for conversion pathdiff --git a/api/tests/unit_tests/core/workflow/test_enrich_pause_reasons.py b/api/tests/unit_tests/core/workflow/test_enrich_pause_reasons.pyindex d6a33889cbb..55aa24ce44d 100644--- a/api/tests/unit_tests/core/workflow/test_enrich_pause_reasons.py+++ b/api/tests/unit_tests/core/workflow/test_enrich_pause_reasons.py@@ -1,10 +1,16 @@+from datetime import datetime+import pytest-from core.repositories.human_input_repository import HumanInputFormSubmissionRepository+from core.repositories.human_input_repository import HumanInputFormRecordfrom core.workflow.human_input_policy import FormDisposition, enrich_human_input_pause_reasons-from core.workflow.nodes.human_input.boundary import enrich_graph_pause_reasons+from core.workflow.nodes.human_input.boundary import build_human_input_pause_reason+from core.workflow.nodes.human_input.entities import FormDefinition, SelectInputConfig, StringListSource+from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus, ValueSourceTypefrom core.workflow.nodes.human_input.pause_reason import DifyHITLEventTypefrom graphon.entities.pause_reason import HitlRequired+from graphon.runtime import VariablePool+from models.human_input import HumanInputForm_HUMAN_INPUT_REASON = {"TYPE": DifyHITLEventType.HUMAN_INPUT_REQUIRED, "form_id": "f1"}@@ -66,19 +72,44 @@ def test_pause_reason_payload_carries_approval_channels_through_factory():assert payload.form_token is None[email protected]("sqlite_session_factory")-def test_enrich_graph_pause_reasons_raises_when_hitl_form_record_is_missing():- form_repository = HumanInputFormSubmissionRepository()-- with pytest.raises(LookupError, match="form-123"):- enrich_graph_pause_reasons(- reasons=[- HitlRequired(- session_id="form-123",- node_id="node-1",- node_title="Ask Name",- )- ],- form_repository=form_repository,- variable_pool=None,- )+def test_build_pause_reason_keeps_constant_options_from_materialized_form():+ definition = FormDefinition(+ form_content="Choose",+ rendered_content="Choose",+ inputs=[+ SelectInputConfig(+ output_variable_name="decision",+ option_source=StringListSource(type=ValueSourceType.CONSTANT, value=[]),+ )+ ],+ node_title="Choose",+ expiration_time=datetime(2026, 1, 2),+ )+ record = HumanInputFormRecord.from_models(+ HumanInputForm(+ id="form-123",+ tenant_id="tenant",+ app_id="app",+ workflow_run_id="run",+ node_id="human-input",+ form_kind=HumanInputFormKind.RUNTIME,+ form_definition=definition.model_dump_json(),+ rendered_content=definition.rendered_content,+ created_at=datetime(2026, 1, 1),+ expiration_time=definition.expiration_time,+ status=HumanInputFormStatus.WAITING,+ ),+ None,+ )+ parent_pool = VariablePool()+ parent_pool.add(("start", "options"), ["wrong"])++ reason = build_human_input_pause_reason(+ reason=HitlRequired(session_id="form-123", node_id="human-input", node_title="Choose"),+ record=record,+ variable_pool=parent_pool,+ )++ assert isinstance(reason.inputs[0], SelectInputConfig)+ assert reason.inputs[0].option_source.type == ValueSourceType.CONSTANT+ assert reason.inputs[0].option_source.value == []diff --git a/api/tests/unit_tests/core/workflow/test_human_input_adapter.py b/api/tests/unit_tests/core/workflow/test_human_input_adapter.pyindex 30352738d0b..cdb374e207a 100644--- a/api/tests/unit_tests/core/workflow/test_human_input_adapter.py+++ b/api/tests/unit_tests/core/workflow/test_human_input_adapter.py@@ -21,6 +21,15 @@ from graphon.nodes.base.variable_template_parser import VariableTemplateParserfrom graphon.runtime import VariablePool+def test_adapt_legacy_variable_aggregator_without_changing_saved_data() -> None:+ data = {"type": "variable-assigner", "output_type": "string", "variables": [["source", "value"]]}++ normalized = adapt_node_data_for_graph(data)++ assert normalized == {**data, "type": "variable-aggregator"}+ assert data["type"] == "variable-assigner"++def test_email_delivery_config_helpers_render_and_sanitize_text() -> None:variable_pool = VariablePool()variable_pool.add(["node", "value"], "42")diff --git a/api/tests/unit_tests/core/workflow/test_human_input_callback.py b/api/tests/unit_tests/core/workflow/test_human_input_callback.pyindex a18fa3c8a21..4ef27b8223d 100644--- a/api/tests/unit_tests/core/workflow/test_human_input_callback.py+++ b/api/tests/unit_tests/core/workflow/test_human_input_callback.py@@ -8,10 +8,21 @@ import pytestfrom pytest_mock import MockerFixturefrom sqlalchemy.orm import Session, sessionmaker-from core.repositories.human_input_repository import FormCreateParams, HumanInputFormRepositoryImpl+from core.repositories.human_input_repository import (+ FormCreateParams,+ HumanInputFormEntity,+ HumanInputFormRepositoryImpl,+ HumanInputFormSubmissionRepository,+)from core.workflow.nodes.human_input.callback import DifyHITLCallback-from core.workflow.nodes.human_input.entities import HumanInputNodeData, ParagraphInputConfig, UserActionConfig-from core.workflow.nodes.human_input.enums import HumanInputFormStatus+from core.workflow.nodes.human_input.entities import (+ HumanInputNodeData,+ ParagraphInputConfig,+ SelectInputConfig,+ StringListSource,+ UserActionConfig,+)+from core.workflow.nodes.human_input.enums import HumanInputFormStatus, ValueSourceTypefrom core.workflow.nodes.human_input.session_binding import SessionBindingfrom graphon.runtime import VariablePoolfrom graphon.variables.factory import build_segment@@ -75,6 +86,38 @@ def test_dify_hitl_callback_creates_pause_requested_for_new_form(assert params.node_id == "node-1"+def test_dify_hitl_callback_persists_variable_select_options(+ repository: HumanInputFormRepositoryImpl, mocker: MockerFixture+) -> None:+ create = mocker.spy(repository, "create_form")+ variable_pool = VariablePool()+ variable_pool.add(("source", "options"), ["approve", "reject"])+ callback = DifyHITLCallback(+ form_repository=repository,+ node_data=HumanInputNodeData(+ title="Approval",+ inputs=[+ SelectInputConfig(+ output_variable_name="decision",+ option_source=StringListSource(+ type=ValueSourceType.VARIABLE,+ selector=("source", "options"),+ ),+ )+ ],+ ),+ )++ callback(_Context(workflow_execution_id="run-1", node_id="node-1", variable_pool=variable_pool))++ params: FormCreateParams = create.call_args.args[0]+ select_input = params.form_config.inputs[0]+ assert isinstance(select_input, SelectInputConfig)+ assert select_input.option_source.type == ValueSourceType.CONSTANT+ assert select_input.option_source.selector == ()+ assert select_input.option_source.value == ["approve", "reject"]++def test_dify_hitl_callback_scopes_form_to_node_execution(repository: HumanInputFormRepositoryImpl, mocker: MockerFixture) -> None:@@ -177,33 +220,37 @@ def test_dify_hitl_callback_returns_timeout_for_explicit_timeout_form(}-def test_dify_hitl_callback_returns_timeout_for_waiting_form_past_node_deadline(- repository: HumanInputFormRepositoryImpl, sqlite_session: Session-) -> None:- form = HumanInputForm(- id="form-1",- rendered_content="<p>Please approve</p>",- selected_action_id=None,- submitted_data=None,- status=HumanInputFormStatus.WAITING,- created_at=naive_utc_now(),- expiration_time=naive_utc_now() - timedelta(minutes=1),- submitted_at=None,- tenant_id="tenant-1",- app_id="app-1",- workflow_run_id="run-1",- node_id="node-1",- form_definition="{}",- )- sqlite_session.add(form)- sqlite_session.commit()+def _past_deadline_callback(+ session: Session,+) -> tuple[DifyHITLCallback, HumanInputFormRepositoryImpl]:+ repository = HumanInputFormRepositoryImpl(tenant_id="tenant-1", app_id="app-1", workflow_execution_id="run-1")callback = DifyHITLCallback(form_repository=repository,- node_data=HumanInputNodeData(title="Approval", form_content="Please approve"),+ node_data=HumanInputNodeData(+ title="Approval",+ form_content="<p>Please approve</p>",+ inputs=[ParagraphInputConfig(output_variable_name="answer")],+ user_actions=[UserActionConfig(id="approve", title="Approve")],+ ),+ execution_id_getter=lambda: "form-1",)+ callback(_ctx("run-1", "node-1"))+ form = session.get(HumanInputForm, "form-1")+ assert form is not None+ form.expiration_time = naive_utc_now() - timedelta(minutes=1)+ session.commit()+ return callback, repository+++def test_dify_hitl_callback_persists_timeout_for_waiting_form_past_node_deadline(sqlite_session: Session) -> None:+ callback, _ = _past_deadline_callback(sqlite_session)decision = callback(_ctx("run-1", "node-1"))+ sqlite_session.expire_all()+ form = sqlite_session.get(HumanInputForm, "form-1")+ assert form is not None+ assert form.status == HumanInputFormStatus.TIMEOUTassert decision.selected_handle == "__timeout"assert decision.outputs == {"__action_id": build_segment(""),@@ -212,6 +259,47 @@ def test_dify_hitl_callback_returns_timeout_for_waiting_form_past_node_deadline(}[email protected]("terminal_status", [HumanInputFormStatus.SUBMITTED, HumanInputFormStatus.EXPIRED])+def test_dify_hitl_callback_timeout_uses_concurrent_terminal_state(+ monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, terminal_status: HumanInputFormStatus+) -> None:+ callback, repository = _past_deadline_callback(sqlite_session)+ get_form = repository.get_form++ def finish_after_read(node_id: str, *, form_id: str | None = None) -> HumanInputFormEntity | None:+ waiting_form = get_form(node_id, form_id=form_id)+ monkeypatch.setattr(repository, "get_form", get_form)+ if terminal_status == HumanInputFormStatus.SUBMITTED:+ HumanInputFormSubmissionRepository().mark_submitted(+ form_id="form-1",+ recipient_id=None,+ selected_action_id="approve",+ form_data={"answer": "accepted before timeout"},+ submission_user_id="user-1",+ submission_end_user_id=None,+ )+ else:+ form = sqlite_session.get(HumanInputForm, "form-1")+ assert form is not None+ form.status = HumanInputFormStatus.EXPIRED+ sqlite_session.commit()+ return waiting_form++ monkeypatch.setattr(repository, "get_form", finish_after_read)+ if terminal_status == HumanInputFormStatus.SUBMITTED:+ decision = callback(_ctx("run-1", "node-1"))+ assert decision.selected_handle == "approve"+ assert decision.inputs == {"answer": build_segment("accepted before timeout")}+ else:+ with pytest.raises(AssertionError, match="globally expired human input form"):+ callback(_ctx("run-1", "node-1"))++ sqlite_session.expire_all()+ form = sqlite_session.get(HumanInputForm, "form-1")+ assert form is not None+ assert form.status == terminal_status++def test_dify_hitl_callback_rejects_expired_form_as_invalid_resume_state(repository: HumanInputFormRepositoryImpl, sqlite_session: Session) -> None:diff --git a/api/tests/unit_tests/core/workflow/test_human_input_policy.py b/api/tests/unit_tests/core/workflow/test_human_input_policy.pyindex 99eb1ba01c4..ee3af428f7c 100644--- a/api/tests/unit_tests/core/workflow/test_human_input_policy.py+++ b/api/tests/unit_tests/core/workflow/test_human_input_policy.py@@ -9,6 +9,7 @@ from core.workflow.human_input_policy import (from core.workflow.nodes.human_input.entities import SelectInputConfig, StringListSourcefrom core.workflow.nodes.human_input.enums import ValueSourceTypefrom graphon.runtime import VariablePool+from graphon.variables import ArrayStringSegmentfrom models.human_input import RecipientType@@ -57,9 +58,10 @@ def test_preferred_form_token_skips_prioritized_type_with_empty_token() -> None:assert get_preferred_form_token(recipients) == "console-token"-def test_resolve_variable_select_input_options_uses_runtime_values() -> None:[email protected]("child_options", [["approve", "reject"], []])+def test_resolve_variable_select_input_options_uses_runtime_values(child_options: list[str]) -> None:variable_pool = VariablePool()- variable_pool.add(("start", "options"), ["approve", "reject"])+ variable_pool.add(("start", "options"), ArrayStringSegment(value=child_options))inputs: list[SelectInputConfig] = [SelectInputConfig(output_variable_name="decision",@@ -73,7 +75,9 @@ def test_resolve_variable_select_input_options_uses_runtime_values() -> None:resolved = resolve_variable_select_input_options(inputs, variable_pool=variable_pool)assert isinstance(resolved[0], SelectInputConfig)- assert resolved[0].option_source.value == ["approve", "reject"]+ assert resolved[0].option_source.type == ValueSourceType.CONSTANT+ assert resolved[0].option_source.selector == ()+ assert resolved[0].option_source.value == child_optionsdef test_resolve_variable_select_input_options_keeps_original_when_value_not_string_list() -> None:diff --git a/api/tests/unit_tests/core/workflow/test_llm_node.py b/api/tests/unit_tests/core/workflow/test_llm_node.pyindex cafaa3f47a9..f4dbb3af149 100644--- a/api/tests/unit_tests/core/workflow/test_llm_node.py+++ b/api/tests/unit_tests/core/workflow/test_llm_node.py@@ -12,7 +12,8 @@ from graphon.nodes.llm.runtime_protocols import LLMPollingCapableProtocoldef test_dify_llm_node_finalizes_polling_when_generator_is_closed(monkeypatch: pytest.MonkeyPatch) -> None:def invoke(*args: object, **kwargs: object) -> Generator[object]:- _ = args, kwargs+ _ = args+ assert kwargs["file_outputs"] == []yield sentinel.eventyield sentinel.unconsumed@@ -22,6 +23,7 @@ def test_dify_llm_node_finalizes_polling_when_generator_is_closed(monkeypatch: pnode._polling_finalizer = finalizerevents = node._invoke_llm_with_polling(+ file_outputs=[],polling_model=cast(LLMPollingCapableProtocol, SimpleNamespace()),prompt_messages=[],stop=None,@@ -44,6 +46,7 @@ def test_dify_llm_node_finalizes_polling_when_polling_fails(monkeypatch: pytest.node = object.__new__(DifyLLMNode)node._polling_finalizer = finalizerevents = node._invoke_llm_with_polling(+ file_outputs=[],polling_model=cast(LLMPollingCapableProtocol, SimpleNamespace()),prompt_messages=[],stop=None,diff --git a/api/tests/unit_tests/core/workflow/test_node_factory.py b/api/tests/unit_tests/core/workflow/test_node_factory.pyindex 43788877965..990f634f82e 100644--- a/api/tests/unit_tests/core/workflow/test_node_factory.py+++ b/api/tests/unit_tests/core/workflow/test_node_factory.py@@ -16,7 +16,9 @@ from core.workflow.llm_node import DifyLLMNodefrom core.workflow.node_runtime import DifyPreparedLLMfrom core.workflow.nodes.knowledge_index import KNOWLEDGE_INDEX_NODE_TYPEfrom graphon.entities.base_node_data import BaseNodeData-from graphon.enums import BuiltinNodeTypes, NodeType+from graphon.enums import BuiltinNodeTypes, NodeExecutionType, NodeType+from graphon.graph import Graph+from graphon.graph.validation import GraphValidationErrorfrom graphon.model_runtime.entities.common_entities import I18nObjectfrom graphon.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelFeature, ModelTypefrom graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModel@@ -25,10 +27,12 @@ from graphon.nodes.llm.entities import LLMNodeDatafrom graphon.nodes.llm.node import LLMNodefrom graphon.nodes.llm.runtime_protocols import LLMPollingCapableProtocolfrom graphon.nodes.parameter_extractor.entities import ParameterExtractorNodeData+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variables.segments import ArrayObjectSegment, ObjectSegment, StringSegmentfrom models.base import TypeBasefrom models.model import AppMode, Conversation, ConversationFromSourcefrom tests.unit_tests.core.model_fixtures import make_model_instance+from tests.workflow_test_utils import build_test_graph_init_params@pytest.fixture@@ -247,6 +251,59 @@ class TestDifyGraphInitContext:assert result.call_depth == 2+ "invalid_kind",+ [None, "CYCLE", "MISSING_NODE", "INVALID_EDGE", "DUPLICATE_NODE_ID", "DUPLICATE_EDGE_ID"],+)+def test_graph_validation_with_dify_node_factory(invalid_kind, monkeypatch):+ graph_config = {+ "nodes": [+ {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}},+ *[+ {+ "id": node_id,+ "data": {"type": "template-transform", "title": node_id, "template": node_id, "variables": []},+ }+ for node_id in ("left", "right")+ ],+ {"id": "end", "data": {"type": "end", "title": "End", "outputs": []}},+ ],+ "edges": [+ {"id": f"{source}-{target}", "source": source, "target": target}+ for source, target in (("start", "left"), ("start", "right"), ("left", "end"), ("right", "end"))+ ],+ }+ match invalid_kind:+ case "CYCLE":+ graph_config["edges"].append({"source": "end", "target": "left"})+ case "MISSING_NODE":+ graph_config["edges"][0]["target"] = "missing"+ case "INVALID_EDGE":+ graph_config["edges"][0]["sourceHandle"] = None+ case "DUPLICATE_NODE_ID":+ graph_config["nodes"].append(graph_config["nodes"][0])+ case "DUPLICATE_EDGE_ID":+ graph_config["edges"].append(graph_config["edges"][0])++ factory = node_factory.DifyNodeFactory(+ build_test_graph_init_params(graph_config=graph_config),+ RuntimeState(variable_pool=VariablePool(), start_at=0, workflow_id="workflow"),+ )+ if invalid_kind is None:+ graph = Graph.init(graph_config=graph_config, node_factory=factory, root_node_id="start")+ assert set(graph.nodes) == {"start", "left", "right", "end"}+ assert len(graph.edges) == 4+ else:++ def reject_node_creation(*_args, **_kwargs):+ pytest.fail("Invalid graphs must fail before node construction")++ monkeypatch.setattr(node_factory.DifyNodeFactory, "create_node", reject_node_creation)+ with pytest.raises(GraphValidationError) as error:+ Graph.init(graph_config=graph_config, node_factory=factory, root_node_id="start")+ assert [issue.code for issue in error.value.issues] == [invalid_kind]++class TestDefaultWorkflowCodeExecutor:def test_execute_delegates_to_code_executor(self, monkeypatch: pytest.MonkeyPatch):executor = node_factory.DefaultWorkflowCodeExecutor()@@ -347,8 +404,8 @@ class TestDifyNodeFactoryCreateNode:@pytest.fixturedef factory(self):factory = object.__new__(node_factory.DifyNodeFactory)- factory.graph_init_params = sentinel.graph_init_params- factory.graph_runtime_state = SimpleNamespace(variable_pool=MagicMock())+ factory.init_params = sentinel.init_params+ factory.runtime_state = SimpleNamespace(variable_pool=MagicMock())factory._dify_context = SimpleNamespace(tenant_id="tenant-id",app_id="app-id",@@ -382,6 +439,50 @@ class TestDifyNodeFactoryCreateNode:with pytest.raises(ValueError, match="No class mapping found for node type: missing"):factory.create_node({"id": "node-id", "data": {"type": "missing"}})+ def test_validate_node_resolves_schema_without_constructing_node(self, monkeypatch: pytest.MonkeyPatch, factory):+ constructor = _node_constructor(return_value=sentinel.node)+ constructor.execution_type = NodeExecutionType.BRANCH+ monkeypatch.setattr(factory, "_resolve_node_class", MagicMock(return_value=constructor))++ execution_type = factory.validate_node({"id": "node-id", "data": {"type": BuiltinNodeTypes.START}})++ assert execution_type == NodeExecutionType.BRANCH+ constructor.validate_node_data.assert_called_once()+ constructor.assert_not_called()++ def test_validate_node_rejects_invalid_human_input_delivery_config(+ self,+ monkeypatch: pytest.MonkeyPatch,+ factory,+ ):+ constructor = _node_constructor(return_value=sentinel.node)+ monkeypatch.setattr(factory, "_resolve_node_class", MagicMock(return_value=constructor))++ with pytest.raises(+ ValueError,+ match=r"(?s)0\.email\.config\.subject.*valid string.*0\.email\.config\.body.*valid string",+ ):+ factory.validate_node(+ {+ "id": "human-input",+ "data": {+ "type": BuiltinNodeTypes.HUMAN_INPUT,+ "delivery_methods": [+ {+ "type": "email",+ "config": {+ "recipients": {"items": []},+ "subject": 123,+ "body": [],+ },+ }+ ],+ },+ }+ )++ constructor.assert_not_called()+def test_rejects_missing_class_mapping(self, monkeypatch: pytest.MonkeyPatch, factory):monkeypatch.setattr(factory,@@ -419,8 +520,8 @@ class TestDifyNodeFactoryCreateNode:kwargs = matched_node_class.call_args.kwargsassert kwargs["node_id"] == "node-id"_assert_constructor_node_data(kwargs["data"], node_id="node-id", node_type=BuiltinNodeTypes.START, version="9")- assert kwargs["graph_init_params"] is sentinel.graph_init_params- assert kwargs["graph_runtime_state"] is factory.graph_runtime_state+ assert kwargs["init_params"] is sentinel.init_params+ assert kwargs["runtime_state"] is factory.runtime_statelatest_node_class.assert_not_called()def test_falls_back_to_latest_class_when_version_specific_mapping_is_missing(@@ -441,8 +542,8 @@ class TestDifyNodeFactoryCreateNode:kwargs = latest_node_class.call_args.kwargsassert kwargs["node_id"] == "node-id"_assert_constructor_node_data(kwargs["data"], node_id="node-id", node_type=BuiltinNodeTypes.START, version="9")- assert kwargs["graph_init_params"] is sentinel.graph_init_params- assert kwargs["graph_runtime_state"] is factory.graph_runtime_state+ assert kwargs["init_params"] is sentinel.init_params+ assert kwargs["runtime_state"] is factory.runtime_state@pytest.mark.parametrize(("node_type", "constructor_name"),@@ -478,8 +579,8 @@ class TestDifyNodeFactoryCreateNode:kwargs = constructor.call_args.kwargsassert kwargs["node_id"] == "node-id"_assert_constructor_node_data(kwargs["data"], node_id="node-id", node_type=node_type)- assert kwargs["graph_init_params"] is sentinel.graph_init_params- assert kwargs["graph_runtime_state"] is factory.graph_runtime_state+ assert kwargs["init_params"] is sentinel.init_params+ assert kwargs["runtime_state"] is factory.runtime_stateif constructor_name == "CodeNode":assert kwargs["code_executor"] is sentinel.code_executor@@ -612,7 +713,7 @@ class TestDifyNodeFactoryCreateNode:"vision": {"enabled": False},})- factory.graph_runtime_state.variable_pool.get.return_value = ObjectSegment(+ factory.runtime_state.variable_pool.get.return_value = ObjectSegment(value={"provider": "new-provider","name": "new-model",@@ -627,7 +728,7 @@ class TestDifyNodeFactoryCreateNode:assert result.model.name == "new-model"assert result.model.mode == node_data.model.modeassert result.model.completion_params == {"temperature": 0.8}- factory.graph_runtime_state.variable_pool.get.assert_called_once_with(("env", "for_summarize"))+ factory.runtime_state.variable_pool.get.assert_called_once_with(("env", "for_summarize"))def test_resolve_llm_model_reference_keeps_node_parameters_for_legacy_variable(self, factory):node_data = LLMNodeData.model_validate(@@ -646,7 +747,7 @@ class TestDifyNodeFactoryCreateNode:"vision": {"enabled": False},})- factory.graph_runtime_state.variable_pool.get.return_value = ObjectSegment(+ factory.runtime_state.variable_pool.get.return_value = ObjectSegment(value={"provider": "new-provider", "name": "new-model", "mode": "chat"})@@ -666,7 +767,7 @@ class TestDifyNodeFactoryCreateNode:"vision": {"enabled": False},})- factory.graph_runtime_state.variable_pool.get.return_value = ObjectSegment(+ factory.runtime_state.variable_pool.get.return_value = ObjectSegment(value={"provider": "provider", "name": "model", "mode": "completion"})@@ -685,7 +786,7 @@ class TestDifyNodeFactoryCreateNode:"vision": {"enabled": False},})- factory.graph_runtime_state.variable_pool.get.return_value = None+ factory.runtime_state.variable_pool.get.return_value = Nonewith pytest.raises(ValueError, match="shared_model.*not found"):factory._resolve_llm_model_reference(node_data)@@ -706,7 +807,7 @@ class TestDifyNodeFactoryCreateNode:result = factory._resolve_llm_model_reference(node_data)assert result is node_data- factory.graph_runtime_state.variable_pool.get.assert_not_called()+ factory.runtime_state.variable_pool.get.assert_not_called()def test_build_llm_compatible_node_init_kwargs_uses_polling_wrapper_for_polling_llm_node(self, factory):node_data = LLMNodeData.model_validate(@@ -766,13 +867,13 @@ class TestDifyNodeFactoryCreateNode:assert type(wrapped) is DifyPreparedLLMassert not isinstance(wrapped, LLMPollingCapableProtocol)- def test_create_node_passes_alias_preserving_llm_data_to_constructor(self, monkeypatch, factory):+ def test_create_node_passes_canonical_llm_data_to_constructor(self, monkeypatch, factory):created_node = object()constructor = _node_constructor(return_value=created_node)constructor.validate_node_data.side_effect = lambda node_data: LLMNodeData.model_validate(node_data.model_dump(mode="python") if isinstance(node_data, BaseNodeData) else node_data)- monkeypatch.setattr(factory, "_resolve_node_class", MagicMock(return_value=constructor))+ monkeypatch.setattr(factory, "_resolve_node_class", lambda **_kwargs: constructor)monkeypatch.setattr(factory, "_build_llm_compatible_node_init_kwargs", MagicMock(return_value={}))node_config = {@@ -799,12 +900,12 @@ class TestDifyNodeFactoryCreateNode:data = constructor.call_args.kwargs["data"]assert isinstance(data, Mapping)- assert data["structured_output_enabled"] is True- assert "structured_output_switch_on" not in data+ assert data["structured_output_switch_on"] is True+ assert "structured_output_enabled" not in dataassert LLMNodeData.model_validate(data).structured_output_enabled is Truedef test_create_node_preserves_structured_output_switch_after_graphon_constructor(self, monkeypatch, factory):- factory.graph_init_params = SimpleNamespace(+ factory.init_params = SimpleNamespace(workflow_id="workflow-id",graph_config={},run_context={},@@ -849,7 +950,7 @@ class TestDifyNodeFactoryCreateNode:assert node.node_data.structured_output_enabled is Truedef test_create_node_uses_dify_llm_node_for_persisted_version_one(self, monkeypatch, factory):- factory.graph_init_params = SimpleNamespace(+ factory.init_params = SimpleNamespace(workflow_id="workflow-id",graph_config={},run_context={},@@ -933,7 +1034,7 @@ class TestDifyNodeFactoryCreateNode:monkeypatch.setattr(factory,"_resolve_node_class",- MagicMock(return_value=constructor),+ lambda **_kwargs: constructor,)llm_init_kwargs = {"credentials_provider": sentinel.credentials_provider,@@ -964,8 +1065,8 @@ class TestDifyNodeFactoryCreateNode:constructor_kwargs = constructor.call_args.kwargsassert constructor_kwargs["node_id"] == "node-id"_assert_constructor_node_data(constructor_kwargs["data"], node_id="node-id", node_type=node_type)- assert constructor_kwargs["graph_init_params"] is sentinel.graph_init_params- assert constructor_kwargs["graph_runtime_state"] is factory.graph_runtime_state+ assert constructor_kwargs["init_params"] is sentinel.init_params+ assert constructor_kwargs["runtime_state"] is factory.runtime_stateassert constructor_kwargs["credentials_provider"] is sentinel.credentials_providerassert constructor_kwargs["model_factory"] is sentinel.model_factoryassert constructor_kwargs["model_instance"] is sentinel.model_instance@@ -1015,7 +1116,7 @@ class TestDifyNodeFactoryRetrieverAttachmentAccess:@pytest.fixturedef factory(self):factory = object.__new__(node_factory.DifyNodeFactory)- factory.graph_runtime_state = SimpleNamespace(variable_pool=MagicMock())+ factory.runtime_state = SimpleNamespace(variable_pool=MagicMock())return factorydef test_retriever_attachment_loader_is_typed_for_llm_node_data_only(self):@@ -1025,7 +1126,7 @@ class TestDifyNodeFactoryRetrieverAttachmentAccess:def test_build_retriever_attachment_loader_uses_llm_context_selector(self, factory):factory._file_reference_factory = sentinel.file_reference_factory- factory.graph_runtime_state.variable_pool.get.return_value = ArrayObjectSegment(+ factory.runtime_state.variable_pool.get.return_value = ArrayObjectSegment(value=[{"metadata": {@@ -1050,16 +1151,16 @@ class TestDifyNodeFactoryRetrieverAttachmentAccess:assert loader._segment_access_checker is not Noneassert loader._segment_access_checker("allowed-segment") is True- factory.graph_runtime_state.variable_pool.get.assert_called_once_with(["knowledge-node", "result"])+ factory.runtime_state.variable_pool.get.assert_called_once_with(["knowledge-node", "result"])def test_checker_rejects_missing_context_selector_without_reading_variable_pool(self, factory):checker = factory._build_retriever_segment_access_checker(None)assert checker("segment-id") is False- factory.graph_runtime_state.variable_pool.get.assert_not_called()+ factory.runtime_state.variable_pool.get.assert_not_called()def test_checker_rejects_non_knowledge_context_items(self, factory):- factory.graph_runtime_state.variable_pool.get.return_value = ArrayObjectSegment.model_construct(+ factory.runtime_state.variable_pool.get.return_value = ArrayObjectSegment.model_construct(value=["plain-text",{"metadata": "not-a-mapping"},@@ -1071,14 +1172,14 @@ class TestDifyNodeFactoryRetrieverAttachmentAccess:assert checker("segment-id") is Falsedef test_checker_rejects_non_array_context_value(self, factory):- factory.graph_runtime_state.variable_pool.get.return_value = StringSegment(value="not knowledge context")+ factory.runtime_state.variable_pool.get.return_value = StringSegment(value="not knowledge context")checker = factory._build_retriever_segment_access_checker(["knowledge-node", "result"])assert checker("segment-id") is Falsedef test_checker_allows_only_segments_from_selected_knowledge_context(self, factory):- factory.graph_runtime_state.variable_pool.get.return_value = ArrayObjectSegment(+ factory.runtime_state.variable_pool.get.return_value = ArrayObjectSegment(value=[{"metadata": {@@ -1093,7 +1194,7 @@ class TestDifyNodeFactoryRetrieverAttachmentAccess:assert checker("allowed-segment") is Trueassert checker("other-segment") is False- factory.graph_runtime_state.variable_pool.get.assert_any_call(["knowledge-node", "result"])+ factory.runtime_state.variable_pool.get.assert_any_call(["knowledge-node", "result"])class TestDifyNodeFactoryModelInstance:@@ -1146,7 +1247,7 @@ class TestDifyNodeFactoryMemory:def factory(self):factory = object.__new__(node_factory.DifyNodeFactory)factory._dify_context = SimpleNamespace(app_id="app-id")- factory.graph_runtime_state = SimpleNamespace(variable_pool=MagicMock())+ factory.runtime_state = SimpleNamespace(variable_pool=MagicMock())return factorydef test_returns_none_when_memory_is_not_configured(self, factory):@@ -1156,11 +1257,11 @@ class TestDifyNodeFactoryMemory:)assert result is None- factory.graph_runtime_state.variable_pool.get.assert_not_called()+ factory.runtime_state.variable_pool.get.assert_not_called()def test_uses_string_segment_conversation_id(self, monkeypatch: pytest.MonkeyPatch, factory):memory_config = sentinel.memory_config- factory.graph_runtime_state.variable_pool.get.return_value = StringSegment(value="conversation-id")+ factory.runtime_state.variable_pool.get.return_value = StringSegment(value="conversation-id")fetch_memory = MagicMock(return_value=sentinel.memory)monkeypatch.setattr(node_factory, "fetch_memory", fetch_memory)@@ -1170,7 +1271,7 @@ class TestDifyNodeFactoryMemory:)assert result is sentinel.memory- factory.graph_runtime_state.variable_pool.get.assert_called_once_with(("sys", "conversation_id"))+ factory.runtime_state.variable_pool.get.assert_called_once_with(("sys", "conversation_id"))fetch_memory.assert_called_once_with(conversation_id="conversation-id",app_id="app-id",@@ -1180,7 +1281,7 @@ class TestDifyNodeFactoryMemory:def test_ignores_non_string_segment_conversation_ids(self, monkeypatch: pytest.MonkeyPatch, factory):memory_config = sentinel.memory_config- factory.graph_runtime_state.variable_pool.get.return_value = sentinel.segment+ factory.runtime_state.variable_pool.get.return_value = sentinel.segmentfetch_memory = MagicMock(return_value=sentinel.memory)monkeypatch.setattr(node_factory, "fetch_memory", fetch_memory)diff --git a/api/tests/unit_tests/core/workflow/test_node_runtime.py b/api/tests/unit_tests/core/workflow/test_node_runtime.pyindex 5c4262da297..c6b12df6346 100644--- a/api/tests/unit_tests/core/workflow/test_node_runtime.py+++ b/api/tests/unit_tests/core/workflow/test_node_runtime.py@@ -1075,16 +1075,12 @@ def test_dify_human_input_runtime_restore_submitted_data_rehydrates_files() -> N),]runtime._file_reference_factory.build_from_mapping = MagicMock(side_effect=[file_value, *file_list_value]) # type: ignore[method-assign]- node_data = HumanInputNodeData(- title="Human Input",++ restored = runtime.restore_submitted_data(inputs=[FileInputConfig(output_variable_name="attachment"),FileListInputConfig(output_variable_name="attachments", number_limits=2),],- )-- restored = runtime.restore_submitted_data(- node_data=node_data,submitted_data={"attachment": {"upload_file_id": "upload-1", "type": "document", "transfer_method": "local_file"},"attachments": [diff --git a/api/tests/unit_tests/core/workflow/test_response_stream_filter_issue_170.py b/api/tests/unit_tests/core/workflow/test_response_stream_filter_issue_170.pyindex 8056da32acd..9a5476cc245 100644--- a/api/tests/unit_tests/core/workflow/test_response_stream_filter_issue_170.py+++ b/api/tests/unit_tests/core/workflow/test_response_stream_filter_issue_170.py@@ -1,9 +1,9 @@"""Unit coverage for Dify's in-memory ResponseStreamFilter boundary behavior."""from core.workflow.workflow_entry import iter_dify_graph_engine_events-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import GraphRunSucceededEvent, NodeRunStreamChunkEvent+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine_events import GraphRunSucceededEvent, NodeRunStreamChunkEventfrom tests.unit_tests.core.workflow.graph_engine.test_mock_config import MockConfig, MockConfigBuilderfrom tests.unit_tests.core.workflow.graph_engine.test_table_runner import WorkflowRunner@@ -54,12 +54,10 @@ def test_dify_response_stream_filter_handles_issue_170_shape() -> None:expected_answer = "# Du Fu\n\nSpring View\n\n# Li Bai\n\nQuiet Night Thought"- engine = GraphEngine(- workflow_id="test_workflow",+ engine = Engine(graph=graph,- graph_runtime_state=graph_runtime_state,+ runtime_state=graph_runtime_state,command_channel=InMemoryChannel(),- config=GraphEngineConfig(),)events = list(iter_dify_graph_engine_events(engine))diff --git a/api/tests/unit_tests/core/workflow/test_response_stream_filter_pause_resume.py b/api/tests/unit_tests/core/workflow/test_response_stream_filter_pause_resume.pyindex 02993ac9043..2f16ccbca0a 100644--- a/api/tests/unit_tests/core/workflow/test_response_stream_filter_pause_resume.py+++ b/api/tests/unit_tests/core/workflow/test_response_stream_filter_pause_resume.py@@ -20,11 +20,11 @@ from core.workflow.nodes.human_input.entities import HumanInputNodeData, UserActfrom core.workflow.nodes.human_input.enums import HumanInputFormStatusfrom core.workflow.system_variables import build_system_variablesfrom core.workflow.workflow_entry import iter_dify_graph_engine_events-from graphon.filters import GraphEventFilterContext, ResponseStreamFilter, filter_graph_events+from graphon.engine import Engine+from graphon.engine.command import InMemoryChannel+from graphon.engine.filter import EngineEventFilterContext, ResponseStreamFilter, filter_engine_events+from graphon.engine_events import GraphRunPausedEvent, GraphRunSucceededEvent, NodeRunStreamChunkEventfrom graphon.graph import Graph-from graphon.graph_engine import GraphEngine, GraphEngineConfig-from graphon.graph_engine.command_channels import InMemoryChannel-from graphon.graph_events import GraphRunPausedEvent, GraphRunSucceededEvent, NodeRunStreamChunkEventfrom graphon.nodes.answer.answer_node import AnswerNodefrom graphon.nodes.answer.entities import AnswerNodeDatafrom graphon.nodes.human_input.entities import HumanInputNodeData as GraphonHumanInputNodeData@@ -33,7 +33,7 @@ from graphon.nodes.if_else.entities import IfElseNodeDatafrom graphon.nodes.if_else.if_else_node import IfElseNodefrom graphon.nodes.start.entities import StartNodeDatafrom graphon.nodes.start.start_node import StartNode-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.utils.condition.entities import Conditionfrom libs.datetime_utils import naive_utc_nowfrom models.human_input import HumanInputForm@@ -42,7 +42,7 @@ from tests.workflow_test_utils import build_test_graph_init_paramsWORKFLOW_EXECUTION_ID = "wf-exec-38525"-def _build_graph(runtime_state: GraphRuntimeState, form_repository: HumanInputFormRepository) -> Graph:+def _build_graph(runtime_state: RuntimeState, form_repository: HumanInputFormRepository) -> Graph:params = build_test_graph_init_params(workflow_id="wf",graph_config={"nodes": [], "edges": []},@@ -53,8 +53,8 @@ def _build_graph(runtime_state: GraphRuntimeState, form_repository: HumanInputFostart_node = StartNode(node_id="start",data=StartNodeData(title="start", variables=[]),- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)if_else_node = IfElseNode(@@ -75,8 +75,8 @@ def _build_graph(runtime_state: GraphRuntimeState, form_repository: HumanInputFo)],),- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)human_data = HumanInputNodeData(@@ -88,30 +88,30 @@ def _build_graph(runtime_state: GraphRuntimeState, form_repository: HumanInputFohuman_node = HumanInputNode(node_id="human_input",data=GraphonHumanInputNodeData.model_validate(human_data.model_dump()),- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,hitl_callback=DifyHITLCallback(form_repository=form_repository, node_data=human_data),)answer_false_node = AnswerNode(node_id="answer_false",data=AnswerNodeData(title="answer_false", answer="unreachable branch"),- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)answer_after_pause = AnswerNode(node_id="answer_after_pause",data=AnswerNodeData(title="answer_after_pause", answer="Post-branch answer chunk 1"),- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)answer_after_pause_2 = AnswerNode(node_id="answer_after_pause_2",data=AnswerNodeData(title="answer_after_pause_2", answer="Post-branch answer chunk 2"),- graph_init_params=params,- graph_runtime_state=runtime_state,+ init_params=params,+ runtime_state=runtime_state,)return (@@ -126,7 +126,7 @@ def _build_graph(runtime_state: GraphRuntimeState, form_repository: HumanInputFo)-def _build_runtime_state() -> GraphRuntimeState:+def _build_runtime_state() -> RuntimeState:variable_pool = VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id=WORKFLOW_EXECUTION_ID,@@ -138,7 +138,7 @@ def _build_runtime_state() -> GraphRuntimeState:conversation_variables=[],)variable_pool.add(("start", "category"), "fruit") # drives the if-else "true" branch- return GraphRuntimeState(variable_pool=variable_pool, start_at=0.0)+ return RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0.0)def test_if_else_human_input_pause_resume_answer_chunks_survive_resume(@@ -154,18 +154,16 @@ def test_if_else_human_input_pause_resume_answer_chunks_survive_resume(# ---- Phase 1: run to GraphRunPausedEvent ----runtime_state_1 = _build_runtime_state()graph_1 = _build_graph(runtime_state_1, repository)- engine_1 = GraphEngine(- workflow_id="wf",+ engine_1 = Engine(graph=graph_1,- graph_runtime_state=runtime_state_1,+ runtime_state=runtime_state_1,command_channel=InMemoryChannel(),- config=GraphEngineConfig(),)filter_1 = ResponseStreamFilter()phase1_events = list(- filter_graph_events(+ filter_engine_events(engine_1.run(),- context=GraphEventFilterContext.from_engine(engine_1),+ context=EngineEventFilterContext.from_engine(engine_1),filters=[filter_1],))@@ -187,14 +185,12 @@ def test_if_else_human_input_pause_resume_answer_chunks_survive_resume(form.status = HumanInputFormStatus.SUBMITTED# ---- Phase 2: rebuild engine + filter from snapshots, resume to completion ----- runtime_state_2 = GraphRuntimeState.from_snapshot(runtime_snapshot)+ runtime_state_2 = RuntimeState.from_snapshot(runtime_snapshot)graph_2 = _build_graph(runtime_state_2, repository)- engine_2 = GraphEngine(- workflow_id="wf",+ engine_2 = Engine(graph=graph_2,- graph_runtime_state=runtime_state_2,+ runtime_state=runtime_state_2,command_channel=InMemoryChannel(),- config=GraphEngineConfig(),)filter_2 = ResponseStreamFilter()filter_2.loads(response_filter_snapshot)diff --git a/api/tests/unit_tests/core/workflow/test_workflow_entry.py b/api/tests/unit_tests/core/workflow/test_workflow_entry.pyindex 23577105945..304b53344fb 100644--- a/api/tests/unit_tests/core/workflow/test_workflow_entry.py+++ b/api/tests/unit_tests/core/workflow/test_workflow_entry.py@@ -53,6 +53,150 @@ def _mock_ssrf_head(monkeypatch: pytest.MonkeyPatch):class TestWorkflowEntry:"""Test WorkflowEntry class methods."""+ @pytest.mark.parametrize("value", [0, False, "", [], {}])+ def test_qualified_input_preserves_falsy_values(self, value):+ variable_pool = VariablePool()++ WorkflowEntry.mapping_user_inputs_to_variable_pool(+ variable_mapping={"node.value": ["source", "value"]},+ user_inputs={"node.value": value, "value": "fallback"},+ variable_pool=variable_pool,+ tenant_id="tenant",+ )++ variable = variable_pool.get(["source", "value"])+ assert variable is not None+ assert variable.value == value++ @pytest.mark.parametrize("root_name", ["object", "structured_output"])+ @pytest.mark.parametrize("value", [0, False, "", [], {}])+ def test_nested_input_overrides_preserve_siblings(self, root_name, value):+ variable_pool = VariablePool()+ original = {"keep": "root sibling", "nested": {"leaf": "old", "keep": "nested sibling"}}+ variable_pool.add(["source", root_name], original)+ key = f"node.#source.{root_name}.nested.leaf#"++ WorkflowEntry.mapping_user_inputs_to_variable_pool(+ variable_mapping={key: ["source", root_name, "nested", "leaf"]},+ user_inputs={key: value, key.split(".", 1)[1]: "fallback"},+ variable_pool=variable_pool,+ tenant_id="tenant",+ )++ variable = variable_pool.get(["source", root_name])+ assert variable is not None+ assert variable.value == {"keep": "root sibling", "nested": {"leaf": value, "keep": "nested sibling"}}+ assert original["nested"]["leaf"] == "old"++ def test_nested_inputs_build_missing_root_and_preserve_other_explicit_inputs(self):+ variable_pool = VariablePool()++ WorkflowEntry.mapping_user_inputs_to_variable_pool(+ variable_mapping={+ "node.#source.object.leaf#": ["source", "object", "leaf"],+ "node.#source.object.nested.other#": ["source", "object", "nested", "other"],+ },+ user_inputs={"#source.object.leaf#": 0, "#source.object.nested.other#": False},+ variable_pool=variable_pool,+ tenant_id="tenant",+ )++ variable = variable_pool.get(["source", "object"])+ assert variable is not None+ assert variable.value == {"leaf": 0, "nested": {"other": False}}++ @pytest.mark.parametrize("descendant_first", [False, True])+ @pytest.mark.parametrize("value", ["Alice", 0, False, "", [], {}])+ def test_nested_overrides_take_precedence_over_explicit_ancestors_in_either_order(self, descendant_first, value):+ variable_pool = VariablePool()+ mappings = [+ ("node.#source.object#", ["source", "object"]),+ ("node.#source.object.name#", ["source", "object", "name"]),+ ]+ if descendant_first:+ mappings.reverse()+ root_input = {"other": "kept", "name": "old"}++ WorkflowEntry.mapping_user_inputs_to_variable_pool(+ variable_mapping=dict(mappings),+ user_inputs={"#source.object#": root_input, "#source.object.name#": value},+ variable_pool=variable_pool,+ tenant_id="tenant",+ )++ variable = variable_pool.get(["source", "object"])+ assert variable is not None+ assert variable.value == {"other": "kept", "name": value}+ assert root_input == {"other": "kept", "name": "old"}++ @pytest.mark.parametrize(+ ("data", "inputs", "expected"),+ [+ (+ {"type": "end", "outputs": [{"variable": "result", "value_selector": ["source", "value"]}]},+ {"#source.value#": "end input"},+ "end input",+ ),+ (+ {+ "type": "variable-aggregator",+ "output_type": "string",+ "variables": [["missing", "value"], ["source", "value"]],+ },+ {"#source.value#": "second branch"},+ "second branch",+ ),+ (+ {+ "type": "variable-assigner",+ "output_type": "string",+ "variables": [["source", "value"]],+ },+ {"#source.value#": "legacy branch"},+ "legacy branch",+ ),+ (+ {+ "type": "list-operator",+ "variable": ["source", "value"],+ "filter_by": {"enabled": False},+ "order_by": {"enabled": False},+ "limit": {"enabled": False},+ },+ {"#source.value#": [3, 1, 2]},+ [3, 1, 2],+ ),+ ],+ )+ def test_single_step_loads_explicit_inputs_for_nodes_without_graphon_extractors(self, data, inputs, expected):+ workflow = Workflow.new(+ tenant_id="tenant",+ app_id="app",+ type=WorkflowType.WORKFLOW,+ version=Workflow.VERSION_DRAFT,+ graph=json.dumps({"nodes": [{"id": "node", "data": {"title": "Node", **data}}], "edges": []}),+ features="{}",+ created_by="account",+ environment_variables=[],+ conversation_variables=[],+ rag_pipeline_variables=[],+ )+ workflow.id = "workflow"+ variable_pool = VariablePool()++ WorkflowEntry.single_step_run(+ workflow=workflow,+ node_id="node",+ user_id="user",+ user_inputs=inputs,+ variable_pool=variable_pool,+ execution_driver=None,+ )++ variable = variable_pool.get(["source", "value"])+ assert variable is not None+ assert variable.value == expected+def test_mapping_user_inputs_to_variable_pool_with_system_variables(self):"""Test mapping system variables from user inputs to variable pool."""# Initialize variable pool with system variables@@ -149,6 +293,7 @@ class TestWorkflowEntry:user_id="user",user_inputs={},variable_pool=variable_pool,+ execution_driver=None,)# Assertdiff --git a/api/tests/unit_tests/core/workflow/test_workflow_entry_helpers.py b/api/tests/unit_tests/core/workflow/test_workflow_entry_helpers.pyindex 384340c7d0b..36e60ce7a7f 100644--- a/api/tests/unit_tests/core/workflow/test_workflow_entry_helpers.py+++ b/api/tests/unit_tests/core/workflow/test_workflow_entry_helpers.py@@ -4,24 +4,25 @@ from types import SimpleNamespacefrom unittest.mock import MagicMock, patch, sentinelimport pytest+from pytest_mock import MockerFixturefrom core.app.apps.exc import GenerateTaskStoppedErrorfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFromfrom core.credit_usage import CreditUsageAppTypefrom core.workflow import workflow_entryfrom core.workflow.system_variables import default_system_variables+from graphon.engine.filter import ResponseStreamFilter+from graphon.engine_events import GraphRunFailedEvent, NodeRunSucceededEventfrom graphon.entities.base_node_data import BaseNodeDatafrom graphon.enums import NodeType, WorkflowNodeExecutionStatusfrom graphon.errors import WorkflowNodeRunFailedErrorfrom graphon.file import File, FileTransferMethod, FileType-from graphon.filters import ResponseStreamFilter-from graphon.graph_events import GraphRunFailedEvent, NodeRunSucceededEvent+from graphon.file.runtime import peek_workflow_file_runtime, use_workflow_file_runtimefrom graphon.node_events import NodeRunResultfrom graphon.nodes import BuiltinNodeTypesfrom graphon.runtime import VariablePoolfrom graphon.variables.variables import StringVariablefrom models.workflow import Workflow-from tests.unit_tests.config_override import config_overrides_contextfrom tests.unit_tests.model_factories import make_workflow@@ -39,26 +40,18 @@ def _build_minimal_workflow_entry(*,response_stream_filter: ResponseStreamFilter | None = None,) -> workflow_entry.WorkflowEntry:- """Construct a minimal WorkflowEntry with GraphEngine construction mocked out."""+ """Construct a minimal WorkflowEntry with Engine construction mocked out."""graph_engine = MagicMock()- graph_runtime_state = SimpleNamespace(_execution_context=None)+ graph_runtime_state = sentinel.runtime_state- monkeypatch.setattr(workflow_entry, "capture_current_context", lambda: sentinel.execution_context)- monkeypatch.setattr(workflow_entry, "GraphEngine", MagicMock(return_value=graph_engine))- monkeypatch.setattr(workflow_entry, "GraphEngineConfig", MagicMock(return_value=sentinel.graph_engine_config))+ monkeypatch.setattr(workflow_entry, "ExecutionContextLayer", MagicMock())+ monkeypatch.setattr(workflow_entry, "Engine", MagicMock(return_value=graph_engine))monkeypatch.setattr(workflow_entry, "InMemoryChannel", MagicMock(return_value=sentinel.command_channel))return workflow_entry.WorkflowEntry(- tenant_id="tenant-id",- app_id="app-id",- workflow_id="workflow-id",graph_config={"nodes": [], "edges": []},graph=sentinel.graph,- user_id="user-id",- user_from=UserFrom.ACCOUNT,- invoke_from=InvokeFrom.DEBUGGER,call_depth=0,- variable_pool=sentinel.variable_pool,graph_runtime_state=graph_runtime_state,response_stream_filter=response_stream_filter,)@@ -70,82 +63,12 @@ class TestWorkflowEntryInit:with pytest.raises(ValueError, match="Max workflow call depth"):workflow_entry.WorkflowEntry(- tenant_id="tenant-id",- app_id="app-id",- workflow_id="workflow-id",graph_config={"nodes": [], "edges": []},graph=sentinel.graph,- user_id="user-id",- user_from=UserFrom.ACCOUNT,- invoke_from=InvokeFrom.DEBUGGER,call_depth=call_depth,- variable_pool=sentinel.variable_pool,graph_runtime_state=sentinel.graph_runtime_state,)- def test_applies_debug_and_observability_layers(self):- graph_engine = MagicMock()- graph_runtime_state = SimpleNamespace(_execution_context=None)- debug_layer = sentinel.debug_layer- execution_limits_layer = sentinel.execution_limits_layer- observability_layer = sentinel.observability_layer-- with (- config_overrides_context(DEBUG=True, ENABLE_OTEL=False),- patch.object(workflow_entry, "is_instrument_flag_enabled", return_value=True),- patch.object(workflow_entry, "capture_current_context", return_value=sentinel.execution_context),- patch.object(workflow_entry, "GraphEngine", return_value=graph_engine) as graph_engine_cls,- patch.object(workflow_entry, "GraphEngineConfig", return_value=sentinel.graph_engine_config),- patch.object(workflow_entry, "InMemoryChannel", return_value=sentinel.command_channel),- patch.object(workflow_entry, "DebugLoggingLayer", return_value=debug_layer) as debug_logging_layer,- patch.object(- workflow_entry,- "ExecutionLimitsLayer",- return_value=execution_limits_layer,- ) as execution_limits_layer_cls,- patch.object(workflow_entry, "ObservabilityLayer", return_value=observability_layer),- ):- entry = workflow_entry.WorkflowEntry(- tenant_id="tenant-id",- app_id="app-id",- workflow_id="workflow-id-123456",- graph_config={"nodes": [], "edges": []},- graph=sentinel.graph,- user_id="user-id",- user_from=UserFrom.ACCOUNT,- invoke_from=InvokeFrom.DEBUGGER,- call_depth=0,- variable_pool=sentinel.variable_pool,- graph_runtime_state=graph_runtime_state,- command_channel=None,- )-- assert entry.command_channel is sentinel.command_channel- graph_engine_cls.assert_called_once_with(- workflow_id="workflow-id-123456",- graph=sentinel.graph,- graph_runtime_state=graph_runtime_state,- command_channel=sentinel.command_channel,- config=sentinel.graph_engine_config,- )- assert graph_runtime_state._execution_context is sentinel.execution_context- debug_logging_layer.assert_called_once_with(- level="DEBUG",- include_inputs=True,- include_outputs=True,- include_process_data=False,- logger_name="GraphEngine.Debug.workflow",- )- execution_limits_layer_cls.assert_called_once_with(- max_steps=workflow_entry.dify_config.WORKFLOW_MAX_EXECUTION_STEPS,- max_time=workflow_entry.dify_config.WORKFLOW_MAX_EXECUTION_TIME,- )- assert graph_engine.layer.call_args_list == [- ((debug_layer,), {}),- ((execution_limits_layer,), {}),- ((observability_layer,), {}),- ]-def test_workflow_entry_stores_supplied_response_stream_filter(self, monkeypatch: pytest.MonkeyPatch) -> None:supplied_filter = ResponseStreamFilter()entry = _build_minimal_workflow_entry(monkeypatch, response_stream_filter=supplied_filter)@@ -173,7 +96,7 @@ class TestWorkflowEntryRun:with (patch.object(- workflow_entry.GraphEventFilterContext,+ workflow_entry.EngineEventFilterContext,"from_engine",return_value=sentinel.filter_context,) as from_engine,@@ -189,16 +112,16 @@ class TestWorkflowEntryRun:),patch.object(workflow_entry,- "filter_graph_events",+ "filter_engine_events",return_value=iter([sentinel.filtered_event]),- ) as filter_graph_events,+ ) as filter_engine_events,):events = list(workflow_entry.iter_dify_graph_engine_events(graph_engine))assert events == [sentinel.filtered_event]from_engine.assert_called_once_with(graph_engine)response_stream_filter_cls.assert_called_once_with()- filter_graph_events.assert_called_once_with(+ filter_engine_events.assert_called_once_with(graph_engine.run.return_value,context=sentinel.filter_context,filters=[sentinel.human_input_filter, sentinel.response_stream_filter],@@ -247,6 +170,7 @@ class TestWorkflowEntrySingleStepRun:user_id="user-id",user_inputs={},variable_pool=sentinel.variable_pool,+ execution_driver=None,)def test_preloads_constructor_variables_before_creating_memory_node(self):@@ -283,7 +207,7 @@ class TestWorkflowEntrySingleStepRun:patch.object(workflow_entry, "DifyGraphInitContext", return_value=sentinel.graph_init_context),patch.object(workflow_entry,- "GraphRuntimeState",+ "RuntimeState",return_value=SimpleNamespace(variable_pool=variable_pool),),patch.object(workflow_entry, "build_dify_run_context", return_value={"_dify": "context"}),@@ -311,6 +235,7 @@ class TestWorkflowEntrySingleStepRun:user_inputs={},variable_pool=variable_pool,variable_loader=variable_loader,+ execution_driver=None,)assert node.id == "node-id"@@ -339,7 +264,7 @@ class TestWorkflowEntrySingleStepRun:return_value=_build_typed_node_config(BuiltinNodeTypes.START),),patch.object(workflow_entry, "DifyGraphInitContext", return_value=sentinel.graph_init_context),- patch.object(workflow_entry, "GraphRuntimeState", return_value=sentinel.graph_runtime_state),+ patch.object(workflow_entry, "RuntimeState", return_value=sentinel.graph_runtime_state),patch.object(workflow_entry, "build_dify_run_context", return_value={"_dify": "context"}),patch.object(workflow_entry.time, "perf_counter", return_value=123.0),patch.object(workflow_entry, "resolve_workflow_node_class", return_value=FakeNode),@@ -363,6 +288,7 @@ class TestWorkflowEntrySingleStepRun:user_id="user-id",user_inputs={"question": "hello"},variable_pool=sentinel.variable_pool,+ execution_driver=None,)assert node.id == "node-id"@@ -406,7 +332,7 @@ class TestWorkflowEntrySingleStepRun:return_value=_build_typed_node_config(BuiltinNodeTypes.DATASOURCE),),patch.object(workflow_entry, "DifyGraphInitContext", return_value=sentinel.graph_init_context),- patch.object(workflow_entry, "GraphRuntimeState", return_value=sentinel.graph_runtime_state),+ patch.object(workflow_entry, "RuntimeState", return_value=sentinel.graph_runtime_state),patch.object(workflow_entry, "build_dify_run_context", return_value={"_dify": "context"}),patch.object(workflow_entry.time, "perf_counter", return_value=123.0),patch.object(workflow_entry, "resolve_workflow_node_class", return_value=FakeDatasourceNode),@@ -430,6 +356,7 @@ class TestWorkflowEntrySingleStepRun:user_id="user-id",user_inputs={"question": "hello"},variable_pool=sentinel.variable_pool,+ execution_driver=None,)assert node.id == "node-id"@@ -464,7 +391,7 @@ class TestWorkflowEntrySingleStepRun:return_value=_build_typed_node_config(BuiltinNodeTypes.START),),patch.object(workflow_entry, "DifyGraphInitContext", return_value=sentinel.graph_init_context),- patch.object(workflow_entry, "GraphRuntimeState", return_value=sentinel.graph_runtime_state),+ patch.object(workflow_entry, "RuntimeState", return_value=sentinel.graph_runtime_state),patch.object(workflow_entry, "build_dify_run_context", return_value={"_dify": "context"}),patch.object(workflow_entry.time, "perf_counter", return_value=123.0),patch.object(workflow_entry, "resolve_workflow_node_class", return_value=FakeNode),@@ -486,6 +413,7 @@ class TestWorkflowEntrySingleStepRun:user_id="user-id",user_inputs={},variable_pool=sentinel.variable_pool,+ execution_driver=None,)@@ -569,7 +497,7 @@ class TestWorkflowEntryHelpers:patch.object(workflow_entry, "DifyGraphInitContext", return_value=sentinel.graph_init_context) as graph_init_context_cls,- patch.object(workflow_entry, "GraphRuntimeState", return_value=sentinel.graph_runtime_state),+ patch.object(workflow_entry, "RuntimeState", return_value=sentinel.graph_runtime_state),patch.object(workflow_entry, "build_dify_run_context", return_value={"_dify": "context"}) as build_dify_run_context,@@ -619,7 +547,7 @@ class TestWorkflowEntryHelpers:)dify_node_factory_cls.assert_called_once_with(graph_init_context=sentinel.graph_init_context,- graph_runtime_state=sentinel.graph_runtime_state,+ runtime_state=sentinel.graph_runtime_state,)mapping_user_inputs_to_variable_pool.assert_called_once_with(variable_mapping={},@@ -656,7 +584,7 @@ class TestWorkflowEntryHelpers:patch.object(workflow_entry, "VariablePool", return_value=sentinel.variable_pool),patch.object(workflow_entry, "add_variables_to_pool"),patch.object(workflow_entry, "DifyGraphInitContext", return_value=sentinel.graph_init_context),- patch.object(workflow_entry, "GraphRuntimeState", return_value=sentinel.graph_runtime_state),+ patch.object(workflow_entry, "RuntimeState", return_value=sentinel.graph_runtime_state),patch.object(workflow_entry, "build_dify_run_context", return_value={"_dify": "context"}),patch.object(workflow_entry.time, "perf_counter", return_value=123.0),patch.object(@@ -750,8 +678,43 @@ class TestMappingUserInputsBranches:class TestWorkflowEntryNodeLayers:- def test_run_node_with_layers_reports_success(self):+ @pytest.mark.parametrize("completion", ["close", "throw"])+ def test_standalone_scope_restores_caller_during_streaming_and_cleanup(self, completion: str) -> None:observability_layer = MagicMock()+ observed_runtime = []++ def run_node():+ observed_runtime.append(peek_workflow_file_runtime())+ yield sentinel.event++ node = MagicMock()+ node.run.side_effect = run_node+ error = RuntimeError("consumer failed")+ with (+ patch.object(workflow_entry, "ObservabilityLayer", return_value=observability_layer),+ patch.object(workflow_entry, "create_dify_workflow_file_runtime", return_value=sentinel.node_runtime),+ use_workflow_file_runtime(sentinel.caller_runtime),+ ):+ events = workflow_entry.WorkflowEntry._run_node_with_layers(node)+ assert next(events) is sentinel.event+ assert peek_workflow_file_runtime() is sentinel.caller_runtime+ if completion == "throw":+ with pytest.raises(RuntimeError, match="consumer failed"):+ events.throw(error)+ else:+ events.close()+ assert peek_workflow_file_runtime() is sentinel.caller_runtime++ assert observed_runtime == [sentinel.node_runtime]+ observability_layer.on_node_run_end.assert_called_once_with(+ node, error if completion == "throw" else None, None+ )+ observability_layer.on_graph_end.assert_called_once_with(error if completion == "throw" else None)++ def test_run_node_with_layers_reports_success(self, mocker: MockerFixture):+ observability_layer = workflow_entry.ObservabilityLayer()+ for method in ("initialize", "on_graph_start", "on_node_run_start", "on_node_run_end", "on_graph_end"):+ mocker.spy(observability_layer, method)result_event = NodeRunSucceededEvent(id="execution-id",node_id="node-id",@@ -761,7 +724,7 @@ class TestWorkflowEntryNodeLayers:)class FakeNode:- graph_runtime_state = sentinel.graph_runtime_state+ runtime_state = sentinel.graph_runtime_statedef bind_execution_id(self, _execution_id):return None@@ -775,11 +738,11 @@ class TestWorkflowEntryNodeLayers:patch.object(workflow_entry, "InMemoryChannel", return_value=sentinel.command_channel),patch.object(workflow_entry,- "ReadOnlyGraphRuntimeStateWrapper",+ "ReadOnlyRuntimeStateWrapper",return_value=sentinel.read_only_runtime_state,) as runtime_state_wrapper,):- events = list(workflow_entry.WorkflowEntry._run_node_with_layers(node, tenant_id="tenant-id"))+ events = list(workflow_entry.WorkflowEntry._run_node_with_layers(node))assert events == [result_event]runtime_state_wrapper.assert_called_once_with(sentinel.graph_runtime_state)@@ -790,11 +753,13 @@ class TestWorkflowEntryNodeLayers:layer.on_node_run_end.assert_called_once_with(node, None, result_event)layer.on_graph_end.assert_called_once_with(None)- def test_run_node_with_layers_reports_errors(self):- observability_layer = MagicMock()+ def test_run_node_with_layers_reports_errors(self, mocker: MockerFixture):+ observability_layer = workflow_entry.ObservabilityLayer()+ for method in ("initialize", "on_graph_start", "on_node_run_start", "on_node_run_end", "on_graph_end"):+ mocker.spy(observability_layer, method)class FakeNode:- graph_runtime_state = sentinel.graph_runtime_state+ runtime_state = sentinel.graph_runtime_statedef bind_execution_id(self, _execution_id):return None@@ -808,12 +773,12 @@ class TestWorkflowEntryNodeLayers:patch.object(workflow_entry, "ObservabilityLayer", return_value=observability_layer),patch.object(workflow_entry,- "ReadOnlyGraphRuntimeStateWrapper",+ "ReadOnlyRuntimeStateWrapper",return_value=sentinel.read_only_runtime_state,),):with pytest.raises(RuntimeError, match="boom"):- list(workflow_entry.WorkflowEntry._run_node_with_layers(node, tenant_id="tenant-id"))+ list(workflow_entry.WorkflowEntry._run_node_with_layers(node))for layer in (observability_layer,):assert layer.on_node_run_end.call_args.args[0] is nodediff --git a/api/tests/unit_tests/core/workflow/test_workflow_entry_redis_channel.py b/api/tests/unit_tests/core/workflow/test_workflow_entry_redis_channel.pyindex 5df526d1750..46a11f9bede 100644--- a/api/tests/unit_tests/core/workflow/test_workflow_entry_redis_channel.py+++ b/api/tests/unit_tests/core/workflow/test_workflow_entry_redis_channel.py@@ -2,10 +2,9 @@from unittest.mock import MagicMock, patch-from core.app.entities.app_invoke_entities import InvokeFrom, UserFromfrom core.workflow.workflow_entry import WorkflowEntry-from graphon.graph_engine.command_channels import RedisChannel-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.engine.command import RedisChannel+from graphon.runtime import RuntimeState, VariablePoolclass TestWorkflowEntryRedisChannel:@@ -17,31 +16,24 @@ class TestWorkflowEntryRedisChannel:mock_graph = MagicMock()mock_graph_config = {"nodes": [], "edges": []}variable_pool = VariablePool()- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0)+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0)# Create a mock Redis channelmock_redis_client = MagicMock()redis_channel = RedisChannel(mock_redis_client, "test:channel:key")- # Patch GraphEngine to verify it receives the Redis channel- with patch("core.workflow.workflow_entry.GraphEngine", autospec=True) as mock_graph_engine_type:+ # Patch Engine to verify it receives the Redis channel+ with patch("core.workflow.workflow_entry.Engine", autospec=True) as mock_graph_engine_type:# Create WorkflowEntry with Redis channelworkflow_entry = WorkflowEntry(- tenant_id="test-tenant",- app_id="test-app",- workflow_id="test-workflow",graph_config=mock_graph_config,graph=mock_graph,- user_id="test-user",- user_from=UserFrom.ACCOUNT,- invoke_from=InvokeFrom.DEBUGGER,call_depth=0,- variable_pool=variable_pool,graph_runtime_state=graph_runtime_state,command_channel=redis_channel, # Provide Redis channel)- # Verify GraphEngine was initialized with the Redis channel+ # Verify Engine was initialized with the Redis channelmock_graph_engine_type.assert_called_once()call_args = mock_graph_engine_type.call_args[1]assert call_args["command_channel"] == redis_channel@@ -53,26 +45,19 @@ class TestWorkflowEntryRedisChannel:mock_graph = MagicMock()mock_graph_config = {"nodes": [], "edges": []}variable_pool = VariablePool()- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0)+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0)- # Patch GraphEngine and InMemoryChannel+ # Patch Engine and InMemoryChannelwith (- patch("core.workflow.workflow_entry.GraphEngine", autospec=True) as mock_graph_engine_type,+ patch("core.workflow.workflow_entry.Engine", autospec=True) as mock_graph_engine_type,patch("core.workflow.workflow_entry.InMemoryChannel", autospec=True) as mock_in_memory_channel,):mock_inmemory_channel = mock_in_memory_channel.return_value# Create WorkflowEntry without providing a channelworkflow_entry = WorkflowEntry(- tenant_id="test-tenant",- app_id="test-app",- workflow_id="test-workflow",graph_config=mock_graph_config,graph=mock_graph,- user_id="test-user",- user_from=UserFrom.ACCOUNT,- invoke_from=InvokeFrom.DEBUGGER,call_depth=0,- variable_pool=variable_pool,graph_runtime_state=graph_runtime_state,command_channel=None, # No channel provided)@@ -80,7 +65,7 @@ class TestWorkflowEntryRedisChannel:# Verify InMemoryChannel was createdmock_in_memory_channel.assert_called_once()- # Verify GraphEngine was initialized with the InMemory channel+ # Verify Engine was initialized with the InMemory channelmock_graph_engine_type.assert_called_once()call_args = mock_graph_engine_type.call_args[1]assert call_args["command_channel"] == mock_inmemory_channel@@ -92,7 +77,7 @@ class TestWorkflowEntryRedisChannel:mock_graph = MagicMock()mock_graph_config = {"nodes": [], "edges": []}variable_pool = VariablePool()- graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=0)+ graph_runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=variable_pool, start_at=0)# Create a mock Redis channelmock_redis_client = MagicMock()@@ -102,24 +87,17 @@ class TestWorkflowEntryRedisChannel:mock_event1 = MagicMock()mock_event2 = MagicMock()- # Patch GraphEngine- with patch("core.workflow.workflow_entry.GraphEngine", autospec=True) as mock_graph_engine_type:+ # Patch Engine+ with patch("core.workflow.workflow_entry.Engine", autospec=True) as mock_graph_engine_type:mock_graph_engine = MagicMock()mock_graph_engine.run.return_value = iter([mock_event1, mock_event2])mock_graph_engine_type.return_value = mock_graph_engine# Create WorkflowEntry with Redis channelworkflow_entry = WorkflowEntry(- tenant_id="test-tenant",- app_id="test-app",- workflow_id="test-workflow",graph_config=mock_graph_config,graph=mock_graph,- user_id="test-user",- user_from=UserFrom.ACCOUNT,- invoke_from=InvokeFrom.DEBUGGER,call_depth=0,- variable_pool=variable_pool,graph_runtime_state=graph_runtime_state,command_channel=redis_channel,)diff --git a/api/tests/unit_tests/extensions/otel/decorators/handlers/test_workflow_app_runner_handler.py b/api/tests/unit_tests/extensions/otel/decorators/handlers/test_workflow_app_runner_handler.pyindex 3e312a665eb..0886568e002 100644--- a/api/tests/unit_tests/extensions/otel/decorators/handlers/test_workflow_app_runner_handler.py+++ b/api/tests/unit_tests/extensions/otel/decorators/handlers/test_workflow_app_runner_handler.py@@ -6,9 +6,36 @@ Test objectives:2. Verify span attribute mapping correctness"""+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import StatusCode+from sqlalchemy import Engine+from extensions.otel.decorators.handlers.workflow_app_runner_handler import WorkflowAppRunnerHandlerfrom extensions.otel.semconv import DifySpanAttributes, GenAIAttributes+from services.workflow_run_agg import WorkflowRunAggfrom tests.unit_tests.config_override import config_overrides_context+from tests.unit_tests.services.workflow.test_workflow_run_agg import make_workflow_runner+++def test_service_driver_emits_workflow_span(sqlite_engine: Engine) -> None:+ runner, runs, nodes = make_workflow_runner(sqlite_engine)+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ try:+ WorkflowAppRunnerHandler().wrapper(provider.get_tracer(__name__), WorkflowRunAgg.run, runner, None, runs, nodes)+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ attributes = spans[0].attributes+ assert attributes is not None+ assert attributes[DifySpanAttributes.WORKFLOW_ID] == runner.application_generate_entity.app_config.workflow_id+ assert attributes[DifySpanAttributes.APP_ID] == runner.application_generate_entity.app_config.app_id+ assert attributes[GenAIAttributes.USER_ID] == runner.application_generate_entity.user_id+ finally:+ provider.shutdown()class TestWorkflowAppRunnerHandler:@@ -59,7 +86,7 @@ class TestWorkflowAppRunnerHandler:mock_workflow_runner.application_generate_entity.app_config.tenant_id = test_tenant_idmock_workflow_runner.application_generate_entity.app_config.workflow_id = test_workflow_id- def runner_run(self):+ def runner_run(runner):return "result"handler.wrapper(tracer, runner_run, mock_workflow_runner)diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.pyindex 13715853de2..784fa01c36f 100644--- a/api/tests/unit_tests/extensions/test_ext_application_services.py+++ b/api/tests/unit_tests/extensions/test_ext_application_services.py@@ -987,7 +987,7 @@ def test_build_application_services_uses_supplied_redis_for_both_workflow_stop_sassert redis.values["generate_task_stopped:workflow-task"] == b"1"assert redis.expirations["generate_task_stopped:workflow-task"] == 600assert [json.loads(command) for command in redis.commands["workflow:workflow-task:commands"]] == [- {"command_type": "abort", "payload": None, "reason": "User requested stop"}+ {"command_type": "abort", "reason": "User requested stop"}]assert redis.expirations["workflow:workflow-task:commands"] == 3600diff --git a/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py b/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.pyindex ce774e4d160..88ca0a95cd5 100644--- a/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py+++ b/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py@@ -6,7 +6,7 @@ import pytestfrom core.app.apps.workflow.active_workflow_tasks import reset_active_workflow_tasksfrom core.app.apps.workflow.command_channels import CelerySignalCommandChannelfrom extensions import workflow_warm_shutdown-from graphon.graph_engine.entities.commands import AbortCommand+from graphon.engine.command import AbortCommand@pytest.fixture(autouse=True)diff --git a/api/tests/unit_tests/libs/test_flask_utils.py b/api/tests/unit_tests/libs/test_flask_utils.pyindex 9cab0db24cb..a10a242d1f6 100644--- a/api/tests/unit_tests/libs/test_flask_utils.py+++ b/api/tests/unit_tests/libs/test_flask_utils.py@@ -1,8 +1,9 @@import contextvarsimport threading+from contextlib import nullcontextimport pytest-from flask import Flask+from flask import Flask, current_appfrom flask_login import LoginManager, UserMixin, current_user, login_userfrom libs.flask_utils import preserve_flask_contexts@@ -121,3 +122,38 @@ def test_current_user_accessible_with_preserve_flask_contexts(login_app: Flask,assert result["error"] is Noneassert result["user_accessible"] is Trueassert result["user_id"] == "test_user"++[email protected]("caller_value", [None, "caller"])[email protected]("failure_stage", [None, "body", "teardown"])+def test_preserve_flask_contexts_restores_bindings_after_teardown(caller_value, failure_stage):+ flask_app = Flask(__name__)+ caller_app = current_app._get_current_object()+ test_var = contextvars.ContextVar("preserved_var")+ captured = contextvars.copy_context()+ captured.run(test_var.set, "captured")+ token = test_var.set(caller_value) if caller_value is not None else None+ teardown_values = []++ @flask_app.teardown_appcontext+ def teardown(_error):+ teardown_values.append(test_var.get())+ if failure_stage == "teardown":+ raise ValueError("teardown failed")++ try:+ with pytest.raises(ValueError, match=f"{failure_stage} failed") if failure_stage else nullcontext():+ with preserve_flask_contexts(flask_app, captured):+ assert current_app == flask_app+ assert test_var.get() == "captured"+ test_var.set("body")+ if failure_stage == "body":+ raise ValueError("body failed")++ assert teardown_values == ["body"]+ assert current_app == caller_app+ assert test_var.get(None) == caller_value+ assert (test_var in contextvars.copy_context()) == (caller_value is not None)+ finally:+ if token is not None:+ test_var.reset(token)diff --git a/api/tests/unit_tests/migrations/test_workflow_pause_task_id_migration.py b/api/tests/unit_tests/migrations/test_workflow_pause_task_id_migration.pynew file mode 100644index 00000000000..b9001f0c5c8--- /dev/null+++ b/api/tests/unit_tests/migrations/test_workflow_pause_task_id_migration.py@@ -0,0 +1,163 @@+from __future__ import annotations++import importlib.util+from io import StringIO+from pathlib import Path+from types import ModuleType++import pytest+import sqlalchemy as sa+from alembic import command+from alembic.config import Config+from alembic.migration import MigrationContext+from alembic.operations import Operations+from alembic.script import ScriptDirectory+from flask import Flask+from flask_migrate import Migrate+from flask_sqlalchemy import SQLAlchemy++_MIGRATIONS_DIR = Path(__file__).resolve().parents[3] / "migrations"+_MIGRATION_PATH = _MIGRATIONS_DIR / "versions/2026_10_10_1200-c6f4a90d8b31_add_workflow_pause_task_id.py"+++def _load_migration_module(path: Path) -> ModuleType:+ spec = importlib.util.spec_from_file_location(path.stem, path)+ if spec is None or spec.loader is None:+ raise RuntimeError("failed to load migration module")+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def test_repository_migration_graph_has_one_head() -> None:+ config = Config()+ config.set_main_option("script_location", str(_MIGRATIONS_DIR))+ assert len(ScriptDirectory.from_config(config).get_heads()) == 1++[email protected]("dialect_name", ["postgresql", "mysql"])+def test_index_uses_concurrent_ddl_outside_postgresql_transactions(dialect_name: str) -> None:+ migration = _load_migration_module(_MIGRATION_PATH)+ output = StringIO()+ context = MigrationContext.configure(+ dialect_name=dialect_name,+ opts={"as_sql": True, "output_buffer": output},+ )+ migration.__dict__["op"] = Operations(context)+ migration.upgrade()+ migration.downgrade()++ statements = [statement.strip() for statement in output.getvalue().split(";") if statement.strip()]+ assert not any(statement.startswith("UPDATE") for statement in statements)+ assert not any(+ "NOT NULL" in statement or "DEFAULT" in statement for statement in statements if "ADD COLUMN" in statement+ )+ if dialect_name == "postgresql":+ create = (+ "CREATE INDEX CONCURRENTLY workflow_run_task_id_idx ON workflow_runs (task_id) WHERE task_id IS NOT NULL"+ )+ drop = "DROP INDEX CONCURRENTLY IF EXISTS workflow_run_task_id_idx"+ index = statements.index(create)+ assert statements[index - 2 : index + 2] == ["COMMIT", drop, create, "BEGIN"]+ assert statements[index + 2 : index + 5] == ["COMMIT", drop, "BEGIN"]+ else:+ assert "CREATE INDEX workflow_run_task_id_idx ON workflow_runs (task_id)" in statements+ assert "DROP INDEX workflow_run_task_id_idx ON workflow_runs" in statements+ assert "CONCURRENTLY" not in output.getvalue()+ assert "IF NOT EXISTS" not in output.getvalue()+++def test_alembic_commits_retryable_columns_before_index_and_stamps_revision_afterward(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ output = StringIO()+ config = Config(str(_MIGRATIONS_DIR / "alembic.ini"), output_buffer=output)+ config.set_main_option("script_location", str(_MIGRATIONS_DIR))+ app = Flask(__name__)+ app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql://"+ database = SQLAlchemy(app)+ Migrate(app, database)+ monkeypatch.setattr("logging.config.fileConfig", lambda _filename: None)++ with app.app_context():+ command.upgrade(config, "f1d3c5b7a902:c6f4a90d8b31", sql=True)+ database.engine.dispose()++ sql = output.getvalue()+ add_task_id = sql.index("ADD COLUMN IF NOT EXISTS task_id")+ add_stop_requested_at = sql.index("ADD COLUMN IF NOT EXISTS stop_requested_at")+ drop_index = sql.index("DROP INDEX CONCURRENTLY IF EXISTS workflow_run_task_id_idx")+ create_index = sql.index("CREATE INDEX CONCURRENTLY workflow_run_task_id_idx")+ stamp_revision = sql.index("UPDATE alembic_version SET version_num='c6f4a90d8b31'")+ assert add_task_id < add_stop_requested_at < sql.index("COMMIT;") < drop_index < create_index < stamp_revision+ assert sql.count("UPDATE alembic_version") == 1++[email protected](("existing_column_count", "index_exists"), [(0, False), (1, False), (2, False), (2, True)])+def test_sqlite_upgrade_keeps_historical_rows_nullable_and_downgrade_preserves_rows(+ existing_column_count: int, index_exists: bool+) -> None:+ engine = sa.create_engine("sqlite:///:memory:")+ metadata = sa.MetaData()+ runs = sa.Table(+ "workflow_runs",+ metadata,+ sa.Column("id", sa.String(36), primary_key=True),+ *[sa.Column("task_id", sa.String(36)), sa.Column("stop_requested_at", sa.DateTime())][:existing_column_count],+ )+ if index_exists:+ sa.Index("workflow_run_task_id_idx", runs.c.task_id)+ metadata.create_all(engine)+ migration = _load_migration_module(_MIGRATION_PATH)++ with engine.begin() as connection:+ connection.execute(runs.insert().values(id="historical-run"))+ if existing_column_count:+ connection.execute(runs.update().values(task_id="historical-task"))+ migration.__dict__["op"] = Operations(MigrationContext.configure(connection))+ migration.upgrade()++ columns = {column["name"]: column for column in sa.inspect(connection).get_columns("workflow_runs")}+ assert columns["task_id"]["nullable"] is True+ assert columns["stop_requested_at"]["nullable"] is True+ assert columns["task_id"]["default"] is None+ assert columns["stop_requested_at"]["default"] is None+ assert connection.execute(sa.text("SELECT id, task_id, stop_requested_at FROM workflow_runs")).one() == (+ "historical-run",+ "historical-task" if existing_column_count else None,+ None,+ )+ assert sa.inspect(connection).get_indexes("workflow_runs")[0]["name"] == "workflow_run_task_id_idx"++ migration.downgrade()++ assert [column["name"] for column in sa.inspect(connection).get_columns("workflow_runs")] == ["id"]+ assert connection.scalar(sa.text("SELECT id FROM workflow_runs")) == "historical-run"+ assert sa.inspect(connection).get_indexes("workflow_runs") == []++ engine.dispose()++[email protected]("remaining_column_count", [2, 1, 0])+def test_sqlite_downgrade_resumes_after_index_or_columns_were_dropped(remaining_column_count: int) -> None:+ engine = sa.create_engine("sqlite:///:memory:")+ metadata = sa.MetaData()+ runs = sa.Table(+ "workflow_runs",+ metadata,+ sa.Column("id", sa.String(36), primary_key=True),+ *[sa.Column("task_id", sa.String(36)), sa.Column("stop_requested_at", sa.DateTime())][:remaining_column_count],+ )+ metadata.create_all(engine)+ migration = _load_migration_module(_MIGRATION_PATH)++ with engine.begin() as connection:+ connection.execute(runs.insert().values(id="historical-run"))+ migration.__dict__["op"] = Operations(MigrationContext.configure(connection))+ migration.downgrade()++ assert [column["name"] for column in sa.inspect(connection).get_columns("workflow_runs")] == ["id"]+ assert connection.scalar(sa.text("SELECT id FROM workflow_runs")) == "historical-run"+ assert sa.inspect(connection).get_indexes("workflow_runs") == []++ engine.dispose()diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.tomlindex 6532914ff23..761a3889586 100644--- a/api/tests/unit_tests/pyrefly.toml+++ b/api/tests/unit_tests/pyrefly.toml@@ -228,7 +228,6 @@ project-excludes = ["core/app/features/test_hosting_moderation.py","core/app/layers/test_conversation_variable_persist_layer.py","core/app/layers/test_pause_state_persist_layer.py",- "core/app/layers/test_suspend_layer.py","core/app/layers/test_timeslice_layer.py","core/app/layers/test_trigger_post_layer.py","core/app/task_pipeline/test_based_generate_task_pipeline.py",diff --git a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.pyindex 6edbda75e83..8ba971a82cd 100644--- a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py+++ b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py@@ -10,19 +10,21 @@ import pytestfrom sqlalchemy.orm import Session, sessionmakerfrom core.workflow.nodes.human_input.entities import FormDefinition, ParagraphInputConfig, UserActionConfig-from core.workflow.nodes.human_input.enums import FormInputType+from core.workflow.nodes.human_input.enums import FormInputType, HumanInputFormStatusfrom core.workflow.nodes.human_input.pause_reason import HumanInputRequiredfrom graphon.entities.pause_reason import HitlRequired, PauseReasonTypefrom graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom models import Message-from models.enums import ConversationFromSource, CreatorUserRole, WorkflowRunTriggeredFrom+from models.enums import ConversationFromSource, CreatorUserRole, MessageStatus, WorkflowRunTriggeredFromfrom models.human_input import HumanInputForm, HumanInputFormRecipient, RecipientTypefrom models.workflow import WorkflowPause, WorkflowPauseReason, WorkflowRunfrom repositories.api_workflow_run_repository import AppWorkflowRunStatsfrom repositories.sqlalchemy_api_workflow_run_repository import (DifyAPISQLAlchemyWorkflowRunRepository,WorkflowRunMessageRef,+ WorkflowRunNotPausedError,WorkflowRunPauseRecord,+ WorkflowTaskOwnerMismatchError,_build_human_input_required_reason,_PrivateWorkflowPauseEntity,)@@ -392,3 +394,103 @@ def test_app_run_stats_scope_runs_by_app_tenant_source_and_window("a-2": AppWorkflowRunStats(status_counts={}, last_created_at=datetime(2026, 1, 1)),}assert repository.get_stats_by_app_ids({}, sources, since) == {}+++ ("tenant_id", "app_id", "owner", "stopped"),+ [+ ("tenant-1", "app-1", None, True),+ ("tenant-1", "app-1", (CreatorUserRole.ACCOUNT, "account-1"), True),+ ("tenant-2", "app-1", None, False),+ ("tenant-1", "app-2", None, False),+ ],+)+def test_stop_paused_task_is_scoped_and_invalidates_pause_and_forms(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ tenant_id: str,+ app_id: str,+ owner: tuple[CreatorUserRole, str] | None,+ stopped: bool,+) -> None:+ run = _workflow_run(run_id="run-1", tenant_id="tenant-1", status=WorkflowExecutionStatus.PAUSED)+ run.task_id = "task-1"+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="state")+ form = _build_form_model()+ message = _message(message_id="message-1", app_id="app-1", workflow_run_id=run.id, conversation_id="conv-1")+ foreign_message = _message(message_id="message-2", app_id="app-2", workflow_run_id=run.id, conversation_id="conv-2")+ message.status = foreign_message.status = MessageStatus.PAUSED+ sqlite_session.add_all([run, pause, form, message, foreign_message])+ sqlite_session.commit()+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ prior_pause = repository.get_workflow_pause(run.id)+ assert prior_pause is not None+ with patch("repositories.sqlalchemy_api_workflow_run_repository.storage.delete") as delete_state:+ result = repository.stop_paused_workflow_task(tenant_id=tenant_id, app_id=app_id, task_id="task-1", owner=owner)+ sqlite_session.expire_all()+ assert (result is not None) is stopped+ if stopped:+ assert run.status == WorkflowExecutionStatus.STOPPED+ assert run.finished_at is not None+ assert form.status == HumanInputFormStatus.EXPIRED+ assert message.status == MessageStatus.NORMAL+ assert repository.get_workflow_pause(run.id) is None+ delete_state.assert_called_once_with("state")+ cancellation_resets: list[None] = []+ with pytest.raises(WorkflowRunNotPausedError):+ repository.resume_workflow_pause(+ run.id, prior_pause, before_resume=lambda: cancellation_resets.append(None)+ )+ assert cancellation_resets == []+ retried = repository.stop_paused_workflow_task(+ tenant_id=tenant_id, app_id=app_id, task_id="task-1", owner=owner+ )+ assert retried is not None+ assert retried.status == WorkflowExecutionStatus.STOPPED+ else:+ assert run.status == WorkflowExecutionStatus.PAUSED+ assert form.status == HumanInputFormStatus.WAITING+ assert message.status == MessageStatus.PAUSED+ delete_state.assert_not_called()+ assert foreign_message.status == MessageStatus.PAUSED++[email protected]("owner", [(CreatorUserRole.ACCOUNT, "other-user"), (CreatorUserRole.END_USER, "account-1")])+def test_paused_task_cannot_be_stopped_by_another_user_or_role(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ owner: tuple[CreatorUserRole, str],+) -> None:+ run = _workflow_run(run_id="run-1", tenant_id="tenant-1", status=WorkflowExecutionStatus.PAUSED)+ run.task_id = "task-1"+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="state")+ sqlite_session.add_all([run, pause])+ sqlite_session.commit()+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ with pytest.raises(WorkflowTaskOwnerMismatchError):+ repository.stop_paused_workflow_task(tenant_id="tenant-1", app_id="app-1", task_id="task-1", owner=owner)+ sqlite_session.expire_all()+ assert run.status == WorkflowExecutionStatus.PAUSED+++def test_stop_after_resume_leaves_running_task_to_the_engine(+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session]+) -> None:+ run = _workflow_run(run_id="run-1", tenant_id="tenant-1", status=WorkflowExecutionStatus.PAUSED)+ run.task_id = "task-1"+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="state")+ sqlite_session.add_all([run, pause])+ sqlite_session.commit()+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ entity = repository.get_workflow_pause(run.id)+ assert entity is not None+ cancellation_resets: list[None] = []+ repository.resume_workflow_pause(run.id, entity, before_resume=lambda: cancellation_resets.append(None))+ with pytest.raises(WorkflowRunNotPausedError):+ repository.resume_workflow_pause(run.id, entity, before_resume=lambda: cancellation_resets.append(None))+ assert cancellation_resets == [None]+ admitted = repository.stop_paused_workflow_task(tenant_id="tenant-1", app_id="app-1", task_id="task-1", owner=None)+ assert admitted is not None+ assert admitted.status == WorkflowExecutionStatus.RUNNING+ sqlite_session.expire_all()+ assert run.status == WorkflowExecutionStatus.RUNNINGdiff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.pyindex 53ae094ceaa..f03c9aabc41 100644--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_service.py@@ -15,14 +15,17 @@ from sqlalchemy.orm import Session, sessionmakerfrom core.app.apps.pipeline.pipeline_generator import PipelineGeneratorfrom core.app.entities.app_invoke_entities import InvokeFromfrom core.rag.index_processor.constant.index_type import IndexStructureType+from graphon.engine_events import NodeRunFailedEventfrom graphon.enums import (BuiltinNodeTypes,ErrorStrategy,WorkflowNodeExecutionMetadataKey,WorkflowNodeExecutionStatus,)-from graphon.graph_events import NodeRunFailedEventfrom graphon.node_events.base import NodeRunResult+from graphon.nodes.start.entities import StartNodeData+from graphon.nodes.start.start_node import StartNode+from graphon.runtime import InitParams, RuntimeState, VariablePoolfrom models import Account, Tenantfrom models.dataset import (Dataset,@@ -206,26 +209,29 @@ def _persist(session: Session, *entities: object) -> None:session.commit()-def _make_failed_published_node_run() -> tuple[SimpleNamespace, NodeRunFailedEvent]:- class FakeVariablePool:- def __init__(self) -> None:- self._values = {- ("sys", "invoke_from"): SimpleNamespace(value=InvokeFrom.PUBLISHED_PIPELINE),- ("sys", "app_id"): SimpleNamespace(value="pipeline-1"),- ("sys", "dataset_id"): SimpleNamespace(value="dataset-1"),- ("sys", "document_id"): SimpleNamespace(value="doc-1"),- }+def _make_start_node(data: StartNodeData | None = None, variable_pool: VariablePool | None = None) -> StartNode:+ return StartNode(+ node_id="node-1",+ data=data if data is not None else StartNodeData(title="Start"),+ init_params=InitParams(workflow_id="wf-1", graph_config={}, run_context={}, call_depth=0),+ runtime_state=RuntimeState(+ workflow_id="wf-1",+ variable_pool=variable_pool if variable_pool is not None else VariablePool(),+ start_at=time.perf_counter(),+ ),+ )- def get(self, path: list[str]) -> SimpleNamespace | None:- return self._values.get(tuple(path))- node_instance = SimpleNamespace(- workflow_id="wf-1",- node_type="start",- title="Start",- graph_runtime_state=SimpleNamespace(variable_pool=FakeVariablePool()),- error_strategy=None,- )+def _make_failed_published_node_run() -> tuple[StartNode, NodeRunFailedEvent]:+ variable_pool = VariablePool()+ for name, value in {+ "invoke_from": InvokeFrom.PUBLISHED_PIPELINE,+ "app_id": "pipeline-1",+ "dataset_id": "dataset-1",+ "document_id": "doc-1",+ }.items():+ variable_pool.add(["sys", name], value)+ node_instance = _make_start_node(variable_pool=variable_pool)run_result = NodeRunResult(status=WorkflowNodeExecutionStatus.FAILED,error="boom",@@ -829,7 +835,7 @@ def test_run_datasource_node_preview_online_document(def test_handle_node_run_result_success(mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext) -> None:- from graphon.graph_events import NodeRunSucceededEvent+ from graphon.engine_events import NodeRunSucceededEventfrom graphon.node_events.base import NodeRunResult# 1. Setup mock node and result@@ -1426,16 +1432,15 @@ def test_handle_node_run_result_default_value_strategy() -> None:from datetime import datetime- from graphon.graph_events import NodeRunFailedEvent+ from graphon.engine_events import NodeRunFailedEventfrom graphon.node_events.base import NodeRunResult- node_instance = SimpleNamespace(- workflow_id="wf-1",- node_type=BuiltinNodeTypes.START,- title="Start",- error_strategy=ErrorStrategy.DEFAULT_VALUE,- default_value_dict={"fallback": "ok"},- graph_runtime_state=SimpleNamespace(variable_pool=mocker.Mock()),+ node_instance = _make_start_node(+ StartNodeData(+ title="Start",+ error_strategy=ErrorStrategy.DEFAULT_VALUE,+ default_value=[{"key": "fallback", "value": "ok", "type": "string"}],+ ))failed_result = NodeRunResult(@@ -1846,13 +1851,7 @@ def test_get_datasource_plugins_raises_when_workflow_not_found(def test_handle_node_run_result_raises_when_no_terminal_event(mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext) -> None:- node_instance = SimpleNamespace(- workflow_id="wf-1",- node_type="start",- title="Start",- graph_runtime_state=SimpleNamespace(variable_pool=SimpleNamespace(get=lambda _: None)),- error_strategy=None,- )+ node_instance = _make_start_node()def _event_generator():yield object()diff --git a/api/tests/unit_tests/services/test_app_generate_service.py b/api/tests/unit_tests/services/test_app_generate_service.pyindex 55fa8da4f60..1cc95f8ddda 100644--- a/api/tests/unit_tests/services/test_app_generate_service.py+++ b/api/tests/unit_tests/services/test_app_generate_service.py@@ -10,7 +10,6 @@ Covers:- generate_single_loop (ADVANCED_CHAT / WORKFLOW / invalid mode)- generate_more_like_this- _get_workflow (debugger / non-debugger / specific id / invalid format / not found)- - get_response_generator (ended / non-ended workflow run)"""import json@@ -27,11 +26,10 @@ from sqlalchemy.orm import Sessionimport services.app_generate_service as ags_modulefrom core.app.entities.app_invoke_entities import InvokeFromfrom enums import DeploymentEdition, QuotaType-from graphon.enums import WorkflowExecutionStatusfrom models.account import Account-from models.enums import AppStatus, CreatorUserRole+from models.enums import AppStatusfrom models.model import App, AppMode-from models.workflow import Workflow, WorkflowRun, WorkflowRunTriggeredFrom, WorkflowType+from models.workflow import Workflow, WorkflowTypefrom services.app_generate_service import AppGenerateServicefrom services.errors.app import (TriggerWorkflowServiceModeUnavailableError,@@ -130,22 +128,6 @@ def _make_workflow()-def _make_workflow_run(*, run_id: str, ended: bool) -> WorkflowRun:- run = WorkflowRun(- tenant_id="tenant-id",- app_id="app-id",- workflow_id="workflow-id",- type=WorkflowType.WORKFLOW,- triggered_from=WorkflowRunTriggeredFrom.APP_RUN,- version="published",- status=WorkflowExecutionStatus.SUCCEEDED if ended else WorkflowExecutionStatus.RUNNING,- created_by_role=CreatorUserRole.ACCOUNT,- created_by="user-id",- )- run.id = run_id- return run--@contextmanagerdef _noop_rate_limit_context(rate_limit, request_id):"""Drop-in replacement for rate_limit_context that doesn't touch Redis."""@@ -477,12 +459,9 @@ class TestGenerate(_RealSessionTest):return_value=MagicMock(workflow_run_id="wfr-1", model_dump_json=MagicMock(return_value="{}")),)delay_spy = mocker.patch("services.app_generate_service.workflow_based_app_execution_task.delay")- gen_instance = MagicMock()- gen_instance.retrieve_events.return_value = iter([])- gen_instance.convert_to_event_stream.side_effect = lambda x: x- mocker.patch(- "services.app_generate_service.AdvancedChatAppGenerator",- return_value=gen_instance,+ retrieve_spy = mocker.patch(+ "services.app_generate_service.AdvancedChatAppGenerator.retrieve_events",+ return_value=iter([]),)AppGenerateService.generate(@@ -494,10 +473,10 @@ class TestGenerate(_RealSessionTest):session=self.session,)# In streaming mode it should go through retrieve_events, not generate- gen_instance.retrieve_events.assert_called_once()+ retrieve_spy.assert_called_once()# Dispatch is gated on subscribe; simulate the SSE layer entering the# subscription, which is what actually invokes on_subscribe.- on_subscribe = gen_instance.retrieve_events.call_args.kwargs["on_subscribe"]+ on_subscribe = retrieve_spy.call_args.kwargs["on_subscribe"]on_subscribe()delay_spy.assert_called_once()@@ -1073,40 +1052,3 @@ class TestGenerateMoreLikeThis(_RealSessionTest):gen_spy.assert_called_once()assert gen_spy.call_args.kwargs["session"] is sessionassert gen_spy.call_args.kwargs["stream"] is True---# ----------------------------------------------------------------------------# get_response_generator-# ----------------------------------------------------------------------------class TestGetResponseGenerator:- def test_non_ended_workflow_run(self, mocker: MockerFixture):- app = _make_app(AppMode.ADVANCED_CHAT)- workflow_run = _make_workflow_run(run_id="run-1", ended=False)-- gen_instance = MagicMock()- gen_instance.retrieve_events.return_value = iter([{"event": "started"}])- gen_instance.convert_to_event_stream.side_effect = lambda x: x- mocker.patch(- "services.app_generate_service.AdvancedChatAppGenerator",- return_value=gen_instance,- )-- AppGenerateService.get_response_generator(app_model=app, workflow_run=workflow_run)- gen_instance.retrieve_events.assert_called_once()-- def test_ended_workflow_run_still_returns_generator(self, mocker: MockerFixture):- """Even when the run is ended, the current code still returns a generator (TODO branch)."""- app = _make_app(AppMode.WORKFLOW)- workflow_run = _make_workflow_run(run_id="run-2", ended=True)-- gen_instance = MagicMock()- gen_instance.retrieve_events.return_value = iter([])- gen_instance.convert_to_event_stream.side_effect = lambda x: x- mocker.patch(- "services.app_generate_service.AdvancedChatAppGenerator",- return_value=gen_instance,- )-- AppGenerateService.get_response_generator(app_model=app, workflow_run=workflow_run)- # current impl falls through the TODO and still creates a generator- gen_instance.retrieve_events.assert_called_once()diff --git a/api/tests/unit_tests/services/test_app_task_control_service.py b/api/tests/unit_tests/services/test_app_task_control_service.pyindex 40cc19ff98a..ead030da63b 100644--- a/api/tests/unit_tests/services/test_app_task_control_service.py+++ b/api/tests/unit_tests/services/test_app_task_control_service.py@@ -3,8 +3,10 @@from unittest.mock import MagicMock, patchimport pytest+from redis.client import Pipelinefrom extensions.ext_redis import RedisClientWrapper+from graphon.engine.command import AbortCommandfrom services.app_task_service import AppTaskControlService@@ -15,38 +17,38 @@ def test_workflow_stop_sets_flag_before_sending_command() -> None:redis, commands = redis_transportevents: list[str] = []+ queued: list[tuple[object, ...]] = []commands.side_effect = lambda *_, **__: events.append("flag")- failure = RuntimeError("command channel unavailable")- def send_stop(requested_task_id: str) -> None:- assert requested_task_id == task_id+ def execute_pipeline(pipeline: "Pipeline[bytes]") -> list[object]:events.append("command")+ queued.extend(args for args, _ in pipeline.command_stack)if command_fails:- raise failure+ raise RuntimeError("command channel unavailable")+ return []- with patch("services.app_task_service.GraphEngineManager") as manager:- manager.return_value.send_stop_command.side_effect = send_stop- service = AppTaskControlService(redis_client=redis)- if command_fails:- with pytest.raises(RuntimeError) as caught:- service.stop_workflow_task_no_user_check(task_id=task_id)- assert caught.value is failure- else:- service.stop_workflow_task_no_user_check(task_id=task_id)- manager.assert_called_once_with(redis)+ with patch.object(Pipeline, "execute", autospec=True, side_effect=execute_pipeline):+ AppTaskControlService(redis_client=redis).stop_workflow_task_no_user_check(task_id=task_id)if task_id:assert events == ["flag", "command"]commands.assert_called_once_with("SETEX", "generate_task_stopped:task-1", 600, 1)+ pushes = [args for args in queued if args[0] == "RPUSH"]+ assert len(pushes) == 1+ _, key, payload = pushes[0]+ assert key == "workflow:task-1:commands"+ assert isinstance(payload, str)+ assert AbortCommand.model_validate_json(payload) == AbortCommand(reason="User requested stop")else:- assert events == ["command"]+ assert events == []+ assert queued == []commands.assert_not_called()def test_flag_failure_does_not_send_command(redis_transport: tuple[RedisClientWrapper, MagicMock]) -> None:redis, commands = redis_transportcommands.side_effect = RuntimeError("flag store unavailable")- with patch("services.app_task_service.GraphEngineManager") as manager:+ with patch.object(Pipeline, "execute", autospec=True) as execute:with pytest.raises(RuntimeError, match="flag store unavailable"):AppTaskControlService(redis_client=redis).stop_workflow_task_no_user_check(task_id="task-1")- manager.assert_not_called()+ execute.assert_not_called()diff --git a/api/tests/unit_tests/services/test_app_task_service.py b/api/tests/unit_tests/services/test_app_task_service.pyindex 603dcc9509b..23fbc074b4d 100644--- a/api/tests/unit_tests/services/test_app_task_service.py+++ b/api/tests/unit_tests/services/test_app_task_service.py@@ -2,19 +2,25 @@ import jsonfrom collections.abc import Generatorfrom contextlib import contextmanagerfrom dataclasses import dataclass, field-from datetime import timedelta+from datetime import datetime, timedeltafrom functools import partialfrom typing import overrideimport pytestfrom redis.exceptions import ConnectionError as RedisConnectionError+from sqlalchemy.orm import Sessionimport core.app.apps.base_app_queue_manager as queue_moduleimport core.app.apps.execution_coordinator as coordinator_moduleimport services.app_task_service as task_modulefrom core.app.entities.app_invoke_entities import InvokeFrom+from core.workflow.nodes.human_input.enums import HumanInputFormStatusfrom extensions.ext_redis import RedisClientWrapper+from graphon.enums import WorkflowExecutionStatus+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.human_input import HumanInputFormfrom models.model import AppMode+from models.workflow import WorkflowPause, WorkflowRunfrom services.app_task_service import AppTaskControlService, AppTaskService_TASK_ID = "task-with-non-uuid-id"@@ -112,27 +118,22 @@ def _assert_stop_flag(redis: _StopRedis) -> None:def _assert_graph_command(redis: _StopRedis) -> None:assert set(redis.commands) == {_COMMAND_KEY}assert [json.loads(command) for command in redis.commands[_COMMAND_KEY]] == [- {"command_type": "abort", "payload": None, "reason": "User requested stop"}+ {"command_type": "abort", "reason": "User requested stop"}]assert redis.expirations[_COMMAND_KEY] == 3600assert redis.values[f"{_COMMAND_KEY}:pending"] == b"1"assert redis.expirations[f"{_COMMAND_KEY}:pending"] == 3600[email protected]("app_mode", list(AppMode))-def test_stop_task_sends_graph_command_only_for_workflow_modes(app_mode: AppMode) -> None:+def test_stop_task_sets_only_the_legacy_flag() -> None:redis = _StopRedis(values={_OWNER_KEY: f"end-user-{_USER_ID}".encode()})- AppTaskControlService(redis_client=redis).stop_task(_TASK_ID, InvokeFrom.WEB_APP, _USER_ID, app_mode)+ assert AppTaskControlService(redis_client=redis).stop_task(_TASK_ID, InvokeFrom.WEB_APP, _USER_ID) is Trueassert redis.reads == [_OWNER_KEY]_assert_stop_flag(redis)- if app_mode in (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW):- assert redis.operations == ["legacy_flag", "graph_command"]- _assert_graph_command(redis)- else:- assert redis.operations == ["legacy_flag"]- assert redis.commands == {}+ assert redis.operations == ["legacy_flag"]+ assert redis.commands == {}@pytest.mark.parametrize(@@ -142,32 +143,213 @@ def test_stop_task_sends_graph_command_only_for_workflow_modes(app_mode: AppMode(InvokeFrom.DEBUGGER, b"account-user-1", True),(InvokeFrom.WEB_APP, b"end-user-user-1", True),(InvokeFrom.SERVICE_API, b"end-user-user-1", True),+ (InvokeFrom.OPENAPI, b"end-user-user-1", True),(InvokeFrom.EXPLORE, b"end-user-user-1", False),(InvokeFrom.DEBUGGER, b"end-user-user-1", False),(InvokeFrom.WEB_APP, b"account-user-1", False),(InvokeFrom.SERVICE_API, b"account-user-1", False),+ (InvokeFrom.OPENAPI, b"account-user-1", False),(InvokeFrom.EXPLORE, b"account-another-user", False),(InvokeFrom.WEB_APP, b"end-user-another-user", False),+ (InvokeFrom.OPENAPI, b"end-user-another-user", False),(InvokeFrom.EXPLORE, None, False),(InvokeFrom.WEB_APP, None, False),+ (InvokeFrom.OPENAPI, None, False),],)-def test_task_ownership_controls_only_the_legacy_flag(+def test_task_ownership_controls_the_legacy_flag(invoke_from: InvokeFrom, owner: bytes | None, should_set_flag: bool) -> None:redis = _StopRedis(values={_OWNER_KEY: owner} if owner is not None else {})- AppTaskControlService(redis_client=redis).stop_task(_TASK_ID, invoke_from, _USER_ID, AppMode.WORKFLOW)+ admitted = AppTaskControlService(redis_client=redis).stop_task(_TASK_ID, invoke_from, _USER_ID)+ assert admitted is should_set_flagassert redis.reads == [_OWNER_KEY]if should_set_flag:_assert_stop_flag(redis)- assert redis.operations == ["legacy_flag", "graph_command"]+ assert redis.operations == ["legacy_flag"]else:assert _STOP_KEY not in redis.values- assert redis.operations == ["graph_command"]- # Preserve the existing behavior even when the legacy ownership check does not match.- _assert_graph_command(redis)+ assert redis.operations == []+ assert redis.commands == {}++[email protected]("app_mode", [AppMode.CHAT, AppMode.COMPLETION])+ ("caller_role", "cached_owner", "admitted"),+ [+ (CreatorUserRole.END_USER, b"end-user-user-1", True),+ (CreatorUserRole.END_USER, b"end-user-other", False),+ (CreatorUserRole.END_USER, b"account-user-1", False),+ (CreatorUserRole.END_USER, None, False),+ (CreatorUserRole.ACCOUNT, b"end-user-other", True),+ (CreatorUserRole.ACCOUNT, None, True),+ ],+)+def test_stop_app_task_applies_legacy_caller_ownership(+ app_mode: AppMode, caller_role: CreatorUserRole, cached_owner: bytes | None, admitted: bool+) -> None:+ redis = _StopRedis(values={_OWNER_KEY: cached_owner} if cached_owner is not None else {})++ result = AppTaskControlService(redis_client=redis).stop_app_task(+ tenant_id="tenant-1",+ app_id="app-1",+ task_id=_TASK_ID,+ app_mode=app_mode,+ caller_role=caller_role,+ caller_id=_USER_ID,+ )++ assert result is admitted+ assert redis.reads == ([_OWNER_KEY] if caller_role is CreatorUserRole.END_USER else [])+ assert redis.operations == (["legacy_flag"] if admitted else [])+ assert redis.commands == {}+ if admitted:+ _assert_stop_flag(redis)+ else:+ assert _STOP_KEY not in redis.values+++def app_task_workflow_run(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> WorkflowRun:+ run = WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ task_id="11111111-1111-1111-1111-111111111111",+ type="workflow",+ version="1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ status=WorkflowExecutionStatus.RUNNING,+ created_by_role=CreatorUserRole.END_USER,+ created_by=_USER_ID,+ )+ sqlite_session.add(run)+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_session.get_bind()))+ return run++[email protected]("app_mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT])+ ("caller_role", "run_owner_role", "run_owner_id"),+ [+ (CreatorUserRole.END_USER, CreatorUserRole.END_USER, _USER_ID),+ (CreatorUserRole.ACCOUNT, CreatorUserRole.END_USER, "other-user"),+ (CreatorUserRole.ACCOUNT, CreatorUserRole.ACCOUNT, "other-account"),+ ],+)+def test_stop_app_task_signals_admitted_running_workflow(+ sqlite_session: Session,+ app_task_workflow_run: WorkflowRun,+ app_mode: AppMode,+ caller_role: CreatorUserRole,+ run_owner_role: CreatorUserRole,+ run_owner_id: str,+) -> None:+ run = app_task_workflow_run+ run.created_by_role = run_owner_role+ run.created_by = run_owner_id+ sqlite_session.commit()+ task_id = run.task_id+ assert task_id is not None+ redis = _StopRedis(read_error=AssertionError("Durable workflow ownership must not depend on Redis"))++ admitted = AppTaskControlService(redis_client=redis).stop_app_task(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ task_id=task_id,+ app_mode=app_mode,+ caller_role=caller_role,+ caller_id=_USER_ID,+ )++ assert admitted is True+ sqlite_session.refresh(run)+ assert run.status == WorkflowExecutionStatus.RUNNING+ assert run.stop_requested_at is not None+ assert redis.reads == []+ assert redis.operations == ["legacy_flag", "graph_command"]+ assert redis.values[f"generate_task_stopped:{task_id}"] == b"1"+ assert [json.loads(command) for command in redis.commands[f"workflow:{task_id}:commands"]] == [+ {"command_type": "abort", "reason": "User requested stop"}+ ]++[email protected]("app_mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT])[email protected]("status", [WorkflowExecutionStatus.RUNNING, WorkflowExecutionStatus.PAUSED])+ ("caller_role", "mismatch"),+ [+ (CreatorUserRole.END_USER, "owner-id"),+ (CreatorUserRole.END_USER, "owner-role"),+ (CreatorUserRole.END_USER, "app"),+ (CreatorUserRole.END_USER, "tenant"),+ (CreatorUserRole.END_USER, "missing"),+ (CreatorUserRole.ACCOUNT, "app"),+ (CreatorUserRole.ACCOUNT, "tenant"),+ (CreatorUserRole.ACCOUNT, "missing"),+ ],+)+def test_stop_app_task_preserves_unadmitted_workflow_and_pause(+ sqlite_session: Session,+ app_task_workflow_run: WorkflowRun,+ app_mode: AppMode,+ status: WorkflowExecutionStatus,+ caller_role: CreatorUserRole,+ mismatch: str,+) -> None:+ run = app_task_workflow_run+ run.status = status+ if mismatch == "owner-id":+ run.created_by = "other-user"+ if mismatch == "owner-role":+ run.created_by_role = CreatorUserRole.ACCOUNT+ pause = None+ form = None+ if status == WorkflowExecutionStatus.PAUSED:+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="state")+ form = HumanInputForm(+ id="form-1",+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ workflow_run_id=run.id,+ node_id="review",+ form_definition="{}",+ rendered_content="Review",+ expiration_time=datetime(2026, 10, 20),+ )+ sqlite_session.add_all([pause, form])+ sqlite_session.commit()+ task_id = "22222222-2222-2222-2222-222222222222" if mismatch == "missing" else run.task_id+ assert task_id is not None+ redis = _StopRedis(read_error=AssertionError("Durable workflow ownership must not depend on Redis"))++ admitted = AppTaskControlService(redis_client=redis).stop_app_task(+ tenant_id="other-tenant" if mismatch == "tenant" else run.tenant_id,+ app_id="other-app" if mismatch == "app" else run.app_id,+ task_id=task_id,+ app_mode=app_mode,+ caller_role=caller_role,+ caller_id=_USER_ID,+ )++ assert admitted is (caller_role is CreatorUserRole.ACCOUNT)+ assert redis.reads == []+ assert redis.operations == []+ assert redis.values == {}+ assert redis.commands == {}+ sqlite_session.refresh(run)+ assert run.status == status+ assert run.stop_requested_at is None+ assert run.finished_at is None+ if pause is not None and form is not None:+ sqlite_session.refresh(pause)+ sqlite_session.refresh(form)+ assert pause.state_object_key == "state"+ assert form.status == HumanInputFormStatus.WAITING+ assert form.submitted_at is None@pytest.mark.parametrize("owner", [None, b"account-another-user"])@@ -204,7 +386,7 @@ def test_flag_write_failure_propagates_before_graph_command(unchecked: bool) ->stop = (partial(service.stop_workflow_task_no_user_check, task_id=_TASK_ID)if unchecked- else partial(service.stop_task, _TASK_ID, InvokeFrom.EXPLORE, _USER_ID, AppMode.WORKFLOW)+ else partial(service.stop_task, _TASK_ID, InvokeFrom.EXPLORE, _USER_ID))with pytest.raises(RedisConnectionError, match="stop flag write failed") as caught:@@ -216,22 +398,17 @@ def test_flag_write_failure_propagates_before_graph_command(unchecked: bool) ->assert redis.commands == {}[email protected]("unchecked", [False, True])-def test_graph_redis_failure_is_swallowed_after_legacy_flag(unchecked: bool, caplog: pytest.LogCaptureFixture) -> None:+def test_graph_redis_failure_is_swallowed_after_legacy_flag(caplog: pytest.LogCaptureFixture) -> None:redis = _StopRedis(values={_OWNER_KEY: b"account-user-1"}, command_error=RedisConnectionError("command write failed"))- service = AppTaskControlService(redis_client=redis)- if unchecked:- service.stop_workflow_task_no_user_check(task_id=_TASK_ID)- else:- service.stop_task(_TASK_ID, InvokeFrom.EXPLORE, _USER_ID, AppMode.WORKFLOW)+ AppTaskControlService(redis_client=redis).stop_workflow_task_no_user_check(task_id=_TASK_ID)_assert_stop_flag(redis)assert redis.operations == ["legacy_flag", "graph_command"]assert redis.commands == {}- assert "Failed to send graph engine command AbortCommand" in caplog.text+ assert "Failed to send Engine abort command" in caplog.textdef test_ownership_read_failure_propagates_without_either_stop_signal() -> None:@@ -239,7 +416,7 @@ def test_ownership_read_failure_propagates_without_either_stop_signal() -> None:redis = _StopRedis(read_error=error)with pytest.raises(RedisConnectionError, match="ownership read failed") as caught:- AppTaskControlService(redis_client=redis).stop_task(_TASK_ID, InvokeFrom.EXPLORE, _USER_ID, AppMode.WORKFLOW)+ AppTaskControlService(redis_client=redis).stop_task(_TASK_ID, InvokeFrom.EXPLORE, _USER_ID)assert caught.value is errorassert redis.reads == [_OWNER_KEY]@@ -254,7 +431,7 @@ def test_service_instances_keep_their_redis_dependencies_separate() -> None:first_service = AppTaskControlService(redis_client=first)second_service = AppTaskControlService(redis_client=second)- first_service.stop_task(_TASK_ID, InvokeFrom.EXPLORE, _USER_ID, AppMode.CHAT)+ first_service.stop_task(_TASK_ID, InvokeFrom.EXPLORE, _USER_ID)second_service.stop_workflow_task_no_user_check(task_id=_TASK_ID)assert first.reads == [_OWNER_KEY]@@ -274,9 +451,94 @@ def test_legacy_static_entry_point_passes_global_client_through_the_same_impleme# Only the legacy composition point gets a global client; lower-level globals remain traps.monkeypatch.setattr(task_module, "redis_client", redis)- AppTaskService.stop_task(_TASK_ID, InvokeFrom.SERVICE_API, _USER_ID, AppMode.ADVANCED_CHAT)+ AppTaskService.stop_task(+ _TASK_ID, InvokeFrom.SERVICE_API, _USER_ID, AppMode.CHAT, tenant_id="tenant-1", app_id="app-1"+ )assert redis.reads == [_OWNER_KEY]- assert redis.operations == ["legacy_flag", "graph_command"]+ assert redis.operations == ["legacy_flag"]_assert_stop_flag(redis)- _assert_graph_command(redis)++[email protected]("cached_owner", [None, b"end-user-other", b"account-user-1"])+def test_live_chatflow_stop_does_not_send_either_signal_for_another_owner(+ monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, cached_owner: bytes | None+) -> None:+ task_id = "11111111-1111-1111-1111-111111111111"+ redis = _StopRedis(values={_OWNER_KEY: cached_owner} if cached_owner else {})+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_session.get_bind()))+ admissions: list[dict[str, object]] = []+ repository_type = task_module.DifyAPISQLAlchemyWorkflowRunRepository+ original_stop = repository_type.stop_paused_workflow_task++ def stop(+ repository: task_module.DifyAPISQLAlchemyWorkflowRunRepository,+ *,+ tenant_id: str,+ app_id: str,+ task_id: str,+ owner: tuple[CreatorUserRole, str] | None,+ ) -> WorkflowRun | None:+ admissions.append({"tenant_id": tenant_id, "app_id": app_id, "task_id": task_id, "owner": owner})+ return original_stop(repository, tenant_id=tenant_id, app_id=app_id, task_id=task_id, owner=owner)++ monkeypatch.setattr(repository_type, "stop_paused_workflow_task", stop)+ monkeypatch.setattr(task_module, "redis_client", redis)+ AppTaskService.stop_task(+ task_id, InvokeFrom.SERVICE_API, _USER_ID, AppMode.ADVANCED_CHAT, tenant_id="tenant-1", app_id="app-1"+ )+ assert (+ admissions+ == [+ {+ "tenant_id": "tenant-1",+ "app_id": "app-1",+ "task_id": task_id,+ "owner": (CreatorUserRole.END_USER, _USER_ID),+ }+ ]+ * 2+ )+ assert redis.reads == []+ assert redis.operations == []+ assert redis.commands == {}+++def test_live_chatflow_stop_sends_signals_for_its_owner(+ monkeypatch: pytest.MonkeyPatch, sqlite_session: Session+) -> None:+ task_id = "11111111-1111-1111-1111-111111111111"+ redis = _StopRedis(values={_OWNER_KEY: b"end-user-user-1"})+ run = WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ task_id=task_id,+ type="workflow",+ version="1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ status=WorkflowExecutionStatus.RUNNING,+ created_by_role=CreatorUserRole.END_USER,+ created_by=_USER_ID,+ )+ sqlite_session.add(run)+ sqlite_session.commit()+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_session.get_bind()))+ monkeypatch.setattr(task_module, "redis_client", redis)+ AppTaskService.stop_task(+ task_id, InvokeFrom.SERVICE_API, _USER_ID, AppMode.ADVANCED_CHAT, tenant_id="tenant-1", app_id="app-1"+ )+ assert redis.values[f"generate_task_stopped:{task_id}"] == b"1"+ assert json.loads(redis.commands[f"workflow:{task_id}:commands"][0])["command_type"] == "abort"+++def test_invalid_workflow_task_id_is_rejected_before_database_lookup(monkeypatch: pytest.MonkeyPatch) -> None:+ def reject_lookup(*_args: object, **_kwargs: object) -> None:+ pytest.fail("Invalid task IDs must be rejected before database lookup")++ monkeypatch.setattr(task_module, "DifyAPISQLAlchemyWorkflowRunRepository", reject_lookup)+ with pytest.raises(ValueError, match="Invalid task ID"):+ AppTaskService.stop_workflow_task(+ tenant_id="tenant-1", app_id="app-1", task_id="not-a-uuid", app_mode=AppMode.WORKFLOW, owner=None+ )diff --git a/api/tests/unit_tests/services/test_human_input_service.py b/api/tests/unit_tests/services/test_human_input_service.pyindex 9af7f4f1216..a8aab964af5 100644--- a/api/tests/unit_tests/services/test_human_input_service.py+++ b/api/tests/unit_tests/services/test_human_input_service.py@@ -32,7 +32,7 @@ from core.workflow.nodes.human_input.entities import ()from core.workflow.nodes.human_input.enums import HumanInputFormKind, HumanInputFormStatus, ValueSourceTypefrom graphon.file import File, FileTransferMethod, FileType-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom libs.datetime_utils import naive_utc_nowfrom models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient, RecipientTypefrom models.model import App, AppMode@@ -287,7 +287,7 @@ def _build_resumption_context_state(*, options: list[str], workflow_run_id: str)call_depth=0,workflow_execution_id=workflow_run_id,)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)runtime_state.variable_pool.add(("start", "options"), options)context = WorkflowResumptionContext(generate_entity=_WorkflowGenerateEntityWrapper(entity=generate_entity),@@ -332,10 +332,33 @@ def test_resolve_form_inputs_uses_runtime_select_options(assert len(resolved_inputs) == 1resolved_input = resolved_inputs[0]assert isinstance(resolved_input, SelectInputConfig)+ assert resolved_input.option_source.type == ValueSourceType.CONSTANT+ assert resolved_input.option_source.selector == ()assert resolved_input.option_source.value == ["approve", "reject"]workflow_run_repo.get_workflow_pause.assert_called_once_with(record.workflow_run_id)+def test_constant_select_rejects_unlisted_submission(+ sample_form_record: HumanInputFormRecord,+ unbound_session_factory: sessionmaker[Session],+) -> None:+ configured_input = SelectInputConfig(+ output_variable_name="decision",+ option_source=StringListSource(+ type=ValueSourceType.CONSTANT,+ value=["approve", "reject"],+ ),+ )+ record = dataclasses.replace(+ sample_form_record,+ definition=sample_form_record.definition.model_copy(update={"inputs": [configured_input]}),+ )+ service = HumanInputService(unbound_session_factory)++ with pytest.raises(InvalidFormDataError, match="Invalid value for select input 'decision': forged"):+ service._validate_submission(Form(record), selected_action_id="submit", form_data={"decision": "forged"})++def test_submit_form_by_token_calls_repository_and_enqueue(form_repository: Callable[[HumanInputFormRecord | None], HumanInputFormSubmissionRepository],sample_form_record: HumanInputFormRecord,@@ -470,6 +493,10 @@ def test_submit_form_by_token_missing_inputs(@pytest.mark.parametrize(("input_definition", "submitted_value", "expected_message"),[+ *[+ ({"type": "paragraph", "output_variable_name": "note"}, value, "note")+ for value in (12, True, None, ["text"], {"text": "text"})+ ],({"type": "select",@@ -515,7 +542,7 @@ def test_submit_form_by_token_missing_inputs(),],)-def test_validate_human_input_submission_rejects_invalid_select_and_file_payloads(+def test_validate_human_input_submission_rejects_invalid_input_payloads(form_repository: Callable[[HumanInputFormRecord | None], HumanInputFormSubmissionRepository],sample_form_record: HumanInputFormRecord,unbound_session_factory: sessionmaker[Session],diff --git a/api/tests/unit_tests/services/test_legacy_workflow_task_recovery.py b/api/tests/unit_tests/services/test_legacy_workflow_task_recovery.pynew file mode 100644index 00000000000..e7943f0c8a0--- /dev/null+++ b/api/tests/unit_tests/services/test_legacy_workflow_task_recovery.py@@ -0,0 +1,426 @@+"""Legacy pauses recover task IDs on demand without an offline backfill."""++import json+from collections.abc import Callable+from datetime import datetime+from uuid import UUID, uuid4++import pytest+from redis import Redis+from sqlalchemy import event, select+from sqlalchemy.engine import Connection, Engine+from sqlalchemy.orm import Session, sessionmaker++import services.app_task_service as task_module+from core.app.app_config.entities import WorkflowUIBasedAppConfig+from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom, WorkflowAppGenerateEntity+from core.app.layers.pause_state_persist_layer import (+ WorkflowResumptionContext,+ _AdvancedChatAppGenerateEntityWrapper,+ _WorkflowGenerateEntityWrapper,+)+from graphon.enums import WorkflowExecutionStatus+from libs.broadcast_channel.redis.pubsub_channel import Topic+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.model import AppMode+from models.workflow import WorkflowPause, WorkflowRun, WorkflowType+from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository+from services.app_task_service import AppTaskControlService, AppTaskService+from tests.unit_tests.services.test_app_task_service import _StopRedis+++def pause_states(monkeypatch: pytest.MonkeyPatch) -> dict[str, bytes]:+ states: dict[str, bytes] = {}++ def load(key: str) -> bytes:+ try:+ return states[key]+ except KeyError as error:+ raise FileNotFoundError(key) from error++ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.load", load)+ return states+++def recovery_redis() -> _StopRedis:+ return _StopRedis(read_error=AssertionError("Recovery must use persisted ownership"))+++def recovery_service(+ sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch, recovery_redis: _StopRedis+) -> AppTaskControlService:+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_engine))+ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.delete", lambda _: None)+ monkeypatch.setattr(AppTaskService, "_persist_stopped_history", lambda _: None)+ monkeypatch.setattr(+ "core.workflow.nodes.agent_v2.session_store.WorkflowAgentWorkspaceStore.retire_workflow_run",+ lambda *_args, **_kwargs: [],+ )+ monkeypatch.setattr("tasks.collect_agent_resources_task.enqueue_agent_resource_collection", lambda **_kwargs: None)+ topic = Topic(Redis(), "recovery-test")+ monkeypatch.setattr(topic, "publish", lambda _payload: None)+ monkeypatch.setattr(task_module.MessageBasedAppGenerator, "get_response_topic", lambda *_: topic)+ return AppTaskControlService(redis_client=recovery_redis)+++def _seed_pause(+ session: Session,+ states: dict[str, bytes],+ app_mode: AppMode = AppMode.WORKFLOW,+ *,+ pause_id: str | None = None,+) -> tuple[WorkflowRun, WorkflowPause, WorkflowResumptionContext]:+ run = WorkflowRun(+ id=str(uuid4()),+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ type=WorkflowType.WORKFLOW if app_mode == AppMode.WORKFLOW else WorkflowType.CHAT,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.PAUSED,+ created_by_role=CreatorUserRole.END_USER,+ created_by="user-1",+ created_at=datetime(2026, 10, 10),+ elapsed_time=0,+ total_tokens=0,+ total_steps=1,+ exceptions_count=0,+ )+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key=f"pause/{run.id}")+ if pause_id is not None:+ pause.id = pause_id+ config = WorkflowUIBasedAppConfig(+ tenant_id=run.tenant_id, app_id=run.app_id, app_mode=app_mode, workflow_id=run.workflow_id+ )+ if app_mode == AppMode.WORKFLOW:+ wrapper = _WorkflowGenerateEntityWrapper(+ entity=WorkflowAppGenerateEntity(+ task_id=str(uuid4()),+ app_config=config,+ inputs={},+ files=[],+ user_id=run.created_by,+ stream=True,+ invoke_from=InvokeFrom.WEB_APP,+ workflow_execution_id=run.id,+ )+ )+ else:+ wrapper = _AdvancedChatAppGenerateEntityWrapper(+ entity=AdvancedChatAppGenerateEntity(+ task_id=str(uuid4()),+ app_config=config,+ inputs={},+ files=[],+ user_id=run.created_by,+ stream=True,+ invoke_from=InvokeFrom.WEB_APP,+ workflow_run_id=run.id,+ query="Approve this run",+ conversation_id="conversation-1",+ )+ )+ context = WorkflowResumptionContext(generate_entity=wrapper, serialized_graph_runtime_state="{}")+ states[pause.state_object_key] = context.dumps().encode()+ session.add_all([run, pause])+ session.commit()+ return run, pause, context+++def _stop(service: AppTaskControlService, task_id: str, app_mode: AppMode = AppMode.WORKFLOW) -> bool:+ return service.stop_workflow_task(+ tenant_id="tenant-1",+ app_id="app-1",+ task_id=task_id,+ app_mode=app_mode,+ owner=(CreatorUserRole.END_USER, "user-1"),+ )++[email protected]("app_mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT])+def test_stop_recovers_and_cancels_a_legacy_pause_without_prior_replay(+ sqlite_session: Session,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ app_mode: AppMode,+ recovery_redis: _StopRedis,+) -> None:+ run, pause, context = _seed_pause(sqlite_session, pause_states, app_mode)+ task_id = context.get_generate_entity().task_id+ assert task_id != run.id+ assert run.task_id is None+ context.get_generate_entity().task_id = task_id.upper()+ pause_states[pause.state_object_key] = context.dumps().encode()++ assert _stop(recovery_service, task_id.upper(), app_mode)++ sqlite_session.refresh(run)+ assert run.task_id == task_id+ assert run.status == WorkflowExecutionStatus.STOPPED+ assert run.stop_requested_at is not None+ assert sqlite_session.scalar(select(WorkflowPause.id).where(WorkflowPause.id == pause.id)) is None+ sqlite_session.commit()+ pause_states.clear()+ assert _stop(recovery_service, task_id, app_mode)+ assert recovery_redis.reads == []+ assert recovery_redis.operations == []++[email protected]("field", ["tenant_id", "app_id", "created_by", "created_by_role"])+def test_recovery_excludes_unauthorized_pauses_before_storage_reads(+ sqlite_session: Session,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ monkeypatch: pytest.MonkeyPatch,+ field: str,+) -> None:+ run, _, context = _seed_pause(sqlite_session, pause_states)+ setattr(run, field, CreatorUserRole.ACCOUNT if field == "created_by_role" else "other-owner")+ sqlite_session.commit()++ def reject_load(_key: str) -> bytes:+ pytest.fail("Unauthorized pauses must not reach snapshot storage")++ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.load", reject_load)+ assert not _stop(recovery_service, context.get_generate_entity().task_id)+ sqlite_session.refresh(run)+ assert run.task_id is None+ assert run.stop_requested_at is None+ assert run.status == WorkflowExecutionStatus.PAUSED++[email protected]("field", ["task_id", "run_id", "tenant_id", "app_id", "user_id", "app_config"])+def test_recovery_rejects_snapshot_identity_mismatches_and_continues(+ sqlite_session: Session,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ field: str,+) -> None:+ invalid, pause, invalid_context = _seed_pause(sqlite_session, pause_states, pause_id=str(UUID(int=1)))+ valid, _, valid_context = _seed_pause(sqlite_session, pause_states, pause_id=str(UUID(int=2)))+ state = json.loads(pause_states[pause.state_object_key])+ entity = state["generate_entity"]["entity"]+ if field in {"tenant_id", "app_id"}:+ entity["app_config"][field] = "other-owner"+ elif field == "app_config":+ del entity["app_config"]+ else:+ entity["workflow_execution_id" if field == "run_id" else field] = "invalid-task-or-owner"+ pause_states[pause.state_object_key] = json.dumps(state).encode()++ assert not _stop(recovery_service, invalid_context.get_generate_entity().task_id)+ sqlite_session.refresh(invalid)+ sqlite_session.refresh(valid)+ assert invalid.task_id is None+ assert invalid.status == WorkflowExecutionStatus.PAUSED+ assert invalid.stop_requested_at is None+ assert valid.task_id is None+ sqlite_session.commit()++ assert _stop(recovery_service, valid_context.get_generate_entity().task_id)+ sqlite_session.refresh(invalid)+ sqlite_session.refresh(valid)+ assert invalid.task_id is None+ assert valid.task_id == valid_context.get_generate_entity().task_id+ assert valid.status == WorkflowExecutionStatus.STOPPED++[email protected]("missing", ["pause", "state", "corrupt_state"])+def test_recovery_skips_unrecoverable_pause_and_finds_later_match(+ sqlite_session: Session,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ missing: str,+) -> None:+ invalid, pause, invalid_context = _seed_pause(sqlite_session, pause_states, pause_id=str(UUID(int=1)))+ valid, _, valid_context = _seed_pause(sqlite_session, pause_states, pause_id=str(UUID(int=2)))+ if missing == "pause":+ sqlite_session.delete(pause)+ sqlite_session.commit()+ elif missing == "state":+ del pause_states[pause.state_object_key]+ else:+ pause_states[pause.state_object_key] = b"invalid JSON"++ assert not _stop(recovery_service, invalid_context.get_generate_entity().task_id)+ assert _stop(recovery_service, valid_context.get_generate_entity().task_id)++ sqlite_session.refresh(invalid)+ sqlite_session.refresh(valid)+ assert invalid.task_id is None+ assert invalid.status == WorkflowExecutionStatus.PAUSED+ assert invalid.stop_requested_at is None+ assert valid.status == WorkflowExecutionStatus.STOPPED++[email protected]("change", ["replaced_pause", "resumed", "already_bound"])+def test_indexing_does_not_overwrite_a_concurrent_transition(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ pause_states: dict[str, bytes],+ change: str,+) -> None:+ run, pause, context = _seed_pause(sqlite_session, pause_states)+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ saved_pause = repository.get_workflow_pause(run.id)+ assert saved_pause is not None+ task_id = str(uuid4()) if change == "already_bound" else None+ if change == "replaced_pause":+ sqlite_session.delete(pause)+ sqlite_session.flush()+ sqlite_session.add(+ WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="replacement")+ )+ elif change == "resumed":+ run.status = WorkflowExecutionStatus.RUNNING+ pause.resumed_at = datetime(2026, 10, 10, 1)+ else:+ run.task_id = task_id+ sqlite_session.commit()++ assert not AppTaskService.index_workflow_pause_task(+ workflow_run=run,+ pause_entity=saved_pause,+ resumption_context=context,+ workflow_run_repo=repository,+ )++ sqlite_session.refresh(run)+ assert run.task_id == task_id++[email protected]("resumed", [False, True])+def test_stop_retries_durable_admission_after_another_request_binds_the_task(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ monkeypatch: pytest.MonkeyPatch,+ resumed: bool,+ recovery_redis: _StopRedis,+) -> None:+ run, pause, context = _seed_pause(sqlite_session, pause_states)+ task_id = context.get_generate_entity().task_id++ def load(key: str) -> bytes:+ with sqlite_session_factory() as session, session.begin():+ concurrent_run = session.get(WorkflowRun, run.id)+ assert concurrent_run is not None+ concurrent_run.task_id = task_id+ if resumed:+ concurrent_run.status = WorkflowExecutionStatus.RUNNING+ concurrent_pause = session.get(WorkflowPause, pause.id)+ assert concurrent_pause is not None+ concurrent_pause.resumed_at = datetime(2026, 10, 10, 1)+ return pause_states[key]++ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.load", load)+ assert _stop(recovery_service, task_id)+ sqlite_session.refresh(run)+ assert run.task_id == task_id+ assert run.stop_requested_at is not None+ assert run.status == (WorkflowExecutionStatus.RUNNING if resumed else WorkflowExecutionStatus.STOPPED)+ assert recovery_redis.reads == []+ assert recovery_redis.operations == (["legacy_flag", "graph_command"] if resumed else [])++[email protected]("status", [WorkflowExecutionStatus.PAUSED, WorkflowExecutionStatus.RUNNING])+def test_prebound_tasks_stop_without_reading_legacy_snapshots(+ sqlite_session: Session,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ monkeypatch: pytest.MonkeyPatch,+ status: WorkflowExecutionStatus,+) -> None:+ run, _, context = _seed_pause(sqlite_session, pause_states)+ run.task_id = context.get_generate_entity().task_id+ run.status = status+ sqlite_session.commit()++ def reject_load(_key: str) -> bytes:+ pytest.fail("Already indexed tasks must not read snapshots")++ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.load", reject_load)+ assert _stop(recovery_service, context.get_generate_entity().task_id)+ sqlite_session.refresh(run)+ assert run.stop_requested_at is not None++[email protected]("excluded", ["terminal", "pipeline", "resumed", "running"])+def test_recovery_excludes_nonrecoverable_runs_before_storage_reads(+ sqlite_session: Session,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ monkeypatch: pytest.MonkeyPatch,+ excluded: str,+) -> None:+ run, pause, context = _seed_pause(sqlite_session, pause_states)+ if excluded == "terminal":+ run.status = WorkflowExecutionStatus.SUCCEEDED+ elif excluded == "pipeline":+ run.type = WorkflowType.RAG_PIPELINE+ elif excluded == "resumed":+ pause.resumed_at = datetime(2026, 10, 10, 1)+ else:+ run.status = WorkflowExecutionStatus.RUNNING+ sqlite_session.commit()++ def reject_load(_key: str) -> bytes:+ pytest.fail("Ineligible runs must not read snapshots")++ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.load", reject_load)+ assert not _stop(recovery_service, context.get_generate_entity().task_id)+ sqlite_session.refresh(run)+ assert run.task_id is None+ assert run.stop_requested_at is None+++def test_recovery_pages_pauses_without_backfilling_unrelated_runs(+ sqlite_session: Session,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+) -> None:+ for index in range(1, 101):+ _seed_pause(sqlite_session, pause_states, pause_id=str(UUID(int=index)))+ run, _, context = _seed_pause(sqlite_session, pause_states, pause_id=str(UUID(int=101)))+ task_id = context.get_generate_entity().task_id++ assert _stop(recovery_service, task_id)++ indexed = sqlite_session.execute(+ select(WorkflowRun.id, WorkflowRun.task_id).where(WorkflowRun.task_id.is_not(None))+ )+ assert indexed.tuples().all() == [(run.id, task_id)]+++def test_snapshot_storage_reads_do_not_hold_a_database_transaction(+ sqlite_session: Session,+ sqlite_engine: Engine,+ pause_states: dict[str, bytes],+ recovery_service: AppTaskControlService,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _, _, context = _seed_pause(sqlite_session, pause_states)+ active_transactions: set[Connection] = set()++ def load(key: str) -> bytes:+ assert not active_transactions+ return pause_states[key]++ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.load", load)+ listeners: tuple[tuple[str, Callable[[Connection], None]], ...] = (+ ("begin", active_transactions.add),+ ("commit", active_transactions.discard),+ ("rollback", active_transactions.discard),+ )+ for name, listener in listeners:+ event.listen(sqlite_engine, name, listener)+ try:+ assert _stop(recovery_service, context.get_generate_entity().task_id)+ finally:+ for name, listener in listeners:+ event.remove(sqlite_engine, name, listener)diff --git a/api/tests/unit_tests/services/test_paused_workflow_stop.py b/api/tests/unit_tests/services/test_paused_workflow_stop.pynew file mode 100644index 00000000000..6b020b54eef--- /dev/null+++ b/api/tests/unit_tests/services/test_paused_workflow_stop.py@@ -0,0 +1,292 @@+"""Paused stops persist before publishing terminal events or retiring resources."""++import json+from collections.abc import Callable+from datetime import datetime++import pytest+from redis import Redis+from sqlalchemy.orm import Session, sessionmaker++import services.app_task_service as task_module+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from core.workflow.nodes.agent_v2.session_store import WorkflowAgentWorkspaceStore+from extensions.logstore.aliyun_logstore import AliyunLogStore+from graphon.entities import WorkflowNodeExecution+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType+from libs.broadcast_channel.redis.pubsub_channel import Topic+from models import EndUser+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.model import AppMode+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowPause, WorkflowRun+from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository+from services.app_task_service import AppTaskService+from tests.unit_tests.services.test_app_task_service import _StopRedis++[email protected]("app_mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT])[email protected]("backend", ["sql", "celery", "logstore"])[email protected]("node_status", [WorkflowNodeExecutionStatus.PAUSED, WorkflowNodeExecutionStatus.RETRY])+def test_paused_stop_publishes_terminal_event_after_commit_and_retires_run_resources(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ app_mode: AppMode,+ backend: str,+ node_status: WorkflowNodeExecutionStatus,+ config_overrides: Callable[..., None],+) -> None:+ config_overrides(+ CORE_WORKFLOW_EXECUTION_REPOSITORY={+ "sql": "core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository",+ "celery": "core.repositories.celery_workflow_execution_repository.CeleryWorkflowExecutionRepository",+ "logstore": (+ "extensions.logstore.repositories.logstore_workflow_execution_repository."+ "LogstoreWorkflowExecutionRepository"+ ),+ }[backend],+ CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY={+ "sql": (+ "core.repositories.sqlalchemy_workflow_node_execution_repository."+ "SQLAlchemyWorkflowNodeExecutionRepository"+ ),+ "celery": (+ "core.repositories.celery_workflow_node_execution_repository.CeleryWorkflowNodeExecutionRepository"+ ),+ "logstore": (+ "extensions.logstore.repositories.logstore_workflow_node_execution_repository."+ "LogstoreWorkflowNodeExecutionRepository"+ ),+ }[backend],+ LOGSTORE_DUAL_WRITE_ENABLED=False,+ )+ monkeypatch.setattr(+ "repositories.sqlalchemy_api_workflow_run_repository.naive_utc_now", lambda: datetime(2026, 10, 10, 0, 1)+ )+ logstore_records: dict[str, list[dict[str, str]]] = {}+ monkeypatch.setattr(AliyunLogStore, "_instance", None)+ monkeypatch.setattr(AliyunLogStore, "_initialized", False)+ monkeypatch.setenv("ALIYUN_SLS_ENDPOINT", "localhost")+ monkeypatch.setattr(AliyunLogStore, "_check_endpoint_connectivity", staticmethod(lambda *_: None))+ monkeypatch.setattr(+ AliyunLogStore,+ "put_log",+ lambda _self, name, record: logstore_records.setdefault(name, []).append(dict(record)),+ )+ queued_node_saves: list[object] = []+ queued_saves: list[object] = []+ monkeypatch.setattr(+ "core.repositories.celery_workflow_node_execution_repository.save_workflow_node_execution_task.delay",+ lambda *args, **kwargs: queued_node_saves.append((args, kwargs)),+ )+ monkeypatch.setattr(+ "core.repositories.celery_workflow_execution_repository.save_workflow_execution_task.delay",+ lambda *args, **kwargs: queued_saves.append((args, kwargs)),+ )+ run = WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.PAUSED,+ created_by_role=CreatorUserRole.END_USER,+ created_by="user-1",+ created_at=datetime(2026, 10, 10),+ elapsed_time=1,+ total_tokens=0,+ total_steps=2,+ exceptions_count=0,+ )+ run.task_id = "11111111-1111-1111-1111-111111111111"+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="state")+ sqlite_session.add_all([run, pause])+ sqlite_session.commit()+ for node_app, origin in (("app-1", WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN),):+ SQLAlchemyWorkflowNodeExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=EndUser(id="user-1"),+ app_id=node_app,+ triggered_from=origin,+ ).save(+ WorkflowNodeExecution(+ id=f"node-{node_app}",+ node_execution_id=f"node-{node_app}",+ workflow_id=f"workflow-{node_app}",+ workflow_execution_id=run.id,+ index=1,+ node_id="review",+ node_type="human-input",+ title="Review",+ status=node_status,+ process_data={"__dify_retry_history": [{"retry_index": 1, "error": "retry failed"}]},+ created_at=run.created_at,+ )+ )+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_session_factory.kw["bind"]))+ redis = _StopRedis()+ monkeypatch.setattr(task_module, "redis_client", redis)+ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.delete", lambda _: None)+ calls: list[str] = []++ def retire(*, tenant_id: str, app_id: str, workflow_run_id: str) -> list[str]:+ assert (tenant_id, app_id, workflow_run_id) == ("tenant-1", "app-1", "run-1")+ with sqlite_session_factory() as session:+ persisted = session.get(WorkflowRun, workflow_run_id)+ assert persisted is not None+ assert persisted.status == WorkflowExecutionStatus.STOPPED+ assert persisted.elapsed_time == 1+ node = session.get(WorkflowNodeExecutionModel, "node-app-1")+ assert node is not None+ assert node.status == WorkflowNodeExecutionStatus.FAILED+ assert node.finished_at == datetime(2026, 10, 10, 0, 1)+ assert node.process_data_dict == {"__dify_retry_history": [{"retry_index": 1, "error": "retry failed"}]}+ calls.append("retire")+ return ["workspace-1"]++ monkeypatch.setattr(WorkflowAgentWorkspaceStore, "retire_workflow_run", staticmethod(retire))+ collections: list[dict[str, object]] = []++ def enqueue(**kwargs: object) -> None:+ calls.append("collect")+ collections.append(kwargs)++ monkeypatch.setattr("tasks.collect_agent_resources_task.enqueue_agent_resource_collection", enqueue)+ publications: list[bytes] = []++ def publish(_channel: str, payload: bytes) -> None:+ calls.append("publish")+ publications.append(payload)++ topic_client = Redis()+ monkeypatch.setattr(topic_client, "publish", publish)+ topic = Topic(topic_client, "workflow-events")+ monkeypatch.setattr(task_module.MessageBasedAppGenerator, "get_response_topic", lambda *_: topic)++ assert run.task_id is not None+ AppTaskService.stop_workflow_task(+ tenant_id="tenant-1",+ app_id="app-1",+ task_id=run.task_id,+ app_mode=app_mode,+ owner=(CreatorUserRole.END_USER, "user-1"),+ )++ assert calls == ["retire", "collect", "publish"]+ assert collections == [{"tenant_id": "tenant-1", "workspace_ids": ["workspace-1"]}]+ assert len(publications) == 1+ event = json.loads(publications[0])+ assert event["event"] == "workflow_finished"+ assert event["data"]["status"] == "stopped"+ assert event["data"]["elapsed_time"] == 1+ assert event["workflow_run_id"] == "run-1"+ assert event["task_id"] == run.task_id+ assert redis.reads == []+ assert redis.operations == []+ assert queued_saves == []+ if backend == "logstore":+ assert len(logstore_records[AliyunLogStore.workflow_execution_logstore]) == 1+ history = logstore_records[AliyunLogStore.workflow_execution_logstore][0]+ assert history["status"] == "stopped"+ assert float(history["elapsed_time"]) == 1+ assert (history["tenant_id"], history["app_id"], history["id"]) == ("tenant-1", "app-1", "run-1")+ node_history = logstore_records[AliyunLogStore.workflow_node_execution_logstore]+ assert all(float(record["elapsed_time"]) == 60 for record in node_history)+ assert all(+ json.loads(record["process_data"])["__dify_retry_history"][0]["error"] == "retry failed"+ for record in node_history+ )+ assert {(record["app_id"], record["triggered_from"], record["status"]) for record in node_history} == {+ ("app-1", "workflow-run", "failed"),+ }++[email protected]("task_id_input", ["11111111-aaaa-1111-1111-111111111111", "11111111AAAA11111111111111111111"])+def test_stop_after_resume_claim_uses_durable_owner_before_queue_is_recreated(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ task_id_input: str,+) -> None:+ run = WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.PAUSED,+ created_by_role=CreatorUserRole.END_USER,+ created_by="user-1",+ )+ run.task_id = "11111111-aaaa-1111-1111-111111111111"+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="state")+ sqlite_session.add_all([run, pause])+ sqlite_session.commit()+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ entity = repository.get_workflow_pause(run.id)+ assert entity is not None+ repository.resume_workflow_pause(run.id, entity)+ redis = _StopRedis()+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_session_factory.kw["bind"]))+ monkeypatch.setattr(task_module, "redis_client", redis)+ AppTaskService.stop_workflow_task(+ tenant_id="tenant-1",+ app_id="app-1",+ task_id=task_id_input,+ app_mode=AppMode.ADVANCED_CHAT,+ owner=(CreatorUserRole.END_USER, "user-1"),+ )+ assert redis.reads == []+ assert redis.values[f"generate_task_stopped:{run.task_id}"] == b"1"+ assert json.loads(redis.commands[f"workflow:{run.task_id}:commands"][0])["command_type"] == "abort"+++ ("tenant_id", "app_id", "owner"),+ [+ ("other-tenant", "app-1", None),+ ("tenant-1", "other-app", None),+ ("tenant-1", "app-1", (CreatorUserRole.END_USER, "other-user")),+ ("tenant-1", "app-1", (CreatorUserRole.ACCOUNT, "user-1")),+ ],+)+def test_running_task_admission_rejects_foreign_tenant_app_user_and_role(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ tenant_id: str,+ app_id: str,+ owner: tuple[CreatorUserRole, str] | None,+) -> None:+ task_id = "22222222-2222-2222-2222-222222222222"+ run = WorkflowRun(+ id="run-1",+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ task_id=task_id,+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.RUNNING,+ created_by_role=CreatorUserRole.END_USER,+ created_by="user-1",+ )+ sqlite_session.add(run)+ sqlite_session.commit()+ monkeypatch.setattr(type(task_module.db), "engine", property(lambda _: sqlite_session_factory.kw["bind"]))+ redis = _StopRedis(values={f"generate_task_belong:{task_id}": b"end-user-user-1"})+ monkeypatch.setattr(task_module, "redis_client", redis)+ AppTaskService.stop_workflow_task(+ tenant_id=tenant_id, app_id=app_id, task_id=task_id, app_mode=AppMode.WORKFLOW, owner=owner+ )+ sqlite_session.expire_all()+ assert run.status == WorkflowExecutionStatus.RUNNING+ assert run.stop_requested_at is None+ assert redis.reads == []+ assert redis.operations == []diff --git a/api/tests/unit_tests/services/test_snippet_generate_service.py b/api/tests/unit_tests/services/test_snippet_generate_service.pyindex 946d9ed4699..44517ec3de0 100644--- a/api/tests/unit_tests/services/test_snippet_generate_service.py+++ b/api/tests/unit_tests/services/test_snippet_generate_service.py@@ -1,5 +1,6 @@import jsonimport unittest.mock+from copy import deepcopyimport pytestfrom sqlalchemy.orm import Session, sessionmaker@@ -7,9 +8,12 @@ from sqlalchemy.orm import Session, sessionmakerfrom controllers.console.app.workflow import LoopNodeRunPayloadfrom core.app.app_config.features.file_upload.manager import FileUploadConfigManagerfrom core.app.apps.workflow.app_generator import WorkflowAppGenerator+from core.workflow.node_factory import DifyNodeFactoryfrom core.workflow.snippet_start import SNIPPET_VIRTUAL_START_NODE_IDfrom graphon.file import FileUploadConfigfrom graphon.file.models import File+from graphon.graph import Graph+from graphon.runtime import RuntimeState, VariablePoolfrom models.account import Accountfrom models.snippet import CustomizedSnippet, SnippetTypefrom models.workflow import Workflow, WorkflowKind, WorkflowNodeExecutionModel@@ -17,6 +21,7 @@ from services.snippet_generate_service import SnippetGenerateServicefrom services.snippet_service import SnippetServicefrom services.workflow_service import WorkflowServicefrom tests.unit_tests.model_factories import make_account, make_workflow+from tests.workflow_test_utils import build_test_graph_init_paramsdef _workflow(graph: dict) -> Workflow:@@ -133,6 +138,129 @@ def test_ensure_start_node_injects_virtual_start_for_root_candidates(monkeypatchmake_transient.assert_called_once_with(workflow)+def _nested_snippet_graph(ownership: str, outer_type: str) -> dict:+ inner_type = "iteration" if outer_type == "loop" else "loop"+ nodes = []+ for node_id, node_type in (("outer", outer_type), ("inner", inner_type)):+ data = {"type": node_type, "title": node_id, "start_node_id": f"{node_id}-start"}+ if node_type == "loop":+ data.update(loop_count=1, break_conditions=[], logical_operator="and")+ else:+ data.update(iterator_selector=["input", "items"], output_selector=[node_id, "item"])+ nodes.extend(+ [+ {"id": node_id, "data": data},+ {"id": f"{node_id}-start", "data": {"type": f"{node_type}-start", "title": "Container start"}},+ ]+ )+ nodes.append({"id": "output", "data": {"type": "end", "title": "Output", "outputs": []}})+ owners = {"outer-start": "outer", "inner": "outer", "inner-start": "inner"}+ for node in nodes:+ owner = owners.get(node["id"], "")+ if ownership == "canonical":+ node["data"]["container_id"] = owner+ node["parentId"] = "stale-parent"+ node["data"].update(iteration_id="stale-iteration", loop_id="stale-loop")+ elif ownership == "parent":+ if owner:+ node["parentId"] = owner+ elif owner:+ node["data"][f"{outer_type}_id"] = "outer"+ if owner == "inner":+ node["data"][f"{inner_type}_id"] = "inner"+ return {+ "nodes": nodes,+ "edges": [{"source": "outer", "target": "output"}, {"source": "outer-start", "target": "inner"}],+ }++[email protected]("ownership", ["canonical", "parent", "legacy"])[email protected]("outer_type", ["iteration", "loop"])+def test_virtual_start_preserves_nested_container_graphs(ownership: str, outer_type: str) -> None:+ graph_config = _nested_snippet_graph(ownership, outer_type)+ original = deepcopy(graph_config)++ injected = SnippetGenerateService._inject_virtual_start_node(graph_config, [])++ assert [edge["target"] for edge in injected["edges"] if edge["source"] == SNIPPET_VIRTUAL_START_NODE_ID] == [+ "outer"+ ]+ factory = DifyNodeFactory(+ init_params=build_test_graph_init_params(graph_config=injected),+ runtime_state=RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=0.0),+ )+ graph = Graph.init(graph_config=injected, node_factory=factory, root_node_id=SNIPPET_VIRTUAL_START_NODE_ID)+ assert set(graph.nodes) == {SNIPPET_VIRTUAL_START_NODE_ID, "outer", "output"}+ for container_id, expected in (("outer", {"outer-start", "inner"}), ("inner", {"inner-start"})):+ graph = Graph.init(+ graph_config=graph.graph_config,+ node_factory=graph.node_factory,+ root_node_id=f"{container_id}-start",+ container_id=container_id,+ )+ assert set(graph.nodes) == expected+ assert graph_config == original+++def test_virtual_start_connects_all_flat_roots_without_changing_the_input() -> None:+ graph_config = {+ "nodes": [{"id": node_id, "data": {"type": "end"}} for node_id in ("first", "second", "downstream")],+ "edges": [{"source": "first", "target": "downstream"}],+ }+ original = deepcopy(graph_config)++ injected = SnippetGenerateService._inject_virtual_start_node(graph_config, [])++ assert [edge["target"] for edge in injected["edges"] if edge["source"] == SNIPPET_VIRTUAL_START_NODE_ID] == [+ "first",+ "second",+ ]+ assert graph_config == original+++def test_virtual_start_does_not_promote_disconnected_owned_nodes() -> None:+ graph_config = _nested_snippet_graph("canonical", "loop")+ graph_config["nodes"].append({"id": "detached", "data": {"type": "end", "container_id": "inner"}})++ injected = SnippetGenerateService._inject_virtual_start_node(graph_config, [])++ assert [edge["target"] for edge in injected["edges"] if edge["source"] == SNIPPET_VIRTUAL_START_NODE_ID] == [+ "outer"+ ]+++def test_virtual_start_keeps_missing_container_ownership_invalid() -> None:+ graph_config = _nested_snippet_graph("legacy", "loop")+ nodes = {node["id"]: node for node in graph_config["nodes"]}+ nodes["inner-start"]["data"]["container_id"] = "missing-container"+ original = deepcopy(graph_config)++ injected = SnippetGenerateService._inject_virtual_start_node(graph_config, [])++ assert not any(+ edge["source"] == SNIPPET_VIRTUAL_START_NODE_ID and edge["target"] == "inner-start"+ for edge in injected["edges"]+ )+ factory = DifyNodeFactory(+ init_params=build_test_graph_init_params(graph_config=injected),+ runtime_state=RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=0.0),+ )+ with pytest.raises(ValueError, match="unknown or cyclic containers"):+ Graph.init(graph_config=injected, node_factory=factory, root_node_id=SNIPPET_VIRTUAL_START_NODE_ID)+ assert graph_config == original+++def test_virtual_start_rejects_ambiguous_legacy_container_ownership() -> None:+ graph_config = _nested_snippet_graph("legacy", "loop")+ nodes = {node["id"]: node for node in graph_config["nodes"]}+ del nodes["inner"]["data"]["loop_id"]+ original = deepcopy(graph_config)++ with pytest.raises(ValueError, match="ambiguous legacy container owners"):+ SnippetGenerateService._inject_virtual_start_node(graph_config, [])+ assert graph_config == original++def test_parse_files_returns_empty_when_upload_config_disabled(monkeypatch: pytest.MonkeyPatch):workflow = _workflow({"nodes": [], "edges": []})diff --git a/api/tests/unit_tests/services/test_workflow_node_variables.py b/api/tests/unit_tests/services/test_workflow_node_variables.pynew file mode 100644index 00000000000..b26843991db--- /dev/null+++ b/api/tests/unit_tests/services/test_workflow_node_variables.py@@ -0,0 +1,333 @@+from collections.abc import Mapping, Sequence+from typing import Literal++import pytest+from sqlalchemy import Engine+from sqlalchemy.orm import Session++from core.workflow.nodes.human_input.entities import HumanInputNodeData+from core.workflow.workflow_entry import WorkflowEntry+from graphon.entities.graph_config import NodeConfigDictAdapter+from graphon.nodes.list_operator.entities import FilterOperator+from graphon.runtime import VariablePool+from graphon.variable_loader import DUMMY_VARIABLE_LOADER+from graphon.variables import StringSegment, VariableBase+from models.workflow import WorkflowDraftVariable+from services.workflow_draft_variable_service import DraftVarLoader+from services.workflow_node_variables import get_human_input_form_variable_mapping, load_additional_node_variables++[email protected]("container_type", [None, "iteration", "loop"])[email protected]("saved_root", [False, True])+def test_human_input_body_and_email_keep_nested_editor_selectors(+ container_type: Literal["iteration", "loop"] | None, saved_root: bool+) -> None:+ human: dict[str, object] = {+ "id": "human",+ "data": {+ "type": "human-input",+ "title": "Human",+ "form_content": "Hello {{#source.object.name#}} {{#$output.comment#}}",+ "inputs": [+ {+ "type": "paragraph",+ "output_variable_name": "comment",+ "default": {"type": "variable", "selector": ["source", "default"]},+ }+ ],+ "delivery_methods": [+ {+ "type": "email",+ "enabled": True,+ "config": {+ "recipients": {"items": list[object]()},+ "subject": "Review",+ "body": "Hello {{#source.object.email_name#}} {{#url#}}",+ },+ },+ {+ "type": "email",+ "enabled": False,+ "config": {+ "recipients": {"items": list[object]()},+ "subject": "Disabled",+ "body": "{{#missing.object.name#}}",+ },+ },+ ],+ },+ }+ nodes = [human]+ target: dict[str, object] = human+ if container_type:+ target = {"id": "container", "data": {"type": container_type, "title": "Container"}}+ nodes.insert(0, target)+ prefix = "human." if container_type else ""+ inputs = {+ f"{prefix}#source.object.name#": "Alice",+ f"{prefix}#source.object.email_name#": "Recipient",+ f"{prefix}#source.default#": "Default",+ }+ variable_pool = VariablePool()+ if saved_root:+ variable_pool.add(["source", "object"], {"name": "Old", "email_name": "Old", "keep": "Saved"})++ mapping = load_additional_node_variables(+ node_config=NodeConfigDictAdapter.validate_python(target),+ graph_config={"nodes": nodes},+ variable_mapping={},+ variable_loader=DUMMY_VARIABLE_LOADER,+ variable_pool=variable_pool,+ user_inputs=inputs,+ )+ WorkflowEntry.mapping_user_inputs_to_variable_pool(+ variable_mapping=mapping, user_inputs=inputs, variable_pool=variable_pool, tenant_id="tenant"+ )++ assert mapping == {+ f"{prefix}human.#source.object.name#": ["source", "object", "name"],+ f"{prefix}human.#source.object.email_name#": ["source", "object", "email_name"],+ f"{prefix}human.#source.default#": ["source", "default"],+ }+ variable = variable_pool.get(["source", "object"])+ assert variable is not None+ assert variable.value == {+ "name": "Alice",+ "email_name": "Recipient",+ **({"keep": "Saved"} if saved_root else {}),+ }+ assert get_human_input_form_variable_mapping("human", HumanInputNodeData.model_validate(human["data"])) == {+ "human.#source.object.name#": ["source", "object", "name"],+ "human.#source.default#": ["source", "default"],+ }++[email protected]("operator", [FilterOperator.EMPTY, FilterOperator.NOT_EMPTY])+def test_value_free_list_filters_do_not_require_stale_operands(+ operator: FilterOperator, monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine+) -> None:+ config = NodeConfigDictAdapter.validate_python(+ {+ "id": "list",+ "data": {+ "type": "list-operator",+ "title": "List",+ "variable": ["source", "items"],+ "filter_by": {+ "enabled": True,+ "conditions": [{"comparison_operator": operator, "value": "{{#deleted.old#}}"}],+ },+ "order_by": {"enabled": False},+ "limit": {"enabled": False},+ },+ }+ )+ loader = DraftVarLoader(sqlite_engine, "app", "tenant", "user")+ selectors_loaded: list[list[list[str]]] = []+ original_load = loader.load_variables++ def load_variables(selectors: list[list[str]]) -> list[VariableBase]:+ selectors_loaded.append(selectors)+ return original_load(selectors)++ monkeypatch.setattr(loader, "load_variables", load_variables)++ mapping = load_additional_node_variables(+ node_config=config,+ graph_config={"nodes": [config]},+ variable_mapping={},+ variable_loader=loader,+ variable_pool=VariablePool(),+ user_inputs={"#source.items#": ["", "a"]},+ )++ assert mapping == {"list.#source.items#": ["source", "items"]}+ assert selectors_loaded == [[]]+++ ("data", "inputs", "expected"),+ [+ (+ {+ "type": "human-input",+ "form_content": "Review {{#source.content#}} and {{#$output.comment#}}",+ "inputs": [+ {+ "type": "paragraph",+ "output_variable_name": "comment",+ "default": {"type": "variable", "selector": ["source", "default"]},+ }+ ],+ },+ {"#source.content#": "review", "#source.default#": "comment"},+ {"node.#source.content#": ["source", "content"], "node.#source.default#": ["source", "default"]},+ ),+ (+ {+ "type": "knowledge-index",+ "chunk_structure": "text_model",+ "index_chunk_variable_selector": ["source", "chunks"],+ },+ {"query": {"general_chunks": ["content"]}},+ {"node.query": ["source", "chunks"]},+ ),+ ],+)+def test_dify_node_input_mappings(+ data: Mapping[str, object], inputs: Mapping[str, object], expected: Mapping[str, Sequence[str]]+) -> None:+ config = NodeConfigDictAdapter.validate_python({"id": "node", "data": {"title": "Node", **data}})++ mapping = load_additional_node_variables(+ node_config=config,+ graph_config={"nodes": [config]},+ variable_mapping={},+ variable_loader=DUMMY_VARIABLE_LOADER,+ variable_pool=VariablePool(),+ user_inputs=inputs,+ )++ assert mapping == expected+++def test_missing_end_input_loads_saved_draft_value(+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session: Session+) -> None:+ config = NodeConfigDictAdapter.validate_python(+ {+ "id": "end",+ "data": {+ "type": "end",+ "title": "End",+ "outputs": [{"variable": "result", "value_selector": ["source", "value"]}],+ },+ }+ )+ variable_pool = VariablePool()+ sqlite_session.add(+ WorkflowDraftVariable.new_node_variable(+ app_id="app",+ user_id="user",+ node_id="source",+ name="value",+ value=StringSegment(value="saved"),+ node_execution_id="execution",+ )+ )+ sqlite_session.commit()+ loader = DraftVarLoader(sqlite_engine, "app", "tenant", "user")+ selectors_loaded: list[list[list[str]]] = []+ original_load = loader.load_variables++ def load_variables(selectors: list[list[str]]) -> list[VariableBase]:+ selectors_loaded.append(selectors)+ return original_load(selectors)++ monkeypatch.setattr(loader, "load_variables", load_variables)++ mapping = load_additional_node_variables(+ node_config=config,+ graph_config={"nodes": [config]},+ variable_mapping={},+ variable_loader=loader,+ variable_pool=variable_pool,+ user_inputs={},+ )++ assert mapping == {"end.#source.value#": ["source", "value"]}+ assert selectors_loaded == [[["source", "value"]]]+ variable = variable_pool.get(["source", "value"])+ assert variable is not None+ assert variable.value == "saved"+++def test_unavailable_end_output_remains_unset_after_draft_loading(+ monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine+) -> None:+ config = NodeConfigDictAdapter.validate_python(+ {+ "id": "end",+ "data": {+ "type": "end",+ "title": "End",+ "outputs": [{"variable": "result", "value_selector": ["source", "missing"]}],+ },+ }+ )+ variable_pool = VariablePool()+ loader = DraftVarLoader(sqlite_engine, "app", "tenant", "user")+ selectors_loaded: list[list[list[str]]] = []+ original_load = loader.load_variables++ def load_variables(selectors: list[list[str]]) -> list[VariableBase]:+ selectors_loaded.append(selectors)+ return original_load(selectors)++ monkeypatch.setattr(loader, "load_variables", load_variables)++ mapping = load_additional_node_variables(+ node_config=config,+ graph_config={"nodes": [config]},+ variable_mapping={},+ variable_loader=loader,+ variable_pool=variable_pool,+ user_inputs={},+ )++ assert selectors_loaded == [[["source", "missing"]]]+ assert mapping == {}+ assert variable_pool.get(["source", "missing"]) is None+++def test_container_loads_external_inputs_and_skips_internal_outputs_and_disabled_settings() -> None:+ container = {"id": "iteration", "data": {"type": "iteration", "title": "Iteration"}}+ operator = {+ "id": "list",+ "data": {+ "type": "list-operator",+ "title": "List",+ "variable": ["source", "items"],+ "filter_by": {"enabled": True, "conditions": [{"value": "{{#source.filter#}}"}]},+ "extract_by": {"enabled": False, "serial": "{{#source.unused#}}"},+ "order_by": {"enabled": False},+ "limit": {"enabled": False},+ },+ }+ aggregator = {+ "id": "aggregate",+ "data": {+ "type": "variable-aggregator",+ "title": "Aggregate",+ "output_type": "array[string]",+ "variables": [["source", "unused"]],+ "advanced_settings": {+ "group_enabled": True,+ "groups": [+ {+ "output_type": "array[string]",+ "group_name": "group",+ "variables": [["list", "result"], ["source", "missing"], ["source", "fallback"]],+ }+ ],+ },+ },+ }+ variable_pool = VariablePool()+ variable_pool.add(["source", "fallback"], ["saved"])++ mapping = load_additional_node_variables(+ node_config=NodeConfigDictAdapter.validate_python(container),+ graph_config={"nodes": [container, operator, aggregator]},+ variable_mapping={"iteration.iterator": ["source", "iterator"]},+ variable_loader=DUMMY_VARIABLE_LOADER,+ variable_pool=variable_pool,+ user_inputs={"list.#source.items#": ["one"], "list.#source.filter#": "one"},+ )++ assert mapping == {+ "iteration.iterator": ["source", "iterator"],+ "list.list.#source.items#": ["source", "items"],+ "list.list.#source.filter#": ["source", "filter"],+ "aggregate.aggregate.#source.fallback#": ["source", "fallback"],+ }diff --git a/api/tests/unit_tests/services/test_workflow_service.py b/api/tests/unit_tests/services/test_workflow_service.pyindex 89b27a2c634..facd0b020a7 100644--- a/api/tests/unit_tests/services/test_workflow_service.py+++ b/api/tests/unit_tests/services/test_workflow_service.py@@ -11,7 +11,7 @@ This test suite covers:import jsonimport uuid-from collections.abc import Callable+from collections.abc import Callable, Generatorfrom datetime import datetime, timedeltafrom types import SimpleNamespacefrom typing import Any, cast@@ -24,8 +24,18 @@ from sqlalchemy.dialects import postgresqlfrom sqlalchemy.engine import Enginefrom sqlalchemy.orm import Session, sessionmaker+from core.repositories.celery_workflow_node_execution_repository import CeleryWorkflowNodeExecutionRepository+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepositoryfrom core.workflow.llm_environment_variable import LLMEnvironmentVariable+from core.workflow.node_factory import DifyNodeFactory+from core.workflow.nodes.agent_v2.session_store import WorkflowAgentWorkspaceStore+from core.workflow.system_variables import SystemVariableKey, get_system_textfrom enums import DeploymentEdition+from extensions.ext_database import db+from extensions.logstore.repositories.logstore_workflow_node_execution_repository import (+ LogstoreWorkflowNodeExecutionRepository,+)+from graphon.engine_events import NodeEvent, NodeRunFailedEvent, NodeRunStartedEvent, NodeRunSucceededEventfrom graphon.enums import (BuiltinNodeTypes,ErrorStrategy,@@ -33,10 +43,10 @@ from graphon.enums import (WorkflowNodeExecutionStatus,)from graphon.errors import WorkflowNodeRunFailedError-from graphon.graph_events import NodeRunFailedEvent, NodeRunSucceededEventfrom graphon.model_runtime.entities.model_entities import ModelTypefrom graphon.node_events import NodeRunResultfrom graphon.nodes.http_request import HTTP_REQUEST_CONFIG_FILTER_KEY, HttpRequestNode, HttpRequestNodeConfig+from graphon.runtime import RuntimeState, VariablePoolfrom graphon.variables import StringVariablefrom graphon.variables.input_entities import VariableEntityTypefrom libs.datetime_utils import naive_utc_now@@ -55,7 +65,8 @@ from models.agent_config_entities import AgentSoulConfigfrom models.human_input import HumanInputFormRecipient, RecipientTypefrom models.model import App, AppModefrom models.tools import BuiltinToolProvider, WorkflowToolProvider-from models.workflow import Workflow, WorkflowType+from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowType+from repositories.factory import DifyAPIRepositoryFactoryfrom services.agent.retirement_service import WorkflowAgentRetirementServicefrom services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualErrorfrom services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError@@ -67,6 +78,7 @@ from services.workflow_service import (_rebuild_single_file,_setup_variable_pool,)+from tests.workflow_test_utils import build_test_graph_init_paramsclass TestWorkflowAssociatedDataFactory:@@ -3307,7 +3319,7 @@ class TestWorkflowServiceDraftExecution:# Assertassert result is not Nonemock_run.assert_called_once()- mock_repo.save.assert_called_once()+ mock_repo.save_synchronously.assert_called_once()mock_saver_cls.return_value.save.assert_called_once()def test_run_draft_workflow_node_should_execute_non_start_node_successfully(@@ -3340,7 +3352,7 @@ class TestWorkflowServiceDraftExecution:patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)),patch("services.workflow_service.WorkflowDraftVariableService"),patch("services.workflow_service.VariablePool") as mock_pool_cls,- patch("services.workflow_service.default_system_variables") as mock_default_system_variables,+ patch("services.workflow_service.build_system_variables") as mock_build_system_variables,patch("services.workflow_service.build_bootstrap_variables") as mock_build_bootstrap_variables,patch("services.workflow_service.add_variables_to_pool") as mock_add_variables_to_pool,patch("services.workflow_service.DraftVarLoader"),@@ -3385,15 +3397,193 @@ class TestWorkflowServiceDraftExecution:# Assert# For non-start nodes, bootstrap variables should be loaded into an empty pool.mock_pool_cls.assert_called_once_with()- mock_default_system_variables.assert_called_once()+ mock_build_system_variables.assert_called_once_with()mock_build_bootstrap_variables.assert_called_once_with(- system_variables=mock_default_system_variables.return_value,+ system_variables=mock_build_system_variables.return_value,environment_variables=draft_workflow.environment_variables,)mock_add_variables_to_pool.assert_called_once_with(mock_pool_cls.return_value, mock_build_bootstrap_variables.return_value)+ @pytest.mark.parametrize(+ "repository_type",+ [+ SQLAlchemyWorkflowNodeExecutionRepository,+ CeleryWorkflowNodeExecutionRepository,+ LogstoreWorkflowNodeExecutionRepository,+ ],+ )+ @pytest.mark.parametrize("status", [WorkflowNodeExecutionStatus.SUCCEEDED, WorkflowNodeExecutionStatus.FAILED])+ def test_single_step_persists_caller_before_body_and_keeps_workspace_on_completion(+ self,+ service: WorkflowService,+ sqlite_engine: Engine,+ config_overrides: Callable[..., None],+ repository_type: type[SQLAlchemyWorkflowNodeExecutionRepository]+ | type[CeleryWorkflowNodeExecutionRepository]+ | type[LogstoreWorkflowNodeExecutionRepository],+ status: WorkflowNodeExecutionStatus,+ ) -> None:+ app = TestWorkflowAssociatedDataFactory.create_app()+ account = TestWorkflowAssociatedDataFactory.create_account()+ node_id = "agent-node"+ workflow = TestWorkflowAssociatedDataFactory.create_workflow(+ graph={+ "nodes": [{"id": node_id, "data": {"type": "agent", "version": "2", "title": "Agent"}}],+ "edges": [],+ }+ )+ execution_id = str(uuid.uuid4())+ workspace_binding_id = str(uuid.uuid4())+ workflow_binding_id = str(uuid.uuid4())+ started_at = naive_utc_now()+ config_overrides(AGENT_BACKEND_USE_FAKE=True)+ node = DifyNodeFactory(+ init_params=build_test_graph_init_params(+ workflow_id=workflow.id,+ graph_config=workflow.graph_dict,+ tenant_id=app.tenant_id,+ app_id=app.id,+ user_id=account.id,+ ),+ runtime_state=RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0),+ ).create_node(+ {+ "id": node_id,+ "data": {"type": "agent", "version": "2", "title": "Agent", "agent_node_kind": "dify_agent"},+ }+ )+ config_overrides(LOGSTORE_DUAL_WRITE_ENABLED=False)+ logstore_records: list[dict[str, str]] = []+ logstore = MagicMock()+ logstore.put_log.side_effect = lambda _name, record: logstore_records.append(dict(record))+ logstore.execute_sql.side_effect = lambda **_kwargs: logstore_records[-1:]+ with (+ patch(+ "extensions.logstore.repositories.logstore_workflow_node_execution_repository.AliyunLogStore",+ return_value=logstore,+ ),+ patch(+ "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.AliyunLogStore",+ return_value=logstore,+ ),+ ):+ repository = repository_type(+ session_factory=sqlite_engine,+ tenant_id=app.tenant_id,+ user=account,+ app_id=app.id,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,+ )+ if repository_type is LogstoreWorkflowNodeExecutionRepository:+ config_overrides(+ API_WORKFLOW_NODE_EXECUTION_REPOSITORY=(+ "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository."+ "LogstoreAPIWorkflowNodeExecutionRepository"+ )+ )+ service._node_execution_service_repo = (+ DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(+ session_maker=sessionmaker(sqlite_engine)+ )+ )++ def node_events(variable_pool: VariablePool) -> Generator[NodeEvent]:+ yield NodeRunStartedEvent(+ id=execution_id,+ node_id=node_id,+ node_type=BuiltinNodeTypes.AGENT,+ node_version="2",+ node_title="Agent",+ start_at=started_at,+ )+ with Session(sqlite_engine) as session:+ execution = WorkflowAgentWorkspaceStore._load_execution_by_identity(+ session=session,+ tenant_id=app.tenant_id,+ app_id=app.id,+ workflow_id=workflow.id,+ workflow_run_id=get_system_text(variable_pool, SystemVariableKey.WORKFLOW_EXECUTION_ID),+ node_id=node_id,+ node_execution_id=execution_id,+ )+ assert execution.status == WorkflowNodeExecutionStatus.RUNNING+ assert execution.node_execution_id == execution_id+ assert execution.triggered_from == WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP+ assert execution.finished_at is None+ execution.agent_workspace_binding_id = workspace_binding_id+ execution.process_data = json.dumps({"workflow_agent_binding_id": workflow_binding_id})+ session.commit()++ result = NodeRunResult(+ status=status,+ outputs={"text": "done"} if status == WorkflowNodeExecutionStatus.SUCCEEDED else {},+ process_data={"step": "completed"},+ error="runtime failure" if status == WorkflowNodeExecutionStatus.FAILED else "",+ )+ event_kwargs = {+ "id": execution_id,+ "node_id": node_id,+ "node_type": BuiltinNodeTypes.AGENT,+ "node_run_result": result,+ "start_at": started_at,+ }+ if status == WorkflowNodeExecutionStatus.FAILED:+ yield NodeRunFailedEvent(**event_kwargs, error="runtime failure")+ else:+ yield NodeRunSucceededEvent(**event_kwargs)++ with (+ patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)),+ patch("services.workflow_service.WorkflowDraftVariableService"),+ patch("services.workflow_service.DraftVarLoader"),+ patch(+ "services.workflow_service.WorkflowEntry.single_step_run",+ side_effect=lambda **kwargs: (node, node_events(kwargs["variable_pool"])),+ ),+ patch(+ "services.workflow_service.DifyCoreRepositoryFactory.create_workflow_node_execution_repository",+ return_value=repository,+ ),+ patch("services.workflow_service.DraftVariableSaver"),+ patch("services.workflow_service.enqueue_draft_node_execution_trace"),+ patch(+ "core.repositories.celery_workflow_node_execution_repository.save_workflow_node_execution_task.delay"+ ) as async_save,+ ):+ result = service.run_draft_workflow_node(+ app_model=app, draft_workflow=workflow, node_id=node_id, user_inputs={}, account=account+ )++ assert result.id == execution_id+ assert result.node_execution_id == execution_id+ assert result.status == status+ assert result.created_at == started_at+ assert result.finished_at is not None+ if status == WorkflowNodeExecutionStatus.SUCCEEDED:+ assert result.outputs_dict == {"text": "done"}+ assert result.process_data_dict is not None+ assert result.process_data_dict["step"] == "completed"+ else:+ assert result.error == "runtime failure"+ with Session(sqlite_engine) as session:+ assert list(session.scalars(select(WorkflowNodeExecutionModel.id))) == [execution_id]+ persisted = session.get(WorkflowNodeExecutionModel, execution_id)+ assert persisted is not None+ assert persisted.status == status+ assert persisted.agent_workspace_binding_id == workspace_binding_id+ assert persisted.process_data_dict is not None+ assert persisted.process_data_dict["workflow_agent_binding_id"] == workflow_binding_id+ if repository_type is CeleryWorkflowNodeExecutionRepository:+ async_save.assert_called_once()+ assert async_save.call_args.kwargs["execution_data"]["status"] == status+ else:+ async_save.assert_not_called()+ if repository_type is LogstoreWorkflowNodeExecutionRepository:+ assert len(logstore_records) == 1+ assert logstore_records[0]["status"] == status+# ===========================================================================# TestWorkflowServiceHumanInputOperations@@ -3571,28 +3761,54 @@ class TestWorkflowServiceHumanInputOperations:assert result == []- def test_build_human_input_variable_pool(self, service: WorkflowService, sqlite_engine: Engine) -> None:- workflow = TestWorkflowAssociatedDataFactory.create_workflow()- node_data = MagicMock()- node_data.extract_variable_selector_to_variable_mapping.return_value = {}+ def test_form_preview_resolves_nested_inputs_without_loading_email_dependencies(+ self, service: WorkflowService, sqlite_engine: Engine, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ workflow = TestWorkflowAssociatedDataFactory.create_workflow(+ graph={+ "nodes": [+ {+ "id": "human",+ "data": {+ "type": "human-input",+ "title": "Human",+ "form_content": "Hello {{#source.object.name#}}",+ "inputs": [+ {+ "type": "paragraph",+ "output_variable_name": "comment",+ "default": {"type": "variable", "selector": ["source", "object", "default"]},+ }+ ],+ "delivery_methods": [+ {+ "type": "email",+ "enabled": True,+ "config": {+ "recipients": {"items": []},+ "subject": "Review",+ "body": "{{#email.missing#}} {{#url#}}",+ },+ }+ ],+ },+ }+ ],+ "edges": [],+ }+ )+ monkeypatch.setattr(service, "get_draft_workflow", lambda **_: workflow)+ monkeypatch.setattr(type(db), "engine", property(lambda _: sqlite_engine))+ result = service.get_human_input_form_preview(+ app_model=TestWorkflowAssociatedDataFactory.create_app(),+ account=TestWorkflowAssociatedDataFactory.create_account(),+ node_id="human",+ inputs={"#source.object.name#": "Alice", "#source.object.default#": "Approved"},+ session=sqlite_session,+ )- with (- patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)),- patch("services.workflow_service.WorkflowDraftVariableService"),- patch("services.workflow_service.VariablePool") as mock_pool_cls,- patch("services.workflow_service.DraftVarLoader"),- patch("services.workflow_service.HumanInputNodeData.model_validate", return_value=node_data),- patch("services.workflow_service.load_into_variable_pool"),- patch("services.workflow_service.WorkflowEntry.mapping_user_inputs_to_variable_pool"),- ):- service._build_human_input_variable_pool(- app_model=TestWorkflowAssociatedDataFactory.create_app(),- workflow=workflow,- node_config={"id": "node-1", "data": {}},- manual_inputs={},- user_id="user-1",- )- mock_pool_cls.assert_called_once()+ assert result["form_content"] == "Hello Alice"+ assert result["resolved_default_values"] == {"comment": "Approved"}# ===========================================================================diff --git a/api/tests/unit_tests/services/workflow/test_workflow_event_snapshot_service.py b/api/tests/unit_tests/services/workflow/test_workflow_event_snapshot_service.pyindex dca9ad866e2..d699d841b9e 100644--- a/api/tests/unit_tests/services/workflow/test_workflow_event_snapshot_service.py+++ b/api/tests/unit_tests/services/workflow/test_workflow_event_snapshot_service.py@@ -26,15 +26,19 @@ from core.workflow.nodes.human_input.entities import SelectInputConfig, StringLifrom core.workflow.nodes.human_input.enums import HumanInputFormStatus, ValueSourceTypefrom core.workflow.nodes.human_input.pause_reason import HumanInputRequiredfrom graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus-from graphon.runtime import GraphRuntimeState, VariablePool+from graphon.runtime import RuntimeState, VariablePoolfrom libs.broadcast_channel.exc import SubscriptionClosedErrorfrom libs.datetime_utils import to_utc_timestampfrom models.enums import ConversationFromSource, CreatorUserRolefrom models.human_input import HumanInputForm, HumanInputFormRecipient, RecipientTypefrom models.model import AppMode, Message-from models.workflow import WorkflowRun+from models.workflow import WorkflowPause, WorkflowRunfrom repositories.api_workflow_node_execution_repository import WorkflowNodeExecutionSnapshotfrom repositories.entities.workflow_pause import WorkflowPauseEntity+from repositories.sqlalchemy_api_workflow_node_execution_repository import (+ DifyAPISQLAlchemyWorkflowNodeExecutionRepository,+)+from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepositoryfrom services import workflow_event_snapshot_service as service_modulefrom services.workflow_event_snapshot_service import (BufferState,@@ -141,7 +145,7 @@ def _build_resumption_context(task_id: str, *, select_options: list[str] | Nonecall_depth=0,workflow_execution_id="run-1",)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)if select_options is not None:runtime_state.variable_pool.add(("start", "options"), select_options)runtime_state.set_output("result", "value")@@ -248,7 +252,7 @@ def _build_resumption_context_additional(task_id: str) -> WorkflowResumptionContcall_depth=0,workflow_execution_id="run-1",)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)runtime_state.set_output("answer", "ok")wrapper = _WorkflowGenerateEntityWrapper(entity=generate_entity)return WorkflowResumptionContext(@@ -277,7 +281,7 @@ def _build_advanced_chat_resumption_context(conversation_id: str | None) -> Workworkflow_run_id="run-1",query="hello",)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)+ runtime_state = RuntimeState(workflow_id="test-workflow", variable_pool=VariablePool(), start_at=0.0)wrapper = _AdvancedChatAppGenerateEntityWrapper(entity=generate_entity)return WorkflowResumptionContext(generate_entity=wrapper,@@ -830,7 +834,6 @@ def test_build_advanced_chat_snapshot_requires_conversation_context(create_api_workflow_node_execution_repository=MagicMock(return_value=node_repo),)monkeypatch.setattr(service_module, "DifyAPIRepositoryFactory", factory)- monkeypatch.setattr(service_module.MessageGenerator, "get_response_topic", MagicMock())monkeypatch.setattr(service_module, "_load_resumption_context", MagicMock(return_value=resumption_context))conversation_lookup = MagicMock(return_value=None)app_lookup = MagicMock(return_value=None)@@ -867,7 +870,6 @@ def test_build_non_suspended_advanced_chat_snapshot_uses_app_scoped_fallback(create_api_workflow_node_execution_repository=MagicMock(return_value=node_repo),)monkeypatch.setattr(service_module, "DifyAPIRepositoryFactory", factory)- monkeypatch.setattr(service_module.MessageGenerator, "get_response_topic", MagicMock())load_resumption_context = MagicMock(return_value=None)monkeypatch.setattr(service_module, "_load_resumption_context", load_resumption_context)conversation_lookup = MagicMock(return_value=None)@@ -1457,3 +1459,60 @@ def test_stream_continues_with_live_events_when_all_forms_submitted(snapshot_types = [event["event"] for event in events if isinstance(event, Mapping)]assert StreamEvent.WORKFLOW_PAUSED not in snapshot_typesassert StreamEvent.HUMAN_INPUT_REQUIRED not in snapshot_types++[email protected]("saved_run_id", ["run-1", "other-run"])[email protected]("app_mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT])+def test_authorized_snapshot_indexes_legacy_pause_for_stop(+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ saved_run_id: str,+ app_mode: AppMode,+) -> None:+ task_id = "ccbd2ed3-013f-4de1-82b5-a38356203f8f"+ run = _build_workflow_run(WorkflowExecutionStatus.PAUSED)+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="legacy-state")+ sqlite_session.add_all([run, pause])+ sqlite_session.commit()+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ assert (+ repository.stop_paused_workflow_task(tenant_id=run.tenant_id, app_id=run.app_id, task_id=task_id, owner=None)+ is None+ )+ context = (+ _build_resumption_context(task_id)+ if app_mode == AppMode.WORKFLOW+ else _build_advanced_chat_resumption_context("conv-1")+ )+ generate_entity = context.get_generate_entity()+ generate_entity.task_id = task_id+ if isinstance(generate_entity, AdvancedChatAppGenerateEntity):+ generate_entity.workflow_run_id = saved_run_id+ else:+ generate_entity.workflow_execution_id = saved_run_id+ monkeypatch.setattr(service_module, "_load_resumption_context", lambda _: context)+ monkeypatch.setattr(+ service_module.DifyAPIRepositoryFactory, "create_api_workflow_run_repository", lambda _: repository+ )+ node_repository = DifyAPISQLAlchemyWorkflowNodeExecutionRepository(sqlite_session_factory)+ monkeypatch.setattr(+ service_module.DifyAPIRepositoryFactory,+ "create_api_workflow_node_execution_repository",+ lambda _: node_repository,+ )+ stream = build_workflow_event_stream(+ app_mode=app_mode,+ workflow_run=run,+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ session_maker=sqlite_session_factory,+ )+ stream.close()+ sqlite_session.expire_all()+ assert run.task_id == (task_id if saved_run_id == run.id else None)+ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.delete", lambda _: None)+ stopped = repository.stop_paused_workflow_task(+ tenant_id=run.tenant_id, app_id=run.app_id, task_id=task_id, owner=None+ )+ assert (stopped is not None) is (saved_run_id == run.id)diff --git a/api/tests/unit_tests/services/workflow/test_workflow_run_agg.py b/api/tests/unit_tests/services/workflow/test_workflow_run_agg.pynew file mode 100644index 00000000000..bf13b96f284--- /dev/null+++ b/api/tests/unit_tests/services/workflow/test_workflow_run_agg.py@@ -0,0 +1,604 @@+"""Workflow orchestration against real Engine, SQLAlchemy, and filesystem storage."""++import json+from datetime import UTC, datetime, timedelta+from pathlib import Path+from uuid import uuid4++import pytest+from sqlalchemy import Engine, select+from sqlalchemy.orm import Session++from core.app.apps.workflow.app_config_manager import WorkflowAppConfig+from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager+from core.app.apps.workflow.app_runner import WorkflowAppRunner+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity+from core.app.entities.queue_entities import (+ QueueNodeStartedEvent,+ QueueWorkflowFailedEvent,+ QueueWorkflowPausedEvent,+ QueueWorkflowSucceededEvent,+)+from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext+from core.app.layers.trigger_post_layer import TriggerPostLayer+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from core.workflow.nodes.human_input.entities import HumanInputNodeData, ParagraphInputConfig, UserActionConfig+from extensions.ext_storage import storage+from extensions.storage.opendal_storage import OpenDALStorage+from graphon.engine_events import (+ GraphRunAbortedEvent,+ GraphRunFailedEvent,+ GraphRunPartialSucceededEvent,+ GraphRunPausedEvent,+ GraphRunStartedEvent,+ GraphRunSucceededEvent,+)+from graphon.enums import WorkflowExecutionStatus+from graphon.nodes.start.entities import StartNodeData+from graphon.runtime import RuntimeState+from graphon.variable_loader import DUMMY_VARIABLE_LOADER+from models import Account+from models.enums import AppTriggerType, CreatorUserRole, WorkflowRunTriggeredFrom, WorkflowTriggerStatus+from models.model import AppMode+from models.trigger import WorkflowTriggerLog+from models.workflow import (+ Workflow,+ WorkflowNodeExecutionModel,+ WorkflowNodeExecutionTriggeredFrom,+ WorkflowPause,+ WorkflowRun,+)+from services.workflow.entities import WorkflowScheduleCFSPlanEntity+from services.workflow_run_agg import WorkflowRunAgg+from tasks.workflow_cfs_scheduler.cfs_scheduler import AsyncWorkflowCFSPlanEntity+from tasks.workflow_cfs_scheduler.entities import AsyncWorkflowQueue+++def make_workflow_runner(+ sqlite_engine: Engine, *, human_input: bool = False+) -> tuple[WorkflowAppRunner, SQLAlchemyWorkflowExecutionRepository, SQLAlchemyWorkflowNodeExecutionRepository]:+ """Build the same runner/repositories/queue used by a workflow worker."""+ tenant_id, app_id, workflow_id, user_id, run_id = (str(uuid4()) for _ in range(5))+ graph: dict[str, list[dict[str, object]]] = {+ "nodes": [{"id": "start", "data": {"type": "start", **StartNodeData(title="Start").model_dump(mode="json")}}],+ "edges": [],+ }+ if human_input:+ data = HumanInputNodeData(+ title="Approval",+ form_content="Decision: {{#$output.answer#}}",+ inputs=[ParagraphInputConfig(output_variable_name="answer")],+ user_actions=[UserActionConfig(id="approve", title="Approve")],+ )+ graph["nodes"].append({"id": "approval", "data": {"type": "human-input", **data.model_dump(mode="json")}})+ graph["edges"].append({"id": "start-approval", "source": "start", "target": "approval"})+ graph["nodes"].append(+ {+ "id": "end",+ "data": {+ "type": "end",+ "title": "End",+ "outputs": [+ {"variable": "answer", "value_selector": ["approval", "answer"]},+ ],+ },+ }+ )+ graph["edges"].append({"id": "approval-end", "source": "approval", "target": "end", "sourceHandle": "approve"})+ workflow = Workflow(+ id=workflow_id,+ tenant_id=tenant_id,+ app_id=app_id,+ type="workflow",+ version="draft",+ graph=json.dumps(graph),+ features="{}",+ created_by=user_id,+ )+ entity = WorkflowAppGenerateEntity(+ task_id=str(uuid4()),+ app_config=WorkflowAppConfig(+ tenant_id=tenant_id, app_id=app_id, workflow_id=workflow_id, app_mode=AppMode.WORKFLOW+ ),+ inputs={},+ files=[],+ user_id=user_id,+ stream=True,+ invoke_from=InvokeFrom.DEBUGGER,+ workflow_execution_id=run_id,+ )+ user = Account(name="Workflow Tester", email=f"{user_id}@example.com")+ user.id = user_id+ queue = WorkflowAppQueueManager(entity.task_id, user_id, entity.invoke_from, AppMode.WORKFLOW)+ workflow_repository = SQLAlchemyWorkflowExecutionRepository(+ sqlite_engine,+ tenant_id=tenant_id,+ user=user,+ app_id=app_id,+ triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,+ )+ node_repository = SQLAlchemyWorkflowNodeExecutionRepository(+ sqlite_engine,+ tenant_id=tenant_id,+ user=user,+ app_id=app_id,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ )+ runner = WorkflowAppRunner(+ application_generate_entity=entity,+ queue_manager=queue,+ variable_loader=DUMMY_VARIABLE_LOADER,+ workflow=workflow,+ system_user_id=user_id,+ execution_driver=WorkflowRunAgg.run,+ )+ return runner, workflow_repository, node_repository+++def test_driver_publishes_persisted_start_index_and_success(sqlite_engine: Engine) -> None:+ runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine)+ WorkflowRunAgg.run(runner, None, workflow_repository, node_repository)+ events = [message.event for message in runner._queue_manager.listen()]+ starts = [event for event in events if isinstance(event, QueueNodeStartedEvent)]+ assert len(starts) == 1+ assert isinstance(events[-1], QueueWorkflowSucceededEvent)+ with Session(sqlite_engine) as session:+ execution = session.scalar(select(WorkflowNodeExecutionModel))+ assert execution is not None+ assert execution.index == starts[0].node_run_index == 1+ run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)+ assert run is not None+ assert run.status == "succeeded"+++def _local_storage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(storage, "storage_runner", OpenDALStorage(scheme="fs", root=str(tmp_path)), raising=False)+ from core.app.apps import workflow_app_runner++ monkeypatch.setattr(workflow_app_runner.dispatch_human_input_email_task, "apply_async", lambda **_kwargs: None)++[email protected]("_local_storage")[email protected]("persist_snapshot", [False, True])+def test_pause_publication_persists_summary_and_optional_snapshot(+ sqlite_engine: Engine, persist_snapshot: bool, caplog: pytest.LogCaptureFixture+) -> None:+ runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)+ config = (+ PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id) if persist_snapshot else None+ )+ WorkflowRunAgg.run(runner, config, workflow_repository, node_repository)+ assert "Failed to enqueue human input email task" not in caplog.text+ events = [message.event for message in runner._queue_manager.listen()]+ assert isinstance(events[-1], QueueWorkflowPausedEvent), events+ with Session(sqlite_engine) as session:+ run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)+ assert run is not None+ pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run.id))+ assert run.status == "paused"+ assert run.finished_at is None+ assert run.total_steps == 2+ if not persist_snapshot:+ assert pause is None+ return+ assert pause is not None+ snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())+ restored = RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state)+ assert restored.graph_execution.paused+ assert snapshot.get_response_stream_filter().dumps() == snapshot.serialized_response_stream_filter_state+ assert snapshot.get_generate_entity().model_dump(mode="json") == runner.application_generate_entity.model_dump(+ mode="json"+ )++[email protected]("_local_storage")+def test_pause_database_failure_publishes_failure_instead_of_resume_ready(+ sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch+) -> None:+ def fail_pause(*_args: object, **_kwargs: object) -> None:+ raise RuntimeError("pause-write-failed")++ monkeypatch.setattr(+ "repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository.create_workflow_pause",+ fail_pause,+ )+ retirements: list[dict[str, object]] = []+ collections: list[dict[str, object]] = []++ def retire(_store: object, **kwargs: object) -> list[str]:+ retirements.append(kwargs)+ return ["workspace-1"]++ monkeypatch.setattr(+ "core.workflow.nodes.agent_v2.workspace_retirement_layer.WorkflowAgentWorkspaceStore.retire_workflow_run",+ retire,+ )+ monkeypatch.setattr(+ "core.workflow.nodes.agent_v2.workspace_retirement_layer.enqueue_agent_resource_collection",+ lambda **kwargs: collections.append(kwargs),+ )+ runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)+ trigger_log = WorkflowTriggerLog(+ tenant_id=runner.application_generate_entity.app_config.tenant_id,+ app_id=runner.application_generate_entity.app_config.app_id,+ workflow_id=runner._workflow.id,+ workflow_run_id=None,+ root_node_id=None,+ trigger_metadata="{}",+ trigger_type=AppTriggerType.TRIGGER_SCHEDULE,+ trigger_data="{}",+ inputs="{}",+ outputs=None,+ status=WorkflowTriggerStatus.RUNNING,+ error=None,+ queue_name="workflow",+ celery_task_id=None,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=runner.application_generate_entity.user_id,+ elapsed_time=5.0,+ )+ with Session(sqlite_engine, expire_on_commit=False) as session:+ session.add(trigger_log)+ session.commit()+ start_time = datetime.now(UTC) - timedelta(seconds=10)+ plan = AsyncWorkflowCFSPlanEntity(+ schedule_strategy=WorkflowScheduleCFSPlanEntity.Strategy.Nop,+ queue=AsyncWorkflowQueue.PROFESSIONAL_QUEUE,+ )+ runner._graph_engine_layers = (TriggerPostLayer(plan, start_time, trigger_log.id),)+ WorkflowRunAgg.run(+ runner,+ PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id),+ workflow_repository,+ node_repository,+ )+ events = [message.event for message in runner._queue_manager.listen()]+ assert isinstance(events[-1], QueueWorkflowFailedEvent), events+ assert "pause-write-failed" in events[-1].error+ assert not any(isinstance(event, QueueWorkflowPausedEvent) for event in events)+ assert retirements == [+ {+ "tenant_id": runner.application_generate_entity.app_config.tenant_id,+ "app_id": runner.application_generate_entity.app_config.app_id,+ "workflow_run_id": runner.application_generate_entity.workflow_execution_id,+ }+ ]+ assert collections == [+ {+ "tenant_id": runner.application_generate_entity.app_config.tenant_id,+ "workspace_ids": ["workspace-1"],+ }+ ]+ with Session(sqlite_engine) as session:+ persisted_trigger = session.get(WorkflowTriggerLog, trigger_log.id)+ assert persisted_trigger is not None+ assert persisted_trigger.status == WorkflowTriggerStatus.FAILED+ assert persisted_trigger.finished_at is not None+ assert persisted_trigger.elapsed_time == pytest.approx(+ 5.0 + (persisted_trigger.finished_at.replace(tzinfo=UTC) - start_time).total_seconds()+ )+ run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)+ assert run is not None+ assert run.status == "failed"+ assert session.scalar(select(WorkflowPause)) is None+++def test_late_form_submission_is_refreshed_before_the_real_queue_closes(sqlite_session: Session) -> None:+ from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner+ from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent+ from core.repositories.human_input_repository import HumanInputFormSubmissionRepository+ from core.workflow.nodes.human_input.enums import HumanInputFormStatus+ from graphon.engine_events import NodeRunStartedEvent+ from tests.unit_tests.core.app.apps.test_workflow_human_input_completion import (+ _make_paused_workflow,+ _resume_events,+ _save_form,+ )++ first = _save_form(sqlite_session, status=HumanInputFormStatus.SUBMITTED)+ late = _save_form(sqlite_session, status=HumanInputFormStatus.WAITING)+ queue = WorkflowAppQueueManager("form-refresh-task", "user", InvokeFrom.DEBUGGER, AppMode.WORKFLOW)+ runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")+ entry = _make_paused_workflow(runner, [first, late])+ for event in _resume_events(runner, entry):+ runner.handle_event(entry, event)+ if isinstance(event, NodeRunStartedEvent):+ HumanInputFormSubmissionRepository().mark_submitted(+ form_id=late.id,+ recipient_id=None,+ selected_action_id="approve",+ form_data={"answer": "late answer"},+ submission_user_id=None,+ submission_end_user_id=None,+ )+ events = [message.event for message in queue.listen()]+ completions = [event for event in events if isinstance(event, QueueHumanInputFormFilledEvent)]+ assert [event.form_id for event in completions] == [first.id, late.id]+ assert completions[-1].rendered_content == "Decision: late answer"+ assert isinstance(events[-1], QueueWorkflowSucceededEvent)++[email protected]("_local_storage")+def test_durable_pause_resumes_with_one_form_completion_and_continuing_indexes(sqlite_engine: Engine) -> None:+ from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent, QueueWorkflowStartedEvent+ from core.repositories.human_input_repository import HumanInputFormSubmissionRepository+ from models.human_input import HumanInputForm++ original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)+ config = PauseStateLayerConfig(sqlite_engine, original.application_generate_entity.user_id)+ WorkflowRunAgg.run(original, config, workflow_repository, node_repository)+ before = [message.event for message in original._queue_manager.listen()]+ assert isinstance(before[-1], QueueWorkflowPausedEvent)+ with Session(sqlite_engine) as session:+ pause = session.scalar(select(WorkflowPause))+ form = session.scalar(select(HumanInputForm))+ assert pause is not None+ assert form is not None+ form_id = form.id+ snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())+ HumanInputFormSubmissionRepository().mark_submitted(+ form_id=form_id,+ recipient_id=None,+ selected_action_id="approve",+ form_data={"answer": "approved result"},+ submission_user_id=None,+ submission_end_user_id=None,+ )+ entity = original.application_generate_entity.model_copy(update={"task_id": str(uuid4())})+ queue = WorkflowAppQueueManager(entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW)+ resumed = WorkflowAppRunner(+ application_generate_entity=entity,+ queue_manager=queue,+ variable_loader=DUMMY_VARIABLE_LOADER,+ workflow=original._workflow,+ system_user_id=entity.user_id,+ graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),+ response_stream_filter=snapshot.get_response_stream_filter(),+ execution_driver=WorkflowRunAgg.run,+ )+ WorkflowRunAgg.run(resumed, config, workflow_repository, node_repository)+ after = [message.event for message in queue.listen()]+ assert isinstance(after[0], QueueWorkflowStartedEvent)+ assert len(after[0].node_execution_snapshots) == 2+ completions = [event for event in after if isinstance(event, QueueHumanInputFormFilledEvent)]+ assert [event.form_id for event in completions] == [form_id]+ starts = [event for event in after if isinstance(event, QueueNodeStartedEvent)]+ assert [(event.node_id, event.node_run_index) for event in starts] == [("approval", 2), ("end", 3)]+ assert isinstance(after[-1], QueueWorkflowSucceededEvent)+ assert after[-1].outputs == {"answer": "approved result"}+ with Session(sqlite_engine) as session:+ executions = session.scalars(+ select(WorkflowNodeExecutionModel).order_by(WorkflowNodeExecutionModel.index)+ ).all()+ assert [(row.node_id, row.index) for row in executions] == [("start", 1), ("approval", 2), ("end", 3)]+++def test_active_execution_snapshot_failure_is_not_published_as_a_pause(sqlite_engine: Engine) -> None:+ from graphon.engine_events import GraphRunFailedEvent, GraphRunPausedEvent++ runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)+ prepared = runner.prepare()+ assert prepared is not None+ aggregate = WorkflowRunAgg(+ prepared,+ runner,+ PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id),+ workflow_execution_repository=workflow_repository,+ workflow_node_execution_repository=node_repository,+ )+ # A participant that has not shut down must prevent a resume-ready snapshot.+ with prepared.entry.graph_engine.runtime_state.graph_execution.track_execution():+ events = list(aggregate.iter_events())+ assert isinstance(events[-1], GraphRunFailedEvent)+ assert "during active execution" in events[-1].error+ assert not any(isinstance(event, GraphRunPausedEvent) for event in events)+ with Session(sqlite_engine) as session:+ run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)+ assert run is not None+ assert run.status == "failed"+ assert session.scalar(select(WorkflowPause)) is None++[email protected]("_local_storage")+ ("terminal_event", "status"),+ [+ (GraphRunSucceededEvent(), WorkflowExecutionStatus.SUCCEEDED),+ (GraphRunPartialSucceededEvent(exceptions_count=1), WorkflowExecutionStatus.PARTIAL_SUCCEEDED),+ (GraphRunFailedEvent(error="failed"), WorkflowExecutionStatus.FAILED),+ (GraphRunAbortedEvent(reason="stopped"), WorkflowExecutionStatus.STOPPED),+ ],+)+def test_elapsed_time_accumulates_across_pauses_and_matches_delayed_stream_responses(+ monkeypatch: pytest.MonkeyPatch,+ sqlite_engine: Engine,+ terminal_event: GraphRunSucceededEvent | GraphRunPartialSucceededEvent | GraphRunFailedEvent | GraphRunAbortedEvent,+ status: WorkflowExecutionStatus,+) -> None:+ from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter+ from core.app.entities.task_entities import WorkflowPauseStreamResponse+ from core.repositories.human_input_repository import HumanInputFormSubmissionRepository+ from core.workflow.nodes.human_input.enums import HumanInputFormStatus+ from core.workflow.system_variables import build_system_variables+ from graphon.entities import WorkflowStartReason+ from models.human_input import HumanInputForm++ clock = [100.0]+ monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])+ original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)+ graph = original._workflow.graph_dict+ graph["nodes"].append({"id": "second-approval", "data": graph["nodes"][1]["data"]})+ graph["edges"][1]["target"] = "second-approval"+ graph["edges"].append({"id": "second-end", "source": "second-approval", "target": "end", "sourceHandle": "approve"})+ original._workflow.graph = json.dumps(graph)+ runner = original+ run_id = runner.application_generate_entity.workflow_execution_id+ config = PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id)+ account = Account(name="Test", email="[email protected]")+ account.id = runner.application_generate_entity.user_id+ total = 0.0++ for index, duration in enumerate([2.0, 3.0, 4.0]):+ prepared = runner.prepare()+ assert prepared is not None+ aggregate = WorkflowRunAgg(+ prepared,+ runner,+ config,+ workflow_execution_repository=workflow_repository,+ workflow_node_execution_repository=node_repository,+ )+ is_paused = index < 2+ if is_paused:+ events = []+ for event in aggregate.iter_events():+ if isinstance(event, GraphRunStartedEvent):+ with Session(sqlite_engine) as session:+ started = session.get(WorkflowRun, run_id)+ assert started is not None+ assert started.elapsed_time == total+ clock[0] += duration+ events.append(event)+ assert isinstance(events[-1], GraphRunPausedEvent), events+ else:+ aggregate._persistence_layer.on_event(GraphRunStartedEvent())+ clock[0] += duration+ aggregate._persistence_layer.on_event(terminal_event)+ total += duration+ with Session(sqlite_engine) as session:+ execution = session.get(WorkflowRun, run_id)+ assert execution is not None+ assert execution.elapsed_time == total+ assert execution.status == (WorkflowExecutionStatus.PAUSED if is_paused else status)+ assert (execution.finished_at is None) == is_paused++ # Queue processing and pause waits must not extend a settled segment.+ clock[0] += 1000+ entity = runner.application_generate_entity+ runtime = prepared.entry.graph_engine.runtime_state+ converter = WorkflowResponseConverter(+ application_generate_entity=entity,+ user=account,+ system_variables=build_system_variables(workflow_execution_id=run_id),+ )+ converter.workflow_start_to_stream_response(+ task_id=entity.task_id,+ workflow_run_id=run_id,+ workflow_id=original._workflow.id,+ reason=WorkflowStartReason.INITIAL if index == 0 else WorkflowStartReason.RESUMPTION,+ )+ if is_paused:+ responses = converter.workflow_pause_to_stream_response(+ event=QueueWorkflowPausedEvent(), task_id=entity.task_id, graph_runtime_state=runtime+ )+ pause_response = responses[-1]+ assert isinstance(pause_response, WorkflowPauseStreamResponse)+ assert pause_response.data.elapsed_time == total+ with Session(sqlite_engine) as session:+ pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run_id))+ form = session.scalar(+ select(HumanInputForm).where(+ HumanInputForm.workflow_run_id == run_id, HumanInputForm.status == HumanInputFormStatus.WAITING+ )+ )+ assert pause is not None+ assert form is not None+ form_id = form.id+ snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())+ entity = snapshot.get_generate_entity()+ assert isinstance(entity, WorkflowAppGenerateEntity)+ assert entity.workflow_execution_timing.elapsed_time == snapshot.elapsed_time == total+ HumanInputFormSubmissionRepository().mark_submitted(+ form_id=form_id,+ recipient_id=None,+ selected_action_id="approve",+ form_data={"answer": "approved result"},+ submission_user_id=None,+ submission_end_user_id=None,+ )+ entity.task_id = str(uuid4())+ runner = WorkflowAppRunner(+ application_generate_entity=entity,+ queue_manager=WorkflowAppQueueManager(+ entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW+ ),+ variable_loader=DUMMY_VARIABLE_LOADER,+ workflow=original._workflow,+ system_user_id=entity.user_id,+ graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),+ response_stream_filter=snapshot.get_response_stream_filter(),+ execution_driver=WorkflowRunAgg.run,+ )+ else:+ response = converter.workflow_finish_to_stream_response(+ task_id=entity.task_id, workflow_id=original._workflow.id, status=status, graph_runtime_state=runtime+ )+ assert response.data.elapsed_time == 9.0+ assert entity.workflow_execution_timing.finish() == 9.0++[email protected]("_local_storage")[email protected]("stop_timing", ["before_summary", "during_upload"])+def test_stop_wins_while_engine_is_persisting_a_pause(+ sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch, stop_timing: str+) -> None:+ from sqlalchemy.orm import sessionmaker++ from core.app.entities.queue_entities import QueueStopEvent+ from core.workflow.nodes.human_input.enums import HumanInputFormStatus+ from graphon.enums import WorkflowExecutionStatus+ from models.human_input import HumanInputForm+ from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository++ runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)+ entity = runner.application_generate_entity+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sessionmaker(sqlite_engine))+ stop_calls = []++ def request_stop() -> None:+ admitted = repository.stop_paused_workflow_task(+ tenant_id=entity.app_config.tenant_id,+ app_id=entity.app_config.app_id,+ task_id=entity.task_id,+ owner=None,+ )+ assert admitted is not None+ stop_calls.append(admitted.status)++ if stop_timing == "during_upload":+ original_save = storage.save++ def save(key: str, data: bytes) -> None:+ if key.startswith("workflow-state-"):+ request_stop()+ return original_save(key, data)++ monkeypatch.setattr(storage, "save", save)+ else:+ original_persist = WorkflowRunAgg._persist_pause++ def persist(aggregate: WorkflowRunAgg, event: GraphRunPausedEvent) -> None:+ request_stop()+ return original_persist(aggregate, event)++ monkeypatch.setattr(WorkflowRunAgg, "_persist_pause", persist)++ WorkflowRunAgg.run(+ runner, PauseStateLayerConfig(sqlite_engine, entity.user_id), workflow_repository, node_repository+ )+ events = [message.event for message in runner._queue_manager.listen()]+ assert stop_calls+ assert isinstance(events[-1], QueueStopEvent), events+ assert not any(isinstance(event, QueueWorkflowPausedEvent) for event in events)+ with Session(sqlite_engine) as session:+ run = session.get(WorkflowRun, entity.workflow_execution_id)+ assert run is not None+ assert run.status == WorkflowExecutionStatus.STOPPED+ assert run.stop_requested_at is not None+ assert run.task_id == entity.task_id+ assert session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run.id)) is None+ forms = session.scalars(select(HumanInputForm).where(HumanInputForm.workflow_run_id == run.id)).all()+ assert forms+ assert all(form.status == HumanInputFormStatus.EXPIRED for form in forms)diff --git a/api/tests/unit_tests/services/workflow/test_workflow_run_index.py b/api/tests/unit_tests/services/workflow/test_workflow_run_index.pynew file mode 100644index 00000000000..bec984d4129--- /dev/null+++ b/api/tests/unit_tests/services/workflow/test_workflow_run_index.py@@ -0,0 +1,116 @@+from datetime import datetime++from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.app.app_config.entities import WorkflowUIBasedAppConfig+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from core.workflow.node_factory import DifyNodeFactory+from core.workflow.system_variables import build_system_variables+from graphon.engine import Engine+from graphon.engine_events import NodeRunStartedEvent+from graphon.entities import WorkflowNodeExecution+from graphon.enums import BuiltinNodeTypes, WorkflowType+from graphon.graph import Graph+from graphon.runtime import RuntimeState, VariablePool+from models import Account, AppMode+from models.enums import WorkflowRunTriggeredFrom+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom+from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer+from services.workflow_run_index import WorkflowRunIndex+from tests.workflow_test_utils import build_test_graph_init_params+++def _history(execution_id: str, index: int) -> WorkflowNodeExecution:+ return WorkflowNodeExecution(+ id=execution_id,+ node_execution_id=execution_id,+ workflow_id="workflow",+ workflow_execution_id="run",+ node_id="start",+ node_type=BuiltinNodeTypes.START,+ title="Start",+ index=index,+ created_at=datetime(2026, 9, 7),+ )+++def _engine() -> Engine:+ graph_config: dict[str, object] = {+ "nodes": [+ {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}},+ {"id": "end", "data": {"type": "end", "title": "End", "outputs": []}},+ ],+ "edges": [{"source": "start", "target": "end", "sourceHandle": "source"}],+ }+ state = RuntimeState(+ workflow_id="workflow",+ variable_pool=VariablePool.from_bootstrap(system_variables=build_system_variables(workflow_execution_id="run")),+ start_at=0.0,+ )+ params = build_test_graph_init_params(workflow_id="workflow", graph_config=graph_config)+ return Engine(+ graph=Graph.init(+ graph_config=graph_config,+ node_factory=DifyNodeFactory(init_params=params, runtime_state=state),+ root_node_id="start",+ ),+ runtime_state=state,+ workers=2,+ )+++def test_engine_indices_are_available_before_delivery_and_match_persistence(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ user = Account(name="Tester", email="[email protected]")+ user.id = "user"+ execution_repo = SQLAlchemyWorkflowExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant",+ user=user,+ app_id="app",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ )+ node_repo = SQLAlchemyWorkflowNodeExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant",+ user=user,+ app_id="app",+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ )+ history = _history("earlier", 7)+ node_repo.save(history)+ persisted_history = node_repo.get_by_workflow_execution("run")+ index = WorkflowRunIndex(root_executions=persisted_history)+ engine = _engine()+ layer = WorkflowPersistenceLayer(+ application_generate_entity=WorkflowAppGenerateEntity(+ task_id="task",+ app_config=WorkflowUIBasedAppConfig(+ tenant_id="tenant", app_id="app", app_mode=AppMode.WORKFLOW, workflow_id="workflow"+ ),+ inputs={},+ files=[],+ user_id="user",+ stream=True,+ invoke_from=InvokeFrom.DEBUGGER,+ workflow_execution_id="run",+ ),+ workflow_info=PersistenceWorkflowInfo(+ workflow_id="workflow", workflow_type=WorkflowType.WORKFLOW, version="1", graph_data={}+ ),+ workflow_execution_repository=execution_repo,+ workflow_node_execution_repository=node_repo,+ )+ layer.set_node_run_indices(index.indices)+ layer.set_node_execution_history(persisted_history)+ engine.add_layer(index)+ engine.add_layer(layer)+ started = {event.id: index.index_for(event.id) for event in engine.run() if isinstance(event, NodeRunStartedEvent)}+ assert sorted(started.values()) == [8, 9]+ with sqlite_session_factory() as session:+ stored = {row.id: row.index for row in session.scalars(select(WorkflowNodeExecutionModel))}+ assert stored == {"earlier": 7, **started}diff --git a/api/tests/unit_tests/tasks/test_workflow_execute_task.py b/api/tests/unit_tests/tasks/test_workflow_execute_task.pyindex 2dca4233c78..1e11b3eaec6 100644--- a/api/tests/unit_tests/tasks/test_workflow_execute_task.py+++ b/api/tests/unit_tests/tasks/test_workflow_execute_task.py@@ -3,26 +3,40 @@ from __future__ import annotationsimport jsonimport loggingimport uuid-from collections.abc import Generator, Mapping+from collections.abc import Callable, Generator, Mappingfrom contextlib import nullcontextfrom datetime import datetimefrom decimal import Decimalfrom types import SimpleNamespace-from unittest.mock import MagicMock+from unittest.mock import ANY, MagicMockimport pytestfrom pydantic import BaseModelfrom sqlalchemy import Enginefrom sqlalchemy.orm import Session, sessionmaker+from core.app.app_config.entities import WorkflowUIBasedAppConfig+from core.app.apps.workflow.app_generator import WorkflowAppGeneratorfrom core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom, WorkflowAppGenerateEntity+from core.app.layers.pause_state_persist_layer import (+ WorkflowResumptionContext,+ _AdvancedChatAppGenerateEntityWrapper,+ _WorkflowGenerateEntityWrapper,+)from graphon.entities import WorkflowStartReasonfrom graphon.enums import WorkflowExecutionStatus+from graphon.runtime import RuntimeState, VariablePoolfrom models.account import Accountfrom models.enums import ConversationFromSource, CreatorUserRole, WorkflowRunTriggeredFromfrom models.model import App, AppMode, Conversation, Message-from models.workflow import Workflow, WorkflowRun, WorkflowType-from repositories.sqlalchemy_api_workflow_run_repository import _WorkflowRunError+from models.workflow import Workflow, WorkflowPause, WorkflowRun, WorkflowType+from repositories.entities.workflow_pause import WorkflowPauseEntity+from repositories.sqlalchemy_api_workflow_run_repository import (+ DifyAPISQLAlchemyWorkflowRunRepository,+ WorkflowRunNotPausedError,+ _WorkflowRunError,+)+from tasks import async_workflow_tasksfrom tasks.app_generate import workflow_execute_task as workflow_execute_task_modulefrom tasks.app_generate.workflow_execute_task import (AppExecutionParams,@@ -216,6 +230,7 @@ def _persist_resumption_models()workflow_run = WorkflowRun(id=workflow_run_id,+ task_id="task-id",tenant_id="tenant-id",app_id=app.id,workflow_id=workflow.id,@@ -858,7 +873,7 @@ def test_resume_app_execution_queries_message_by_conversation_and_workflow_run(lambda *_args, **_kwargs: resumption_context,)monkeypatch.setattr(- "tasks.app_generate.workflow_execute_task.GraphRuntimeState.from_snapshot",+ "tasks.app_generate.workflow_execute_task.RuntimeState.from_snapshot",lambda *_args, **_kwargs: MagicMock(),)@@ -871,7 +886,7 @@ def test_resume_app_execution_queries_message_by_conversation_and_workflow_run(_resume_app_execution({"workflow_run_id": workflow_run_id})- workflow_run_repo.resume_workflow_pause.assert_called_once_with(workflow_run_id, pause_entity)+ workflow_run_repo.resume_workflow_pause.assert_called_once_with(workflow_run_id, pause_entity, before_resume=ANY)resume_advanced_chat.assert_called_once()assert resume_advanced_chat.call_args.kwargs["conversation"].id == conversation_idassert resume_advanced_chat.call_args.kwargs["message"].id == "expected-message-id"@@ -906,7 +921,7 @@ def test_resume_app_execution_returns_early_when_advanced_chat_missing_conversatlambda *_args, **_kwargs: resumption_context,)monkeypatch.setattr(- "tasks.app_generate.workflow_execute_task.GraphRuntimeState.from_snapshot",+ "tasks.app_generate.workflow_execute_task.RuntimeState.from_snapshot",lambda *_args, **_kwargs: MagicMock(),)@@ -957,7 +972,7 @@ def test_resume_app_execution_clears_stale_cancellation_signals_before_resuming(lambda *_args, **_kwargs: resumption_context,)monkeypatch.setattr(- "tasks.app_generate.workflow_execute_task.GraphRuntimeState.from_snapshot",+ "tasks.app_generate.workflow_execute_task.RuntimeState.from_snapshot",lambda *_args, **_kwargs: MagicMock(),)monkeypatch.setattr(@@ -966,6 +981,7 @@ def test_resume_app_execution_clears_stale_cancellation_signals_before_resuming(calls: list[str] = []clear_signals = MagicMock(side_effect=lambda task_id: calls.append(f"clear:{task_id}"))+ workflow_run_repo.resume_workflow_pause.side_effect = lambda *_, before_resume: before_resume()resume_workflow = MagicMock(side_effect=lambda **_kwargs: calls.append("resume"))monkeypatch.setattr("tasks.app_generate.workflow_execute_task.clear_app_task_cancellation_signals", clear_signals)monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_workflow", resume_workflow)@@ -1008,7 +1024,7 @@ def test_resume_app_execution_keeps_cancellation_signals_when_resume_is_abandonelambda *_args, **_kwargs: resumption_context,)monkeypatch.setattr(- "tasks.app_generate.workflow_execute_task.GraphRuntimeState.from_snapshot",+ "tasks.app_generate.workflow_execute_task.RuntimeState.from_snapshot",lambda *_args, **_kwargs: MagicMock(),)monkeypatch.setattr(@@ -1024,6 +1040,164 @@ def test_resume_app_execution_keeps_cancellation_signals_when_resume_is_abandoneclear_signals.assert_not_called()[email protected]("entry_point", ["workflow", "chat", "async"])+ "scenario",+ [+ "valid",+ "canonical-task",+ "already-indexed-uppercase",+ "concurrent-index",+ "concurrent-stop",+ "task_id",+ "run_id",+ "tenant_id",+ "app_id",+ "user_id",+ ],+)+def test_resume_recovers_legacy_task_before_claiming_pause(+ monkeypatch: pytest.MonkeyPatch,+ sqlite_engine: Engine,+ sqlite_session_factory: sessionmaker[Session],+ entry_point: str,+ scenario: str,+) -> None:+ workflow_run_id = str(uuid.uuid4())+ task_id = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"+ app_mode = AppMode.ADVANCED_CHAT if entry_point == "chat" else AppMode.WORKFLOW+ _persist_resumption_models(+ sqlite_session_factory,+ workflow_run_id=workflow_run_id,+ conversation_id="conversation-id" if entry_point == "chat" else None,+ )+ with sqlite_session_factory.begin() as session:+ run = session.get(WorkflowRun, workflow_run_id)+ assert run is not None+ run.task_id = task_id if scenario == "already-indexed-uppercase" else None+ run.status = WorkflowExecutionStatus.PAUSED+ pause = WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="legacy-pause")+ session.add(pause)++ config = WorkflowUIBasedAppConfig(tenant_id="tenant-id", app_id="app-id", app_mode=app_mode, workflow_id="wf-id")+ if entry_point == "chat":+ generate_entity = _build_advanced_chat_generate_entity("conversation-id")+ generate_entity.workflow_run_id = workflow_run_id+ else:+ generate_entity = _build_workflow_generate_entity(stream=False)+ generate_entity.workflow_execution_id = workflow_run_id+ generate_entity.app_config = config+ generate_entity.task_id = task_id+ generate_entity.user_id = "account-id"+ generate_entity.invoke_from = InvokeFrom.DEBUGGER+ wrapper = (+ _AdvancedChatAppGenerateEntityWrapper(entity=generate_entity)+ if isinstance(generate_entity, AdvancedChatAppGenerateEntity)+ else _WorkflowGenerateEntityWrapper(entity=generate_entity)+ )+ runtime_state = RuntimeState(workflow_id="wf-id", variable_pool=VariablePool(), start_at=0.0)+ context = WorkflowResumptionContext(generate_entity=wrapper, serialized_graph_runtime_state=runtime_state.dumps())+ state = json.loads(context.dumps())+ saved_entity = state["generate_entity"]["entity"]+ if scenario in {"canonical-task", "already-indexed-uppercase"}:+ saved_entity["task_id"] = task_id.upper()+ elif scenario in {"tenant_id", "app_id"}:+ saved_entity["app_config"][scenario] = "other-owner"+ elif scenario == "run_id":+ saved_entity["workflow_run_id" if entry_point == "chat" else "workflow_execution_id"] = str(uuid.uuid4())+ elif scenario in {"task_id", "user_id"}:+ saved_entity[scenario] = "invalid-task-or-owner"+ monkeypatch.setattr(+ "repositories.sqlalchemy_api_workflow_run_repository.storage.load", lambda _: json.dumps(state).encode()+ )+ monkeypatch.setattr("repositories.sqlalchemy_api_workflow_run_repository.storage.delete", lambda _: None)+ repository = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory)+ module = async_workflow_tasks if entry_point == "async" else workflow_execute_task_module+ monkeypatch.setattr(type(module.db), "engine", property(lambda _: sqlite_engine))+ monkeypatch.setattr(+ module.DifyAPIRepositoryFactory, "create_api_workflow_run_repository", lambda *_args, **_kwargs: repository+ )+ monkeypatch.setattr(+ module, "_get_user" if entry_point == "async" else "_resolve_user_for_run", lambda *_args: _make_account()+ )+ resumed_entities: list[AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity] = []++ def record_app_resume(+ *, generate_entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity, **_kwargs: object+ ) -> None:+ resumed_entities.append(generate_entity)++ def record_workflow_resume(+ _self: WorkflowAppGenerator, *, application_generate_entity: WorkflowAppGenerateEntity, **_kwargs: object+ ) -> dict[str, object]:+ resumed_entities.append(application_generate_entity)+ return {}++ if entry_point == "async":+ monkeypatch.setattr(module, "_query_trigger_log_info", lambda *_args: None)+ monkeypatch.setattr(WorkflowAppGenerator, "resume", record_workflow_resume)+ else:+ monkeypatch.setattr(module, "clear_app_task_cancellation_signals", lambda _: None)+ monkeypatch.setattr(+ module, "_resume_advanced_chat" if entry_point == "chat" else "_resume_workflow", record_app_resume+ )++ if scenario in {"concurrent-index", "concurrent-stop"}:+ index_pause = repository.backfill_workflow_pause_task_id++ def concurrent_index(**kwargs: str) -> bool:+ assert index_pause(**kwargs)+ if scenario == "concurrent-stop":+ with sqlite_session_factory.begin() as session:+ run = session.get(WorkflowRun, workflow_run_id)+ assert run is not None+ run.status = WorkflowExecutionStatus.STOPPED+ run.stop_requested_at = datetime(2026, 10, 19)+ return index_pause(**kwargs)++ monkeypatch.setattr(repository, "backfill_workflow_pause_task_id", concurrent_index)++ resume_pause = repository.resume_workflow_pause+ resumed_pause_runs: list[str] = []++ def record_resume_pause(+ workflow_run_id: str, pause_entity: WorkflowPauseEntity, *, before_resume: Callable[[], None] | None = None+ ) -> WorkflowPauseEntity:+ resumed_pause_runs.append(workflow_run_id)+ return resume_pause(workflow_run_id, pause_entity, before_resume=before_resume)++ monkeypatch.setattr(repository, "resume_workflow_pause", record_resume_pause)+ invoke = async_workflow_tasks.resume_workflow_execution.run if entry_point == "async" else _resume_app_execution+ if entry_point == "async" and scenario == "concurrent-stop":+ with pytest.raises(WorkflowRunNotPausedError):+ invoke({"workflow_run_id": workflow_run_id})+ else:+ invoke({"workflow_run_id": workflow_run_id})++ with sqlite_session_factory() as session:+ persisted = session.get(WorkflowRun, workflow_run_id)+ assert persisted is not None+ if scenario in {"valid", "canonical-task", "already-indexed-uppercase", "concurrent-index"}:+ assert persisted.task_id == task_id+ assert persisted.status == WorkflowExecutionStatus.RUNNING+ assert resumed_pause_runs == [workflow_run_id]+ assert len(resumed_entities) == 1+ assert resumed_entities[0].task_id == task_id+ elif scenario == "concurrent-stop":+ assert persisted.task_id == task_id+ assert persisted.status == WorkflowExecutionStatus.STOPPED+ assert resumed_pause_runs == [workflow_run_id]+ assert resumed_entities == []+ else:+ assert persisted.task_id is None+ assert persisted.status == WorkflowExecutionStatus.PAUSED+ persisted_pause = session.get(WorkflowPause, pause.id)+ assert persisted_pause is not None+ assert persisted_pause.resumed_at is None+ assert resumed_pause_runs == []+ assert resumed_entities == []++def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs(monkeypatch: pytest.MonkeyPatch,sqlite_session: Session,@@ -1038,7 +1212,7 @@ def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs(generator_instance.resume.return_value = response_streammonkeypatch.setattr("tasks.app_generate.workflow_execute_task.AdvancedChatAppGenerator",- lambda: generator_instance,+ lambda *, execution_driver: generator_instance,)publish_streaming_response = MagicMock()@@ -1094,7 +1268,7 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs(generator_instance.resume.return_value = response_streammonkeypatch.setattr("tasks.app_generate.workflow_execute_task.WorkflowAppGenerator",- lambda: generator_instance,+ lambda *, execution_driver: generator_instance,)publish_streaming_response = MagicMock()@@ -1152,7 +1326,7 @@ def test_resume_workflow_ignores_missing_old_pause_after_repause(generator_instance.resume.return_value = response_streammonkeypatch.setattr("tasks.app_generate.workflow_execute_task.WorkflowAppGenerator",- lambda: generator_instance,+ lambda *, execution_driver: generator_instance,)publish_streaming_response = MagicMock()diff --git a/api/tests/unit_tests/tasks/test_workflow_node_execution_tasks.py b/api/tests/unit_tests/tasks/test_workflow_node_execution_tasks.pyindex 2a87d73fd35..4d214c88b6b 100644--- a/api/tests/unit_tests/tasks/test_workflow_node_execution_tasks.py+++ b/api/tests/unit_tests/tasks/test_workflow_node_execution_tasks.py@@ -11,11 +11,13 @@ from sqlalchemy import func, selectfrom sqlalchemy.orm import Session, sessionmakerfrom typing_extensions import TypedDict+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepositoryfrom graphon.entities import WorkflowNodeExecutionfrom graphon.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey-from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus-from models.enums import CreatorUserRole-from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom+from graphon.enums import BuiltinNodeTypes, WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType+from models import Account+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRunfrom tasks.workflow_node_execution_tasks import (_create_node_execution_from_domain,_update_node_execution_from_domain,@@ -338,4 +340,118 @@ def test_task_repeated_delivery_updates_one_row(persisted = observer.get(WorkflowNodeExecutionModel, EXECUTION_ID)assert persisted is not Noneassert persisted.outputs_dict == {"delivery": 2}- assert persisted.created_at == created_at.replace(tzinfo=None)+ assert persisted.created_at == created_at.replace(tzinfo=None)++[email protected]("writer", ["celery", "sql", "sql-data"])[email protected]("existing", [False, True])+ "status",+ [WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.PAUSED, WorkflowNodeExecutionStatus.RETRY],+)+def test_delayed_nonterminal_node_write_cannot_revive_or_insert_a_live_node_after_stop(+ sqlite_session_factory: sessionmaker[Session], writer: str, existing: bool, status: WorkflowNodeExecutionStatus+) -> None:+ finished_at = datetime(2026, 8, 12, 1, 1)+ with sqlite_session_factory.begin() as session:+ session.add(+ WorkflowRun(+ id=WORKFLOW_RUN_ID,+ tenant_id=TENANT_ID,+ app_id=APP_ID,+ workflow_id=WORKFLOW_ID,+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.STOPPED,+ error="User requested stop",+ stop_requested_at=finished_at,+ finished_at=finished_at,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=ACCOUNT_ID,+ )+ )+ if existing:+ node = _stored_execution()+ node.status = WorkflowNodeExecutionStatus.FAILED+ node.finished_at = finished_at+ node.error = "User requested stop"+ session.add(node)+ execution = _execution(status=status)+ if writer == "celery":+ kwargs: _ExecutionTaskKwargs = {+ "tenant_id": TENANT_ID,+ "app_id": APP_ID,+ "triggered_from": WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,+ "creator_user_id": ACCOUNT_ID,+ "creator_user_role": CreatorUserRole.ACCOUNT.value,+ }+ assert save_workflow_node_execution_task.run(execution_data=execution.model_dump(), **kwargs)+ else:+ user = Account(name="Test", email="[email protected]")+ user.id = ACCOUNT_ID+ repository = SQLAlchemyWorkflowNodeExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id=TENANT_ID,+ app_id=APP_ID,+ user=user,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ )+ if writer == "sql":+ repository.save(execution)+ else:+ repository.save_execution_data(execution)+ with sqlite_session_factory() as session:+ stored = session.get(WorkflowNodeExecutionModel, EXECUTION_ID)+ assert stored is not None+ assert stored.status == WorkflowNodeExecutionStatus.FAILED+ assert stored.finished_at == finished_at+ assert stored.error == "User requested stop"+++ "status",+ [WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.PAUSED, WorkflowNodeExecutionStatus.RETRY],+)+def test_delayed_write_preserves_node_cleanup_before_workflow_summary_finishes(+ sqlite_session_factory: sessionmaker[Session], status: WorkflowNodeExecutionStatus+) -> None:+ with sqlite_session_factory.begin() as session:+ session.add(+ WorkflowRun(+ id=WORKFLOW_RUN_ID,+ tenant_id=TENANT_ID,+ app_id=APP_ID,+ workflow_id=WORKFLOW_ID,+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.RUNNING,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=ACCOUNT_ID,+ )+ )+ node = _stored_execution()+ node.status = WorkflowNodeExecutionStatus.FAILED+ node.finished_at = datetime(2026, 8, 12, 1, 1)+ node.elapsed_time = 60+ node.error = "Execution aborted"+ session.add(node)+ session.flush()+ original = dict(session.execute(select(WorkflowNodeExecutionModel.__table__)).mappings().one())++ assert save_workflow_node_execution_task.run(+ execution_data=_execution(status=status).model_dump(),+ tenant_id=TENANT_ID,+ app_id=APP_ID,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN.value,+ creator_user_id=ACCOUNT_ID,+ creator_user_role=CreatorUserRole.ACCOUNT.value,+ )++ with sqlite_session_factory() as session:+ assert dict(session.execute(select(WorkflowNodeExecutionModel.__table__)).mappings().one()) == original+ run = session.get(WorkflowRun, WORKFLOW_RUN_ID)+ assert run is not None+ assert run.status == WorkflowExecutionStatus.RUNNING+ assert run.stop_requested_at is Nonediff --git a/api/tests/workflow_test_utils.py b/api/tests/workflow_test_utils.pyindex 1415bb1d521..a07d8724b2c 100644--- a/api/tests/workflow_test_utils.py+++ b/api/tests/workflow_test_utils.py@@ -3,8 +3,7 @@ from typing import Anyfrom core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_contextfrom core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_pool-from graphon.entities import GraphInitParams-from graphon.runtime import VariablePool+from graphon.runtime import InitParams, VariablePoolfrom graphon.variables.variables import Variable@@ -40,8 +39,8 @@ def build_test_graph_init_params(user_from: UserFrom | str = UserFrom.ACCOUNT,invoke_from: InvokeFrom | str = InvokeFrom.DEBUGGER,extra_context: Mapping[str, Any] | None = None,-) -> GraphInitParams:- return GraphInitParams(+) -> InitParams:+ return InitParams(workflow_id=workflow_id,graph_config=graph_config or {},run_context=build_test_run_context(diff --git a/api/uv.lock b/api/uv.lockindex 75ddd53bcd4..a7e4da58dd8 100644--- a/api/uv.lock+++ b/api/uv.lock@@ -1428,6 +1428,7 @@ dependencies = [{ name = "opentelemetry-propagator-b3" },{ name = "psycogreen" },{ name = "psycopg2-binary" },+ { name = "pypandoc" },{ name = "python-socketio" },{ name = "readabilipy" },{ name = "redis", extra = ["hiredis"] },@@ -1704,7 +1705,7 @@ requires-dist = [{ name = "gmpy2", specifier = ">=2.3.0,<3.0.0" },{ name = "google-api-python-client", specifier = ">=2.201.0,<3.0.0" },{ name = "google-cloud-aiplatform", specifier = ">=2.3.0,<3.0.0" },- { name = "graphon", specifier = "==0.7.0" },+ { name = "graphon", specifier = "==0.8.0" },{ name = "gunicorn", specifier = ">=26.2.0,<27.0.0" },{ name = "httpx", extras = ["socks"], specifier = "==0.28.1" },{ name = "httpx-sse", specifier = "==0.4.3" },@@ -1718,6 +1719,7 @@ requires-dist = [{ name = "opentelemetry-propagator-b3", specifier = ">=1.45.0,<2.0.0" },{ name = "psycogreen", specifier = ">=1.0.2,<2.0.0" },{ name = "psycopg2-binary", specifier = ">=2.9.12,<3.0.0" },+ { name = "pypandoc", specifier = "==1.17" },{ name = "python-socketio", specifier = ">=5.13.0,<6.0.0" },{ name = "readabilipy", specifier = "==0.3.0" },{ name = "redis", extras = ["hiredis"], specifier = ">=7.4.0,<8.0.0" },@@ -3081,18 +3083,19 @@ httpx = [[[package]]name = "graphon"-version = "0.7.0"+version = "0.8.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "charset-normalizer" },+ { name = "defusedxml" },{ name = "httpx" },{ name = "json-repair" },{ name = "jsonschema" },+ { name = "odfdo" },{ name = "orjson" },{ name = "pandas", extra = ["excel"] },{ name = "pydantic" },{ name = "pydantic-extra-types" },- { name = "pypandoc" },{ name = "pypdf" },{ name = "pypdfium2" },{ name = "python-docx" },@@ -3100,12 +3103,12 @@ dependencies = [{ name = "tiktoken" },{ name = "transformers" },{ name = "typing-extensions" },- { name = "unstructured", extra = ["docx", "epub", "md", "ppt", "pptx"] },+ { name = "unstructured", extra = ["docx", "md", "ppt", "pptx"] },{ name = "webvtt-py" },]-sdist = { url = "https://files.pythonhosted.org/packages/6a/c6/6f16398e28bceb11304f8dd743d5f4f2c1e1aedf20eef6531764edb40a7e/graphon-0.7.0.tar.gz", hash = "sha256:e3f19284432d6e4a947fb285ef36bc79eb48c41af36867b03c0d70f062fe8563", size = 266715, upload-time = "2026-07-29T10:00:51.303Z" }+sdist = { url = "https://files.pythonhosted.org/packages/f2/82/a2ded2b6a7879e7ee855936db7410014cd41c1c709155a1ccfe07e208633/graphon-0.8.0.tar.gz", hash = "sha256:654ad8f5cb7ff1711cf6b09e68a6afc7fe1e552b2f8306babdb848142d0a6bf7", size = 281821, upload-time = "2026-09-09T20:03:19.606Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/9c/9b/3b02d944a20b3693bb5bb77cbb16b3afed44a965096970c47625560c0235/graphon-0.7.0-py3-none-any.whl", hash = "sha256:6f7608aaaa65607c4935735137ed4b7029b1e6ca80202cbe829af3370cf4404e", size = 380138, upload-time = "2026-07-29T10:00:49.654Z" },+ { url = "https://files.pythonhosted.org/packages/0d/8d/64552a2dc8ef4d1b0c43d927301000533be75d6c51664d5a33fb1ead00ff/graphon-0.8.0-py3-none-any.whl", hash = "sha256:183b668cdd2b1b6c39e75e68778994855400c9a8e314e0bda3b59a2b0fe7e8a0", size = 400203, upload-time = "2026-09-09T20:03:18.198Z" },][[package]]@@ -4347,6 +4350,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/d9/f4/78229a1066068ca14fc60fb26cf7381cabe4382261392b90e5f9552722d4/oauthlib-4.0.0-py3-none-any.whl", hash = "sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd", size = 159715, upload-time = "2026-09-28T06:01:17.008Z" },]+[[package]]+name = "odfdo"+version = "3.24.6"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "lxml" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/e9/cb/48e616cd7f4d7f48363dcb622833c873bc51c42197270a6788d83e3331f0/odfdo-3.24.6.tar.gz", hash = "sha256:113af78fc76218418553b9980db5df775ddf32ef3676ee9df2fdf2be7507a35f", size = 314402, upload-time = "2026-08-22T21:25:39.679Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/df/e3/cb47b318e0f186cbfeec99200ebda1bdda9b31b4a80a2f7c6925a77ef149/odfdo-3.24.6-py3-none-any.whl", hash = "sha256:ccb9c715627381cceff4fb8b24586f737396a5f6a4fb662e214fd2203bc0d4f5", size = 415768, upload-time = "2026-08-22T21:25:38.426Z" },+]+[[package]]name = "odfpy"version = "1.4.1"@@ -5582,19 +5598,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/0c/86/e2ffa604eacfbec3f430b1d850e7e04c4101eca1a5828f9ae54bf51dfba4/pypandoc-1.17-py3-none-any.whl", hash = "sha256:01fdbffa61edb9f8e82e8faad6954efcb7b6f8f0634aead4d89e322a00225a67", size = 23554, upload-time = "2026-03-14T22:38:46.007Z" },]-[[package]]-name = "pypandoc-binary"-version = "1.17"-source = { registry = "https://pypi.org/simple" }-wheels = [- { url = "https://files.pythonhosted.org/packages/80/85/681a54111f0948821a5cf87ce30a88bb0a3f6848af5112c912abac4a2b77/pypandoc_binary-1.17-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:734726dc618ef276343e272e1a6b4567e59c2ef9ef41d5533042deac3b0531f1", size = 25553945, upload-time = "2026-03-14T22:38:47.91Z" },- { url = "https://files.pythonhosted.org/packages/15/58/8fd107c68522957868c1e785fbea7595608df118e440e424d189668294df/pypandoc_binary-1.17-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fcfd28f347ed998dda28823fc6bc24f9310e7fdf3ddceaf925bf0563a100ab5b", size = 25553944, upload-time = "2026-03-14T22:38:50.74Z" },- { url = "https://files.pythonhosted.org/packages/f4/27/ac1078239aae14b94c51975b7f46ad8e099e47d7ae26c175a5486b1c0099/pypandoc_binary-1.17-py3-none-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d6b620b21c9374e3e48aabd518492bf0776b148442ee28816f6aaf52da3d4387", size = 34460960, upload-time = "2026-03-14T22:38:53.391Z" },- { url = "https://files.pythonhosted.org/packages/8d/7f/1e5612b52900ebe590862dabeadf546f739b27527dcd8bfd632f8adac1be/pypandoc_binary-1.17-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ada156cb980cd54fd6534231788e668c00dbb591cbd24f0be0bd86812eb8788", size = 36867598, upload-time = "2026-03-14T22:38:56.351Z" },- { url = "https://files.pythonhosted.org/packages/3b/31/a5a867159c4080e5d368f4a53540a727501a2f31affc297dc8e0fced96a7/pypandoc_binary-1.17-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2f439dcd211183bb3460253ca4511101df6e1acf4a01f45f5617e1fa2ad24279", size = 36867584, upload-time = "2026-03-14T22:38:59.087Z" },- { url = "https://files.pythonhosted.org/packages/0d/2d/6a51cd4e54bdf132c19416801077c34bd40ba182e85d843360d36ae03a2d/pypandoc_binary-1.17-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f6e6d3e4cfafbe23189a08db3d41f8def260bacd6e7e382bceadab7ba1f17da6", size = 34460949, upload-time = "2026-03-14T22:39:01.71Z" },-]-[[package]]name = "pyparsing"version = "3.2.5"@@ -7450,9 +7453,6 @@ wheels = [docx = [{ name = "python-docx" },]-epub = [- { name = "pypandoc-binary", marker = "sys_platform != 'win32'" },-]md = [{ name = "markdown" },]diff --git a/docker/envs/core-services/shared.env.example b/docker/envs/core-services/shared.env.exampleindex bbb56f547fa..cedce027c4d 100644--- a/docker/envs/core-services/shared.env.example+++ b/docker/envs/core-services/shared.env.example@@ -235,10 +235,7 @@ WORKFLOW_CALL_MAX_DEPTH=5MAX_VARIABLE_SIZE=204800WORKFLOW_GENERATOR_NODE_BUILDER_MAX_WORKERS=6WORKFLOW_FILE_UPLOAD_LIMIT=10-GRAPH_ENGINE_MIN_WORKERS=3GRAPH_ENGINE_MAX_WORKERS=10-GRAPH_ENGINE_SCALE_UP_THRESHOLD=0-GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0ALIYUN_SLS_ACCESS_KEY_ID=ALIYUN_SLS_ACCESS_KEY_SECRET=WEBHOOK_REQUEST_BODY_MAX_SIZE=10485760diff --git a/packages/contracts/generated/api/console/snippets/orpc.gen.ts b/packages/contracts/generated/api/console/snippets/orpc.gen.tsindex fe86a292389..635e7773a69 100644--- a/packages/contracts/generated/api/console/snippets/orpc.gen.ts+++ b/packages/contracts/generated/api/console/snippets/orpc.gen.ts@@ -108,20 +108,15 @@ import {} from './zod.gen.ts'/**- * Stop a running snippet workflow task- *- * Uses both the legacy stop flag mechanism and the graph engine- * command channel for backward compatibility.+ * Stop a running or paused task belonging to this snippet*/export const post = oc.route({- description:- 'Uses both the legacy stop flag mechanism and the graph engine\ncommand channel for backward compatibility.',inputStructure: 'detailed',method: 'POST',operationId: 'postSnippetsBySnippetIdWorkflowRunsTasksByTaskIdStop',path: '/snippets/{snippet_id}/workflow-runs/tasks/{task_id}/stop',- summary: 'Stop a running snippet workflow task',+ summary: 'Stop a running or paused task belonging to this snippet',tags: ['console'],}).input(z.object({ params: zPostSnippetsBySnippetIdWorkflowRunsTasksByTaskIdStopPath }))diff --git a/web/app/components/app/app-publisher/__tests__/sections.spec.tsx b/web/app/components/app/app-publisher/__tests__/sections.spec.tsxindex a2691dafc04..e68840e5e8a 100644--- a/web/app/components/app/app-publisher/__tests__/sections.spec.tsx+++ b/web/app/components/app/app-publisher/__tests__/sections.spec.tsx@@ -1,31 +1,18 @@import type { ComponentProps } from 'react'import type { FetchWorkflowDraftResponse } from '@/types/workflow'import { Popover, PopoverContent, PopoverTrigger } from '@langgenius/dify-ui/popover'-import { act, fireEvent, screen, waitFor, within } from '@testing-library/react'+import { act, fireEvent, screen, waitFor } from '@testing-library/react'import userEvent from '@testing-library/user-event'import { NuqsTestingAdapter } from 'nuqs/adapters/testing'import { useRef } from 'react'+import { consoleQuery } from '@/service/console'import { expectLoadingButton } from '@/test/button'-import { createConsoleQueryWrapper } from '@/test/console/query-data'+import { createConsoleQueryClient, createConsoleQueryWrapper } from '@/test/console/query-data'import { render as renderWithConsoleState } from '@/test/console/render'import { PublisherActionsSection } from '../built-in-publisher/actions-section'import { PublisherSummarySection } from '../built-in-publisher/summary-section'import { usePublishController } from '../publisher-content/use-publish-controller'-vi.mock('../publish-with-multiple-model', () => ({- default: ({- disabled,- onSelect,- }: {- disabled?: boolean- onSelect: (item: Record<string, unknown>) => void- }) => (- <button type="button" disabled={disabled} onClick={() => onSelect({ model: 'gpt-4o' })}>- publish-multiple-model- </button>- ),-}))-const createVersionInfo = (overrides: Partial<FetchWorkflowDraftResponse> = {},): FetchWorkflowDraftResponse => ({@@ -249,17 +236,67 @@ describe('app-publisher sections', () => {expect(screen.getByRole('button', { name: /common\.published\b/ })).toBeDisabled()})- it('keeps multiple-model publication behind the model selection action', async () => {- const user = userEvent.setup()- const handlePublish = vi.fn().mockResolvedValue(undefined)- render(<PublisherPopup debugWithMultipleModel handlePublish={handlePublish} />)- await user.click(screen.getByRole('button', { name: 'Open publisher' }))- const modelButton = screen.getByRole('button', { name: 'publish-multiple-model' })- expect(publishFrom(modelButton).defaultPrevented).toBe(false)- expect(handlePublish).not.toHaveBeenCalled()- await user.click(modelButton)- expect(handlePublish).toHaveBeenCalledWith({ model: 'gpt-4o' })- })+ it.each([false, true])(+ 'keeps multiple-model publication behind model selection (disabled: %s)',+ async (publishDisabled) => {+ const user = userEvent.setup()+ const handlePublish = vi.fn().mockResolvedValue(undefined)+ const modelConfig = { id: 'model-1', provider: 'openai', model: 'gpt-4o', parameters: {} }+ const queryClient = createConsoleQueryClient()+ queryClient.setQueryData(+ consoleQuery.workspaces.current.models.modelTypes.byModelType.get.queryOptions({+ input: { params: { model_type: 'llm' } },+ }).queryKey,+ {+ data: [+ {+ provider: 'openai',+ label: { en_US: 'OpenAI', zh_Hans: 'OpenAI' },+ status: 'active',+ tenant_id: 'workspace-1',+ models: [+ {+ model: 'gpt-4o',+ label: { en_US: 'GPT-4o', zh_Hans: 'GPT-4o' },+ model_type: 'llm',+ model_properties: {},+ fetch_from: 'predefined-model',+ status: 'active',+ deprecated: false,+ has_invalid_load_balancing_configs: false,+ load_balancing_enabled: false,+ },+ ],+ },+ ],+ },+ )+ render(+ <PublisherPopup+ debugWithMultipleModel+ handlePublish={handlePublish}+ multipleModelConfigs={[modelConfig]}+ publishDisabled={publishDisabled}+ />,+ queryClient,+ )+ await user.click(screen.getByRole('button', { name: 'Open publisher' }))+ const modelButton = screen.getByRole('button', { name: /operation\.applyConfig\b/ })+ expect(publishFrom(modelButton).defaultPrevented).toBe(false)+ expect(handlePublish).not.toHaveBeenCalled()+ if (publishDisabled) {+ expect(modelButton).toBeDisabled()+ await user.click(modelButton)+ expect(screen.queryByRole('menuitem', { name: /GPT-4o/ })).not.toBeInTheDocument()+ expect(handlePublish).not.toHaveBeenCalled()+ return+ }+ expect(modelButton).toBeEnabled()+ await user.click(modelButton)+ await user.click(screen.getByRole('menuitem', { name: /GPT-4o/ }))+ expect(handlePublish).toHaveBeenCalledWith(expect.objectContaining(modelConfig))+ },+ )it('should render restore controls for published chat apps', async () => {const user = userEvent.setup()@@ -344,7 +381,6 @@ describe('app-publisher sections', () => {expect(screen.getByText(/(?:^|\.)common\.notPublishedYet(?=$|:)/)).toBeInTheDocument()expect(screen.getByText(/(?:^|\.)common\.publish(?=$|:)/)).toBeInTheDocument()- expect(screen.getByText('P')).toBeInTheDocument()expect(screen.getByRole('status').textContent).toMatch(/common\.currentDraft\b/)})@@ -383,7 +419,6 @@ describe('app-publisher sections', () => {expect(onEditVersion).toHaveBeenCalledTimes(1)const publishButton = screen.getByRole('button', { name: /common\.publishUpdate\b/ })expect(publishButton).toBeEnabled()- expect(within(publishButton).getByText('P')).toBeInTheDocument()expect(screen.getByText(/common\.autoSaved\b/)).toBeInTheDocument()})@@ -426,7 +461,6 @@ describe('app-publisher sections', () => {expect(onEditVersion).toHaveBeenCalledTimes(1)expect(screen.getByRole('button', { name: /common\.publishUpdate\b/ })).toBeEnabled()expect(screen.getByText(/common\.autoSaved\b/)).toBeInTheDocument()- expect(screen.getAllByText(/2 minutes ago/)).not.toHaveLength(0)})it('should keep non-workflow apps free of workflow version details and saved time', () => {@@ -451,35 +485,11 @@ describe('app-publisher sections', () => {)expect(screen.getAllByText(/common\.latestPublished\b/)).toHaveLength(1)- expect(screen.queryByText('#5')).not.toBeInTheDocument()expect(screen.queryByText(/versionHistory\.nameIt\b/)).not.toBeInTheDocument()expect(screen.getByRole('button', { name: /common\.publishUpdate\b/ })).toBeEnabled()expect(screen.getByRole('status').textContent).toMatch(/common\.currentDraft\b/)})- it('should disable multiple-model publishing when publishing is unavailable', () => {- render(- <PublisherPopup- defaultOpen- debugWithMultipleModel- draftUpdatedAt={Date.now()}- formatTimeFromNow={() => '1 minute ago'}- handlePublish={vi.fn()}- handleRestore={vi.fn()}- isChatApp={false}- isPublishing={false}- multipleModelConfigs={[]}- publishDisabled- published={false}- publishedAt={Date.now()}- startNodeLimitExceeded={false}- upgradeHighlightStyle={{}}- />,- )-- expect(screen.getByRole('button', { name: 'publish-multiple-model' })).toBeDisabled()- })-it('should render the upgrade hint when the start node limit is exceeded', () => {render(<PublisherPopup@@ -503,7 +513,7 @@ describe('app-publisher sections', () => {expect(screen.getByText(/(?:^|\.)publishLimit\.startNodeDesc(?=$|:)/)).toBeInTheDocument()})- it('should render the published workflow actions with Workflow as Tool after Marketplace', async () => {+ it('should render the published workflow actions with Workflow as Tool', async () => {const user = userEvent.setup()const handleOpenRunConfig = vi.fn()const onConfigureWorkflowTool = vi.fn()@@ -552,10 +562,6 @@ describe('app-publisher sections', () => {const workflowToolAction = screen.getByRole('button', {name: /common\.workflowAsTool\b/,})- expect(- marketplaceAction.compareDocumentPosition(workflowToolAction) &- Node.DOCUMENT_POSITION_FOLLOWING,- ).toBeTruthy()expect(screen.getByRole('status', { name: /common\.configureRequired\b/ })).toBeInTheDocument()await user.click(marketplaceAction)@@ -600,7 +606,7 @@ describe('app-publisher sections', () => {expect(onConfigureWorkflowTool).toHaveBeenCalledTimes(1)})- it('should show the disabled reason below setup and configured workflow tool actions', () => {+ it('should show the disabled reason for setup and configured workflow tool actions', () => {const commonProps = {appId: 'workflow-app',appMode: 'workflow' as const,@@ -621,9 +627,6 @@ describe('app-publisher sections', () => {const setupReason = screen.getByText('Workflow tool unavailable')expect(setupAction).toBeDisabled()expect(setupReason).toBeVisible()- expect(- setupAction.compareDocumentPosition(setupReason) & Node.DOCUMENT_POSITION_FOLLOWING,- ).toBeTruthy()rerender(<PublisherActionsSection {...commonProps} toolPublished />)@@ -633,9 +636,6 @@ describe('app-publisher sections', () => {expect(configureAction).toBeDisabled()expect(manageAction).toBeDisabled()expect(configuredReason).toBeVisible()- expect(- manageAction.compareDocumentPosition(configuredReason) & Node.DOCUMENT_POSITION_FOLLOWING,- ).toBeTruthy()})it('should surface update-needed and loading states for a configured workflow tool', async () => {@@ -756,14 +756,10 @@ describe('app-publisher sections', () => {/>,)- expect(screen.getByText(/(?:^|\.)common\.openWebApp(?=$|:)/).closest('button')).toBeDisabled()- expect(- screen.getByText(/(?:^|\.)appMenus\.accessPoint(?=$|:)/).closest('button'),- ).toBeDisabled()- expect(screen.getByText(/(?:^|\.)appMenus\.deploy(?=$|:)/).closest('button')).toBeDisabled()- expect(- screen.getByText(/(?:^|\.)common\.workflowAsTool(?=$|:)/).closest('button'),- ).toBeDisabled()+ expect(screen.getByRole('button', { name: /common\.openWebApp\b/ })).toBeDisabled()+ expect(screen.getByRole('button', { name: /appMenus\.accessPoint\b/ })).toBeDisabled()+ expect(screen.getByRole('button', { name: /appMenus\.deploy\b/ })).toBeDisabled()+ expect(screen.getByRole('button', { name: /common\.workflowAsTool\b/ })).toBeDisabled()})it('should show the disabled reason when hovering an unavailable action', async () => {@@ -825,8 +821,8 @@ describe('app-publisher sections', () => {})})-function render(ui: React.ReactElement) {- const { wrapper: QueryWrapper } = createConsoleQueryWrapper()+function render(ui: React.ReactElement, queryClient?: ReturnType<typeof createConsoleQueryClient>) {+ const { wrapper: QueryWrapper } = createConsoleQueryWrapper({ queryClient })return renderWithConsoleState(ui, {wrapper: ({ children }) => (<NuqsTestingAdapter>diff --git a/web/app/components/base/chat/chat/__tests__/hooks.spec.tsx b/web/app/components/base/chat/chat/__tests__/hooks.spec.tsxindex 164d91fbbd5..b674b0c752a 100644--- a/web/app/components/base/chat/chat/__tests__/hooks.spec.tsx+++ b/web/app/components/base/chat/chat/__tests__/hooks.spec.tsx@@ -109,13 +109,6 @@ describe('useChat', () => {vi.restoreAllMocks()})- it('should initialize correctly with empty config', () => {- const { result } = renderHookWithConsoleQuery(() => useChat())- expect(result.current.chatList).toEqual([])- expect(result.current.isResponding).toBe(false)- expect(result.current.suggestedQuestions).toEqual([])- })-it('should initialize with opening statement and suggested questions', () => {const config = {opening_statement: 'Hello {{name}}',@@ -3067,52 +3060,6 @@ describe('useChat', () => {expect(result.current.suggestedQuestions).toHaveLength(0)})- it('should cover handleResume onHumanInputFormFilled splicing and onHumanInputFormTimeout updating', () => {- let resumeCallbacks: HookCallbacks- vi.mocked(sseGet).mockImplementation(async (_url, _params, options) => {- resumeCallbacks = options as HookCallbacks- })-- const prevChatTree = [- {- id: 'q',- content: 'query',- isAnswer: false,- children: [- {- id: 'm-1',- content: 'initial',- isAnswer: true,- siblingIndex: 0,- humanInputFormDataList: [{ node_id: 'n-1', expiration_time: 100 }],- },- ],- },- ]-- const { result } = renderHookWithConsoleQuery(() =>- useChat(undefined, undefined, prevChatTree as ChatItemInTree[]),- )- act(() => {- result.current.handleResume('m-1', 'wr-1', { isPublicAPI: true })- })-- act(() => {- // Hit L535-537: onHumanInputFormTimeout (update)- resumeCallbacks.onHumanInputFormTimeout({ data: { node_id: 'n-1', expiration_time: 200 } })- })- expect(result.current.chatList[1]!.humanInputFormDataList).toEqual([- expect.objectContaining({ node_id: 'n-1', expiration_time: 200 }),- ])- act(() => {- resumeCallbacks.onHumanInputFormFilled({ data: { node_id: 'n-1' } })- })-- const lastResponse = result.current.chatList[1]- expect(lastResponse!.humanInputFormDataList).toHaveLength(0)- expect(lastResponse!.humanInputFilledFormDataList).toHaveLength(1)- })-it('should cover handleAnnotationAdded updating node', async () => {const prevChatTree = [{diff --git a/web/app/components/base/chat/chat/answer/human-input-content/__tests__/submitted.spec.tsx b/web/app/components/base/chat/chat/answer/human-input-content/__tests__/submitted.spec.tsxindex 48860e824cc..1938441cf20 100644--- a/web/app/components/base/chat/chat/answer/human-input-content/__tests__/submitted.spec.tsx+++ b/web/app/components/base/chat/chat/answer/human-input-content/__tests__/submitted.spec.tsx@@ -38,8 +38,8 @@ describe('SubmittedHumanInputContent Integration', () => {/>,)- expect(screen.getByTestId('submitted-field-answer')).toHaveTextContent('approved')- expect(screen.queryByTestId('submitted-content')).not.toBeInTheDocument()+ expect(screen.getByText('approved')).toBeInTheDocument()+ expect(screen.queryByText('Rendered **Markdown** content')).not.toBeInTheDocument()})it('should render submitted select and file fields with the original form layout', () => {@@ -83,7 +83,7 @@ describe('SubmittedHumanInputContent Integration', () => {expect(screen.getByRole('combobox', { name: 'decision' })).toBeDisabled()expect(screen.getByRole('combobox', { name: 'decision' })).toHaveTextContent('approve')- expect(screen.getByTestId('submitted-field-attachment')).toHaveTextContent('decision.pdf')+ expect(screen.getByText('decision.pdf')).toBeInTheDocument()})it.each([undefined, {}])(@@ -102,7 +102,6 @@ describe('SubmittedHumanInputContent Integration', () => {expect(screen.getByText('workflowHumanInput.nodes.humanInput.userActions.triggered'),).toBeInTheDocument()- expect(screen.queryByTestId('submitted-field-values')).not.toBeInTheDocument()},)@@ -118,7 +117,7 @@ describe('SubmittedHumanInputContent Integration', () => {/>,)- expect(screen.getByTestId('submitted-field-answer')).toHaveTextContent('approved')- expect(screen.queryByTestId('submitted-content')).not.toBeInTheDocument()+ expect(screen.getByText('approved')).toBeInTheDocument()+ expect(screen.queryByText('Rendered **Markdown** content')).not.toBeInTheDocument()})})diff --git a/web/app/components/workflow/hooks/use-workflow-run-event/__tests__/use-workflow-node-started.spec.ts b/web/app/components/workflow/hooks/use-workflow-run-event/__tests__/use-workflow-node-started.spec.tsindex fd98a837123..83054b2e3e7 100644--- a/web/app/components/workflow/hooks/use-workflow-run-event/__tests__/use-workflow-node-started.spec.ts+++ b/web/app/components/workflow/hooks/use-workflow-run-event/__tests__/use-workflow-node-started.spec.ts@@ -2,9 +2,11 @@ import { act, waitFor } from '@testing-library/react'import { createNodeTracing } from '../../../__tests__/fixtures'import { baseRunningData } from '../../../__tests__/workflow-test-env'import { NodeRunningStatus } from '../../../types'+import { useWorkflowNodeFinished } from '../use-workflow-node-finished'import { useWorkflowNodeStarted } from '../use-workflow-node-started'import {containerParams,+ createNodeFinishedResponse,createNodeStartedResponse,getEdgeRuntimeState,getNodeRuntimeState,@@ -12,6 +14,42 @@ import {} from './test-helpers'describe('useWorkflowNodeStarted', () => {+ it('keeps one completed trace with execution data when the first node is replayed', () => {+ const { result, store } = renderViewportHook(+ () => ({ ...useWorkflowNodeStarted(), ...useWorkflowNodeFinished() }),+ { initialStoreState: { workflowRunningData: baseRunningData() } },+ )+ const started = createNodeStartedResponse()+ started.data = {+ ...started.data,+ id: 'trace-1',+ index: 1,+ title: 'Request',+ inputs: { request: 'Review this request' },+ extras: { icon: { content: '👤', background: '#ffffff' } },+ }+ const finished = createNodeFinishedResponse({+ data: {+ ...started.data,+ status: NodeRunningStatus.Succeeded,+ process_data: { decision: 'approve' },+ outputs: { text: 'Approved' },+ },+ })+ delete finished.data.extras++ for (const event of [started, { ...started, data: { ...started.data, extras: {} } }]) {+ act(() => {+ result.current.handleWorkflowNodeStarted(event, containerParams)+ result.current.handleWorkflowNodeFinished(finished)+ })+ }++ expect(store.getState().workflowRunningData!.tracing).toEqual([+ { ...finished.data, extras: started.data.extras },+ ])+ })+it('pushes to tracing, sets node running, and adjusts viewport for root node', async () => {const { result, store } = renderViewportHook(() => useWorkflowNodeStarted(), {initialStoreState: { workflowRunningData: baseRunningData() },diff --git a/web/app/components/workflow/hooks/use-workflow-run-event/use-workflow-node-started.ts b/web/app/components/workflow/hooks/use-workflow-run-event/use-workflow-node-started.tsindex 2ee7ea46b97..4423a88ab00 100644--- a/web/app/components/workflow/hooks/use-workflow-run-event/use-workflow-node-started.ts+++ b/web/app/components/workflow/hooks/use-workflow-run-event/use-workflow-node-started.ts@@ -25,11 +25,12 @@ export const useWorkflowNodeStarted = () => {const currentIndex = workflowRunningData?.tracing?.findIndex((item) => item.node_id === data.node_id,)- if (currentIndex && currentIndex > -1) {+ if (currentIndex !== undefined && currentIndex > -1) {setWorkflowRunningData(produce(workflowRunningData!, (draft) => {draft.tracing![currentIndex] = {...data,+ extras: { ...draft.tracing![currentIndex]!.extras, ...data.extras },status: NodeRunningStatus.Running,}}),diff --git a/web/app/components/workflow/nodes/_base/components/variable/__tests__/utils.spec.ts b/web/app/components/workflow/nodes/_base/components/variable/__tests__/utils.spec.tsindex a4b18e79f69..d19ef51afa2 100644--- a/web/app/components/workflow/nodes/_base/components/variable/__tests__/utils.spec.ts+++ b/web/app/components/workflow/nodes/_base/components/variable/__tests__/utils.spec.ts@@ -3,6 +3,7 @@ import type { AgentNodeType } from '@/app/components/workflow/nodes/agent/types'import type { AnswerNodeType } from '@/app/components/workflow/nodes/answer/types'import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'import type { HumanInputNodeType } from '@/app/components/workflow/nodes/human-input/types'+import type { ListFilterNodeType } from '@/app/components/workflow/nodes/list-operator/types'import type { LLMNodeType } from '@/app/components/workflow/nodes/llm/types'import type { EnvironmentVariable, Node, PromptItem } from '@/app/components/workflow/types'import { describe, expect, it } from 'vite-plus/test'@@ -15,7 +16,13 @@ import {PromptRole,VarType,} from '@/app/components/workflow/types'-import { findUsedVarNodes, getNodeUsedVars, toNodeAvailableVars, updateNodeVars } from '../utils'+import {+ findUsedVarNodes,+ getNodeUsedVarPassToServerKey,+ getNodeUsedVars,+ toNodeAvailableVars,+ updateNodeVars,+} from '../utils'const createNode = <T>(data: Node<T>['data']): Node<T> => ({id: 'node-1',@@ -220,6 +227,79 @@ describe('variable utils', () => {})describe('getNodeUsedVars', () => {+ const createListNode = (enabled: boolean) =>+ createNode<ListFilterNodeType>({+ type: BlockEnum.ListFilter,+ title: 'List',+ desc: '',+ variable: ['source', 'items'],+ var_type: VarType.arrayNumber,+ item_var_type: VarType.number,+ filter_by: {+ enabled,+ conditions: [{ key: '', comparison_operator: '≥', value: '{{#source.minimum#}}' }],+ },+ extract_by: { enabled, serial: '{{#source.serial#}}' },+ order_by: { enabled: false, key: '', value: 'asc' },+ limit: { enabled: false },+ })++ it('maps every active list dependency to its container debug input key', () => {+ const node = createListNode(true)+ const selectors = getNodeUsedVars(node, { forExecution: true })++ expect(selectors).toEqual([+ ['source', 'items'],+ ['source', 'minimum'],+ ['source', 'serial'],+ ])+ expect(selectors.map((selector) => getNodeUsedVarPassToServerKey(node, selector))).toEqual([+ '#source.items#',+ '#source.minimum#',+ '#source.serial#',+ ])+ })++ it('excludes disabled list settings from execution while retaining saved references', () => {+ const node = createListNode(false)++ expect(getNodeUsedVars(node, { forExecution: true })).toEqual([['source', 'items']])+ expect(getNodeUsedVars(node)).toEqual([+ ['source', 'items'],+ ['source', 'minimum'],+ ['source', 'serial'],+ ])+ })++ it.each(['empty', 'not empty'] as const)(+ 'ignores hidden operands for %s during execution and keeps them for renaming',+ (comparisonOperator) => {+ const node = createListNode(true)+ node.data.extract_by.enabled = false+ node.data.filter_by.conditions = [+ { key: '', comparison_operator: comparisonOperator, value: '{{#deleted.old#}}' },+ ]++ expect(getNodeUsedVars(node, { forExecution: true })).toEqual([['source', 'items']])+ expect(getNodeUsedVars(node)).toContainEqual(['deleted', 'old'])+ const renamed = updateNodeVars(node, ['deleted', 'old'], ['restored', 'value'])+ expect(getNodeUsedVars(renamed)).toContainEqual(['restored', 'value'])+ },+ )++ it('renames saved list filter and extraction references', () => {+ const node = createListNode(false)++ const renamedFilter = updateNodeVars(node, ['source', 'minimum'], ['renamed', 'minimum'])+ const renamed = updateNodeVars(renamedFilter, ['source', 'serial'], ['renamed', 'serial'])++ expect(getNodeUsedVars(renamed)).toEqual([+ ['source', 'items'],+ ['renamed', 'minimum'],+ ['renamed', 'serial'],+ ])+ })+it('should read variables from llm jinja prompt text', () => {const node = createNode<LLMNodeType>(createLLMNodeData([diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.tsindex 50a155613f0..fbedaa7e0f7 100644--- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts+++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts@@ -59,6 +59,7 @@ import { isAgentV2NodeData } from '@/app/components/workflow/nodes/agent-v2/typeimport DataSourceNodeDefault from '@/app/components/workflow/nodes/data-source/default'import HumanInputNodeDefault from '@/app/components/workflow/nodes/human-input/default'import { DeliveryMethodType } from '@/app/components/workflow/nodes/human-input/types'+import { comparisonOperatorNotRequireValue } from '@/app/components/workflow/nodes/if-else/utils'import ToolNodeDefault from '@/app/components/workflow/nodes/tool/default'import { resolveVarType } from '@/app/components/workflow/nodes/tool/output-schema-utils'import PluginTriggerNodeDefault from '@/app/components/workflow/nodes/trigger-plugin/default'@@ -1213,7 +1214,7 @@ const replaceOldVarInPromptItem = (: {}),})-/** Include saved references for rename/delete/copy; execution excludes inactive route conditions. */+/** Include saved references for rename/delete/copy; execution excludes inactive settings. */export const getNodeUsedVars = (node: Node, { forExecution = false } = {}): ValueSelector[] => {const { data } = nodeconst { type } = data@@ -1405,7 +1406,21 @@ export const getNodeUsedVars = (node: Node, { forExecution = false } = {}): Valu}case BlockEnum.ListFilter: {- res = [(data as ListFilterNodeType).variable]+ const payload = data as ListFilterNodeType+ const templates: string[] = []+ if (!forExecution || payload.filter_by?.enabled) {+ templates.push(+ ...(payload.filter_by?.conditions || []).flatMap((condition) =>+ typeof condition.value === 'string' &&+ (!forExecution || !comparisonOperatorNotRequireValue(condition.comparison_operator))+ ? [condition.value]+ : [],+ ),+ )+ }+ if ((!forExecution || payload.extract_by?.enabled) && payload.extract_by?.serial)+ templates.push(payload.extract_by.serial)+ res = [payload.variable, ...matchNotSystemVars(templates)]break}@@ -1508,7 +1523,8 @@ export const getNodeUsedVarPassToServerKey = (break}- case BlockEnum.VariableAggregator: {+ case BlockEnum.VariableAggregator:+ case BlockEnum.ListFilter: {res = `#${valueSelector.join('.')}#`break}@@ -1869,6 +1885,19 @@ export const updateNodeVars = (const payload = data as ListFilterNodeTypeif (payload.variable.join('.') === oldVarSelector.join('.'))payload.variable = newVarSelector+ payload.filter_by.conditions = payload.filter_by.conditions.map((condition) => ({+ ...condition,+ value:+ typeof condition.value === 'string'+ ? replaceOldVarInText(condition.value, oldVarSelector, newVarSelector)+ : condition.value,+ }))+ if (payload.extract_by.serial)+ payload.extract_by.serial = replaceOldVarInText(+ payload.extract_by.serial,+ oldVarSelector,+ newVarSelector,+ )break}case BlockEnum.HumanInput: {diff --git a/web/app/components/workflow/nodes/_base/components/workflow-panel/last-run/__tests__/use-last-run.spec.ts b/web/app/components/workflow/nodes/_base/components/workflow-panel/last-run/__tests__/use-last-run.spec.tsnew file mode 100644index 00000000000..f44e17180c1--- /dev/null+++ b/web/app/components/workflow/nodes/_base/components/workflow-panel/last-run/__tests__/use-last-run.spec.ts@@ -0,0 +1,144 @@+import type { IterationNodeType } from '@/app/components/workflow/nodes/iteration/types'+import type { ListFilterNodeType } from '@/app/components/workflow/nodes/list-operator/types'+import type { Node } from '@/app/components/workflow/types'+import { act } from '@testing-library/react'+import { renderWorkflowHook } from '@/app/components/workflow/__tests__/workflow-test-env'+import { BlockEnum, ErrorHandleMode, VarType } from '@/app/components/workflow/types'+import useLastRun from '../use-last-run'++const mockHandleSyncWorkflowDraft = vi.fn()+const mockHandleRun = vi.fn()+const mockIterationChildren = vi.hoisted(() => vi.fn())++vi.mock('../../../../../../hooks/use-workflow', async (importOriginal) => {+ const actual = await importOriginal<typeof import('../../../../../../hooks/use-workflow')>()+ return {+ ...actual,+ useWorkflow: () => ({+ getIterationNodeChildren: mockIterationChildren,+ getBeforeNodesInSameBranch: () => [],+ }),+ useIsNodeInIteration: () => ({ isNodeInIteration: () => false }),+ }+})++vi.mock('../../../../../../hooks/use-nodes-sync-draft', async (importOriginal) => {+ const actual =+ await importOriginal<typeof import('../../../../../../hooks/use-nodes-sync-draft')>()++ return {+ ...actual,+ useNodesSyncDraft: () => ({+ handleSyncWorkflowDraft: mockHandleSyncWorkflowDraft,+ }),+ }+})++vi.mock('../../../../../../hooks/use-checklist', () => ({+ useWorkflowRunValidation: () => ({+ warningNodes: [],+ }),+}))++vi.mock('../../../../../../hooks/use-inspect-vars-crud', () => ({+ default: () => ({+ conversationVars: [],+ systemVars: [],+ hasSetInspectVar: vi.fn(() => false),+ }),+}))++vi.mock('@/app/components/workflow/nodes/_base/hooks/use-one-step-run', () => ({+ default: () => ({+ hideSingleRun: vi.fn(),+ handleRun: mockHandleRun,+ getInputVars: vi.fn(() => []),+ toVarInputs: vi.fn(() => []),+ varSelectorsToVarInputs: vi.fn(() => []),+ runInputData: {},+ runInputDataRef: { current: {} },+ setRunInputData: vi.fn(),+ showSingleRun: vi.fn(),+ runResult: {},+ iterationRunResult: [],+ loopRunResult: [],+ setNodeRunning: vi.fn(),+ checkValid: vi.fn(() => ({ isValid: true })),+ }),+}))++vi.mock('@/service/use-workflow', () => ({+ useInvalidateLastRun: () => vi.fn(),+}))++describe('useLastRun', () => {+ beforeEach(() => {+ vi.clearAllMocks()+ mockHandleSyncWorkflowDraft.mockReset()+ })++ it('submits list inputs from the real container form using selector keys', async () => {+ const child: Node<ListFilterNodeType> = {+ id: 'list',+ position: { x: 0, y: 0 },+ data: {+ type: BlockEnum.ListFilter,+ title: 'List',+ desc: '',+ variable: ['source', 'items'],+ var_type: VarType.arrayNumber,+ item_var_type: VarType.number,+ filter_by: {+ enabled: true,+ conditions: [{ key: '', comparison_operator: '≥', value: '{{#source.minimum#}}' }],+ },+ extract_by: { enabled: true, serial: '{{#source.serial#}}' },+ order_by: { enabled: false, key: '', value: 'asc' },+ limit: { enabled: false },+ },+ }+ mockIterationChildren.mockReturnValue([child])+ mockHandleSyncWorkflowDraft.mockImplementation((_force, _sync, options) => options.onSuccess())+ const data: IterationNodeType = {+ type: BlockEnum.Iteration,+ title: 'Iteration',+ desc: '',+ start_node_id: 'iteration-start',+ iterator_selector: ['source', 'items'],+ iterator_input_type: VarType.arrayNumber,+ output_selector: ['list', 'result'],+ output_type: VarType.arrayNumber,+ is_parallel: false,+ parallel_nums: 1,+ error_handle_mode: ErrorHandleMode.Terminated,+ flatten_output: false,+ _children: [{ nodeId: 'list', nodeType: BlockEnum.ListFilter }],+ _isShowTips: false,+ }+ const { result } = renderWorkflowHook(() =>+ useLastRun({+ id: 'iteration',+ resource: { kind: 'app', appId: 'flow-id', mode: 'workflow' },+ data,+ defaultRunInputData: {},+ isPaused: false,+ }),+ )++ await act(async () => {+ await result.current.handleRunWithParams({+ 'source.items': [1, 2],+ 'source.minimum': 0,+ 'source.serial': 1,+ 'iteration.input_selector': [1, 2],+ })+ })++ expect(mockHandleRun).toHaveBeenCalledWith({+ 'list.#source.items#': [1, 2],+ 'list.#source.minimum#': 0,+ 'list.#source.serial#': 1,+ 'iteration.input_selector': [1, 2],+ })+ })+})diff --git a/web/app/components/workflow/panel/__tests__/human-input-form-list.spec.tsx b/web/app/components/workflow/panel/__tests__/human-input-form-list.spec.tsxindex d227d9f3e4c..ec38e11b6f1 100644--- a/web/app/components/workflow/panel/__tests__/human-input-form-list.spec.tsx+++ b/web/app/components/workflow/panel/__tests__/human-input-form-list.spec.tsx@@ -127,6 +127,7 @@ describe('HumanInputFormList', () => {expect(screen.getByText('Need Approval')).toBeInTheDocument()expect(screen.queryByText('Hidden Form')).not.toBeInTheDocument()expect(screen.getByDisplayValue('prefill')).toBeInTheDocument()+ expect(screen.getByText('workflow.common.humanInputEmailTipInDebugMode')).toBeInTheDocument()expect(screen.getByText('workflow.common.humanInputWebappTip')).toBeInTheDocument()await user.clear(screen.getByDisplayValue('prefill'))@@ -152,7 +153,7 @@ describe('HumanInputFormList', () => {render(<HumanInputFormList humanInputFormDataList={[createFormData()]} />)- expect(screen.queryByTestId('tips')).not.toBeInTheDocument()+ expect(screen.queryByText(/workflow\.common\.humanInput.*Tip/)).not.toBeInTheDocument()})it('should reset inputs when the same node produces a new form', async () => {diff --git a/web/app/components/workflow/panel/__tests__/workflow-preview.spec.tsx b/web/app/components/workflow/panel/__tests__/workflow-preview.spec.tsxindex 0776c4fc38c..35752c00852 100644--- a/web/app/components/workflow/panel/__tests__/workflow-preview.spec.tsx+++ b/web/app/components/workflow/panel/__tests__/workflow-preview.spec.tsx@@ -9,6 +9,7 @@ import {createStartNode,createWorkflowRunningData,} from '@/app/components/workflow/__tests__/fixtures'+import { UserActionButtonType } from '@/app/components/workflow/nodes/human-input/types'import { InputVarType, WorkflowRunningStatus } from '@/app/components/workflow/types'import { toast } from '@/app/notifications'import { submitHumanInputForm } from '@/service/workflow'@@ -72,7 +73,7 @@ const createHumanInputFormData = (default: { type: 'constant', value: '', selector: [] },},],- actions: [{ id: 'approve', title: 'Approve', button_style: 'primary' }],+ actions: [{ id: 'approve', title: 'Approve', button_style: UserActionButtonType.Primary }],form_token: 'token-1',resolved_default_values: {},display_in_ui: true,@@ -84,8 +85,8 @@ const createHumanInputFilledFormData = (overrides: Partial<HumanInputFilledFormData> = {},): HumanInputFilledFormData => ({node_id: 'node-1',- node_title: 'Need Approval',- rendered_content: 'rendered',+ node_title: 'Previous approval',+ rendered_content: 'Already approved',action_id: 'approve',action_text: 'Approve',...overrides,@@ -171,7 +172,8 @@ it('opens pending human input on pause and submits the token with the entered re}),},})- expect(await screen.findByText('rendered')).toBeInTheDocument()+ expect(screen.getByText('Need Approval')).toBeInTheDocument()+ expect(await screen.findByText('Already approved', {}, { timeout: 5000 })).toBeInTheDocument()await user.type(screen.getByRole('textbox', { name: 'reason' }), 'Looks good')await user.click(screen.getByRole('button', { name: 'Approve' }))await waitFor(() =>diff --git a/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/handle-resume.spec.ts b/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/handle-resume.spec.tsindex 3e3ee7b0235..06f91252631 100644--- a/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/handle-resume.spec.ts+++ b/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/handle-resume.spec.ts@@ -796,19 +796,6 @@ describe('useChat – handleResume', () => {expect.objectContaining({ node_id: 'rn-human', submitted_data: { a: 'one' } }),])})-- it('should initialize humanInputFilledFormDataList when not present', async () => {- const { result } = await setupResumeWithTree()-- act(() => {- capturedResumeOptions.onHumanInputFormFilled({- data: { node_id: 'rn-human', submitted_data: { b: 'two' } },- })- })-- const answer = result.current.chatList.find((item) => item.id === 'msg-resume')- expect(answer!.humanInputFilledFormDataList).toHaveLength(1)- })})describe('onHumanInputFormTimeout', () => {diff --git a/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/misc.spec.ts b/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/misc.spec.tsindex b3631affc4e..e0b73cc96c2 100644--- a/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/misc.spec.ts+++ b/web/app/components/workflow/panel/debug-and-preview/__tests__/hooks/misc.spec.ts@@ -138,18 +138,6 @@ describe('useChat – handleSwitchSibling', () => {expect(mockSseGet).not.toHaveBeenCalled()})- it('should return undefined from findMessageInTree when not found', () => {- const { result } = renderHook(() => useChat({}), {- resource: { kind: 'app', appId: 'app-1', mode: 'workflow' },- })-- act(() => {- result.current.handleSwitchSibling('nonexistent-id', {})- })-- expect(mockSseGet).not.toHaveBeenCalled()- })-it('should search children recursively in findMessageInTree', async () => {let sendCallbacks: anymockHandleRun.mockImplementation((_params: unknown, callbacks: any) => {diff --git a/web/features/skills/__tests__/detail-page-builder.spec.tsx b/web/features/skills/__tests__/detail-page-builder.spec.tsxindex 51f971b03f1..db18b44c9e6 100644--- a/web/features/skills/__tests__/detail-page-builder.spec.tsx+++ b/web/features/skills/__tests__/detail-page-builder.spec.tsx@@ -590,9 +590,12 @@ describe('SkillDetailPage builder', () => {)renderSkillDetailPage()+ const builder = await screen.findByRole('region', {+ name: 'skill.skillManagement.detail.builder.title',+ })// The settings button waits for the model list and its lazily loaded contract segment.await user.click(- await screen.findByRole(+ await within(builder).findByRole('button',{ name: 'modelProvider.modelProvider.modelSettings' },{ timeout: 5000 },@@ -605,10 +608,12 @@ describe('SkillDetailPage builder', () => {await user.clear(temperature)await user.type(temperature, '1.2')await user.click(within(dialog).getByRole('button', { name: 'common.operation.close' }))- await user.type(+ await waitFor(() => expect(dialog).not.toBeInTheDocument())+ await user.click(screen.getByPlaceholderText('skill.skillManagement.detail.builder.modifyPlaceholder'),- 'Update the skill{Enter}',)+ await user.paste('Update the skill')+ await user.keyboard('{Enter}')expect(mocks.sendSkillAssistMessage).toHaveBeenCalledWith(expect.objectContaining({