Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

54.2 KB
Raw Blame
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path
from uuid import uuid4

import pytest
from sqlalchemy import Engine, select
from sqlalchemy.orm import Session

from clients.agent_backend import FakeAgentBackendRunClient, FakeAgentBackendScenario
from core.app.apps.pipeline.pipeline_runner import PipelineRunner
from core.app.apps.workflow.app_config_manager import WorkflowAppConfig
from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager
from core.app.apps.workflow.app_runner import WorkflowAppRunner
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity
from core.app.entities.queue_entities import (
    QueueNodeStartedEvent,
    QueueWorkflowFailedEvent,
    QueueWorkflowPausedEvent,
    QueueWorkflowSucceededEvent,
)
from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext
from core.app.layers.trigger_post_layer import TriggerPostLayer
from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository
from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
from core.workflow.nodes.human_input.entities import HumanInputNodeData, ParagraphInputConfig, UserActionConfig
from core.workflow.nodes.human_input.pause_reason import HumanInputRequired
from extensions.ext_storage import storage
from extensions.storage.opendal_storage import OpenDALStorage
from graphon.engine_events import (
    GraphRunAbortedEvent,
    GraphRunFailedEvent,
    GraphRunPartialSucceededEvent,
    GraphRunPausedEvent,
    GraphRunStartedEvent,
    GraphRunSucceededEvent,
)
from graphon.entities.pause_reason import HitlRequired
from graphon.enums import WorkflowExecutionStatus
from graphon.nodes.start.entities import StartNodeData
from graphon.runtime import RuntimeState
from graphon.variable_loader import DUMMY_VARIABLE_LOADER
from models import Account
from models.agent import AgentWorkspace, AgentWorkspaceBinding, WorkflowAgentBindingType
from models.dataset import Document
from models.enums import AppTriggerType, CreatorUserRole, WorkflowRunTriggeredFrom, WorkflowTriggerStatus
from models.human_input import HumanInputForm
from models.model import App, AppMode
from models.tools import WorkflowToolProvider
from models.trigger import WorkflowTriggerLog
from models.workflow import (
    Workflow,
    WorkflowNodeExecutionModel,
    WorkflowNodeExecutionTriggeredFrom,
    WorkflowPause,
    WorkflowRun,
)
from models.workflow import WorkflowType as ModelWorkflowType
from repositories.human_input_v2.form_repository import Form
from services.agent.workspace_service import AgentWorkspaceService
from services.workflow.entities import WorkflowScheduleCFSPlanEntity
from services.workflow_run_agg import WorkflowRunAgg
from tasks.workflow_cfs_scheduler.cfs_scheduler import AsyncWorkflowCFSPlanEntity
from tasks.workflow_cfs_scheduler.entities import AsyncWorkflowQueue
from tests.unit_tests.core.app.apps.pipeline.test_pipeline_runner import (
    _build_app_generate_entity,
    _build_runner,
    _document,
    _persist_scope,
    _workflow,
)
from tests.unit_tests.core.workflow.nodes.agent_v2.test_binding_resolver import _agent, _binding, _snapshot
from tests.unit_tests.core.workflow.nodes.agent_v2.test_session_store import _install_backend_client
from tests.unit_tests.core.workflow.nodes.human_input_v2.test_node import form as _v2_form_fixture

"""Pipeline tool lifecycle through the real Engine and SQLAlchemy repositories."""


def _pipeline_tool_runner(
    sqlite_engine: Engine, invoke_from: InvokeFrom, source_node_data: dict[str, object]
) -> tuple[PipelineRunner, SQLAlchemyWorkflowExecutionRepository, SQLAlchemyWorkflowNodeExecutionRepository]:
    source_graph = {
        "nodes": [
            {"id": "source-start", "data": {"type": "start", "title": "Start", "variables": list[object]()}},
            {"id": "source-node", "data": source_node_data},
            {"id": "source-end", "data": {"type": "end", "title": "End", "outputs": list[object]()}},
        ],
        "edges": [
            {"source": "source-start", "target": "source-node"},
            {"source": "source-node", "target": "source-end", "sourceHandle": "source"},
        ],
    }
    with Session(sqlite_engine, expire_on_commit=False) as session:
        session.add(
            App(
                id="source-app",
                tenant_id="tenant",
                name="Source",
                mode=AppMode.WORKFLOW,
                enable_site=False,
                enable_api=False,
            )
        )
        session.add(
            Workflow(
                id="source-workflow",
                tenant_id="tenant",
                app_id="source-app",
                type=ModelWorkflowType.WORKFLOW,
                version="v1",
                graph=json.dumps(source_graph),
                _features="{}",
                created_by="user",
            )
        )
        provider = WorkflowToolProvider(
            tenant_id="tenant",
            app_id="source-app",
            user_id="user",
            version="v1",
            name="source",
            label="Source",
            description="Source",
            icon="{}",
        )
        session.add(provider)
        session.flush()
        graph = {
            "nodes": [
                {"id": "start", "data": {"type": "start", "title": "Start", "variables": list[object]()}},
                {
                    "id": "tool",
                    "data": {
                        "type": "tool",
                        "title": "Tool",
                        "provider_id": provider.id,
                        "provider_type": "workflow",
                        "provider_name": "source",
                        "tool_name": "source",
                        "tool_label": "Source",
                        "tool_configurations": dict[str, object](),
                        "tool_parameters": dict[str, object](),
                    },
                },
            ],
            "edges": [{"source": "start", "target": "tool"}],
        }
        _, _, workflow = _persist_scope(session, workflow=_workflow(graph=graph), documents=(_document(),))
    entity = _build_app_generate_entity().model_copy(
        update={"invoke_from": invoke_from, "document_id": None if invoke_from == InvokeFrom.DEBUGGER else "doc"}
    )
    runner = _build_runner(entity, sqlite_engine, workflow)
    user = Account(name="User", email="[email protected]")
    user.id = entity.user_id
    runs = SQLAlchemyWorkflowExecutionRepository(
        sqlite_engine,
        "tenant",
        user,
        "pipe",
        WorkflowRunTriggeredFrom.RAG_PIPELINE_DEBUGGING
        if invoke_from == InvokeFrom.DEBUGGER
        else WorkflowRunTriggeredFrom.RAG_PIPELINE_RUN,
    )
    nodes = SQLAlchemyWorkflowNodeExecutionRepository(
        sqlite_engine, "tenant", user, "pipe", WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN
    )
    return runner, runs, nodes


class TestPipelineWorkflowToolsCases:
    @staticmethod
    @pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.PUBLISHED_PIPELINE])
    def test_pipeline_rejects_human_input_tool_before_creating_run_or_form(
        sqlite_engine: Engine, invoke_from: InvokeFrom
    ) -> None:
        human_input = HumanInputNodeData(
            title="Approval", user_actions=[UserActionConfig(id="approve", title="Approve")]
        )
        runner, runs, nodes = _pipeline_tool_runner(sqlite_engine, invoke_from, human_input.model_dump(mode="json"))

        with pytest.raises(ValueError, match="Human Input nodes are not supported"):
            WorkflowRunAgg.run(runner, None, runs, nodes)

        with Session(sqlite_engine) as session:
            assert session.scalar(select(WorkflowRun)) is None
            assert session.scalar(select(WorkflowNodeExecutionModel)) is None
            assert session.scalar(select(HumanInputForm)) is None
            document = session.get(Document, "doc")
            assert document is not None
            if invoke_from == InvokeFrom.PUBLISHED_PIPELINE:
                assert document.indexing_status == "error"
                assert document.error == "Human Input nodes are not supported in this execution context."
            else:
                assert document.indexing_status == "waiting"
                assert document.error is None

    @staticmethod
    @pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.PUBLISHED_PIPELINE])
    @pytest.mark.parametrize(
        ("scenario", "status"),
        [(FakeAgentBackendScenario.SUCCESS, "succeeded"), (FakeAgentBackendScenario.FAILED, "failed")],
    )
    def test_pipeline_retires_nested_agent_workspace_when_run_ends(
        sqlite_engine: Engine,
        monkeypatch: pytest.MonkeyPatch,
        invoke_from: InvokeFrom,
        scenario: FakeAgentBackendScenario,
        status: str,
    ) -> None:
        runner, runs, nodes = _pipeline_tool_runner(
            sqlite_engine,
            invoke_from,
            {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Agent"},
        )
        with Session(sqlite_engine) as session:
            agent = _agent(tenant_id="tenant")
            session.add(agent)
            session.flush()
            snapshot = _snapshot(tenant_id="tenant", agent_id=agent.id)
            session.add(snapshot)
            session.flush()
            session.add(
                _binding(
                    ids={
                        "tenant_id": "tenant",
                        "app_id": "source-app",
                        "workflow_id": "source-workflow",
                        "node_id": "source-node",
                    },
                    agent_id=agent.id,
                    snapshot_id=snapshot.id,
                    binding_type=WorkflowAgentBindingType.INLINE_AGENT,
                )
            )
            session.commit()
        backend_client = _install_backend_client(monkeypatch)
        monkeypatch.setattr(
            "clients.agent_backend.factory.create_agent_backend_run_client",
            lambda **_kwargs: FakeAgentBackendRunClient(scenario=scenario),
        )
        monkeypatch.setattr(
            "core.workflow.nodes.agent_v2.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None
        )
        collections: list[dict[str, object]] = []

        def collect_task(**kwargs: object) -> None:
            collections.append(kwargs)

        monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collect_task)

        WorkflowRunAgg.run(runner, None, runs, nodes)
        list(runner._queue_manager.listen())

        with Session(sqlite_engine) as session:
            run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
            assert run is not None
            assert run.status == status
            agent_execution = session.scalars(
                select(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.node_id == "source-node")
            ).one()
            assert agent_execution.status == status
            workspace = session.scalars(select(AgentWorkspace)).one()
            binding = session.scalars(select(AgentWorkspaceBinding)).one()
            assert (workspace.app_id, workspace.owner_id, workspace.status) == ("source-app", run.id, "retired")
            assert binding.status == "retired"
            workspace_id, binding_id = workspace.id, binding.id
        assert collections == [
            {
                "tenant_id": "tenant",
                "workspace_ids": [workspace_id],
                "binding_ids": [],
                "home_snapshot_ids": [],
                "purge_agent_ids": [],
            }
        ]
        AgentWorkspaceService.collect_retired_workspace(tenant_id="tenant", workspace_id=workspace_id)
        backend_client.destroy_execution_binding_sync.assert_called_once()
        with Session(sqlite_engine) as session:
            assert session.get(AgentWorkspace, workspace_id) is None
            assert session.get(AgentWorkspaceBinding, binding_id) is None


"""Workflow orchestration against real Engine, SQLAlchemy, and filesystem storage."""


def make_workflow_runner(
    sqlite_engine: Engine, *, human_input: bool = False
) -> tuple[WorkflowAppRunner, SQLAlchemyWorkflowExecutionRepository, SQLAlchemyWorkflowNodeExecutionRepository]:
    """Build the same runner/repositories/queue used by a workflow worker."""
    tenant_id, app_id, workflow_id, user_id, run_id = (str(uuid4()) for _ in range(5))
    graph: dict[str, list[dict[str, object]]] = {
        "nodes": [{"id": "start", "data": {"type": "start", **StartNodeData(title="Start").model_dump(mode="json")}}],
        "edges": [],
    }
    if human_input:
        data = HumanInputNodeData(
            title="Approval",
            form_content="Decision: {{#$output.answer#}}",
            inputs=[ParagraphInputConfig(output_variable_name="answer")],
            user_actions=[UserActionConfig(id="approve", title="Approve")],
        )
        graph["nodes"].append({"id": "approval", "data": {"type": "human-input", **data.model_dump(mode="json")}})
        graph["edges"].append({"id": "start-approval", "source": "start", "target": "approval"})
        graph["nodes"].append(
            {
                "id": "end",
                "data": {
                    "type": "end",
                    "title": "End",
                    "outputs": [
                        {"variable": "answer", "value_selector": ["approval", "answer"]},
                    ],
                },
            }
        )
        graph["edges"].append({"id": "approval-end", "source": "approval", "target": "end", "sourceHandle": "approve"})
    workflow = Workflow(
        id=workflow_id,
        tenant_id=tenant_id,
        app_id=app_id,
        type=ModelWorkflowType.WORKFLOW,
        version="draft",
        graph=json.dumps(graph),
        _features="{}",
        created_by=user_id,
    )
    entity = WorkflowAppGenerateEntity(
        user_from=UserFrom.ACCOUNT,
        task_id=str(uuid4()),
        app_config=WorkflowAppConfig(
            tenant_id=tenant_id, app_id=app_id, workflow_id=workflow_id, app_mode=AppMode.WORKFLOW
        ),
        inputs={},
        files=[],
        user_id=user_id,
        stream=True,
        invoke_from=InvokeFrom.DEBUGGER,
        workflow_execution_id=run_id,
    )
    user = Account(name="Workflow Tester", email=f"{user_id}@example.com")
    user.id = user_id
    queue = WorkflowAppQueueManager(entity.task_id, user_id, entity.invoke_from, AppMode.WORKFLOW)
    workflow_repository = SQLAlchemyWorkflowExecutionRepository(
        sqlite_engine,
        tenant_id=tenant_id,
        user=user,
        app_id=app_id,
        triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
    )
    node_repository = SQLAlchemyWorkflowNodeExecutionRepository(
        sqlite_engine,
        tenant_id=tenant_id,
        user=user,
        app_id=app_id,
        triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
    )
    runner = WorkflowAppRunner(
        application_generate_entity=entity,
        queue_manager=queue,
        variable_loader=DUMMY_VARIABLE_LOADER,
        workflow=workflow,
        system_user_id=user_id,
        execution_driver=WorkflowRunAgg.run,
    )
    return runner, workflow_repository, node_repository


class TestWorkflowRunAggCases:
    @staticmethod
    def test_driver_publishes_persisted_start_index_and_success(sqlite_engine: Engine) -> None:
        runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine)
        WorkflowRunAgg.run(runner, None, workflow_repository, node_repository)
        events = [message.event for message in runner._queue_manager.listen()]
        starts = [event for event in events if isinstance(event, QueueNodeStartedEvent)]
        assert len(starts) == 1
        assert isinstance(events[-1], QueueWorkflowSucceededEvent)
        with Session(sqlite_engine) as session:
            execution = session.scalar(select(WorkflowNodeExecutionModel))
            assert execution is not None
            assert execution.index == starts[0].node_run_index == 1
            run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
            assert run is not None
            assert run.status == "succeeded"

    @staticmethod
    @pytest.mark.parametrize("snapshot", [None, {}, {"tools": []}])
    def test_invalid_saved_tool_snapshot_is_rejected(sqlite_engine: Engine, snapshot: object) -> None:
        from pydantic import ValidationError

        runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine)
        runner.application_generate_entity.extras["workflow_tool_snapshot"] = snapshot
        with pytest.raises(ValidationError):
            WorkflowRunAgg.run(runner, None, workflow_repository, node_repository)
        with Session(sqlite_engine) as session:
            assert session.scalar(select(WorkflowRun)) is None

    @staticmethod
    @pytest.fixture
    def _local_storage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setattr(storage, "storage_runner", OpenDALStorage(scheme="fs", root=str(tmp_path)), raising=False)
        from core.app.apps import workflow_app_runner

        monkeypatch.setattr(
            workflow_app_runner.dispatch_human_input_form_delivery_task, "apply_async", lambda **_kwargs: None
        )

    @staticmethod
    @pytest.mark.usefixtures("_local_storage")
    @pytest.mark.parametrize("persist_snapshot", [False, True])
    def test_pause_publication_persists_summary_and_optional_snapshot(
        sqlite_engine: Engine, persist_snapshot: bool, caplog: pytest.LogCaptureFixture
    ) -> None:
        runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        config = (
            PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id)
            if persist_snapshot
            else None
        )
        WorkflowRunAgg.run(runner, config, workflow_repository, node_repository)
        assert "Failed to enqueue human input email task" not in caplog.text
        events = [message.event for message in runner._queue_manager.listen()]
        assert isinstance(events[-1], QueueWorkflowPausedEvent), events
        with Session(sqlite_engine) as session:
            run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
            assert run is not None
            pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run.id))
            assert run.status == "paused"
            assert run.finished_at is None
            assert run.total_steps == 2
            if not persist_snapshot:
                assert pause is None
                return
            assert pause is not None
            snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
            restored = RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state)
            assert restored.graph_execution.paused
            assert snapshot.get_response_stream_filter().dumps() == snapshot.serialized_response_stream_filter_state
            assert snapshot.get_generate_entity().model_dump(
                mode="json"
            ) == runner.application_generate_entity.model_dump(mode="json")

    @staticmethod
    @pytest.mark.usefixtures("_local_storage")
    def test_pause_database_failure_publishes_failure_instead_of_resume_ready(
        sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        def fail_pause(*_args: object, **_kwargs: object) -> None:
            raise RuntimeError("pause-write-failed")

        monkeypatch.setattr(
            "repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository.create_workflow_pause",
            fail_pause,
        )
        retirements: list[dict[str, object]] = []
        collections: list[dict[str, object]] = []

        def retire(_store: object, **kwargs: object) -> list[str]:
            retirements.append(kwargs)
            return ["workspace-1"]

        monkeypatch.setattr(
            "core.workflow.nodes.agent_v2.workspace_retirement_layer.WorkflowAgentWorkspaceStore.retire_workflow_run",
            retire,
        )
        monkeypatch.setattr(
            "core.workflow.nodes.agent_v2.workspace_retirement_layer.enqueue_agent_resource_collection",
            lambda **kwargs: collections.append(kwargs),
        )
        runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        trigger_log = WorkflowTriggerLog(
            tenant_id=runner.application_generate_entity.app_config.tenant_id,
            app_id=runner.application_generate_entity.app_config.app_id,
            workflow_id=runner._workflow.id,
            workflow_run_id=None,
            root_node_id=None,
            trigger_metadata="{}",
            trigger_type=AppTriggerType.TRIGGER_SCHEDULE,
            trigger_data="{}",
            inputs="{}",
            outputs=None,
            status=WorkflowTriggerStatus.RUNNING,
            error=None,
            queue_name="workflow",
            celery_task_id=None,
            created_by_role=CreatorUserRole.ACCOUNT,
            created_by=runner.application_generate_entity.user_id,
            elapsed_time=5.0,
        )
        with Session(sqlite_engine, expire_on_commit=False) as session:
            session.add(trigger_log)
            session.commit()
        start_time = datetime.now(UTC) - timedelta(seconds=10)
        plan = AsyncWorkflowCFSPlanEntity(
            schedule_strategy=WorkflowScheduleCFSPlanEntity.Strategy.Nop,
            queue=AsyncWorkflowQueue.PROFESSIONAL_QUEUE,
        )
        runner._graph_engine_layers = (TriggerPostLayer(plan, start_time, trigger_log.id),)
        WorkflowRunAgg.run(
            runner,
            PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id),
            workflow_repository,
            node_repository,
        )
        events = [message.event for message in runner._queue_manager.listen()]
        assert isinstance(events[-1], QueueWorkflowFailedEvent), events
        assert "pause-write-failed" in events[-1].error
        assert not any(isinstance(event, QueueWorkflowPausedEvent) for event in events)
        assert retirements == [
            {
                "tenant_id": runner.application_generate_entity.app_config.tenant_id,
                "app_id": runner.application_generate_entity.app_config.app_id,
                "workflow_run_id": runner.application_generate_entity.workflow_execution_id,
            }
        ]
        assert collections == [
            {
                "tenant_id": runner.application_generate_entity.app_config.tenant_id,
                "workspace_ids": ["workspace-1"],
            }
        ]
        with Session(sqlite_engine) as session:
            persisted_trigger = session.get(WorkflowTriggerLog, trigger_log.id)
            assert persisted_trigger is not None
            assert persisted_trigger.status == WorkflowTriggerStatus.FAILED
            assert persisted_trigger.finished_at is not None
            assert persisted_trigger.elapsed_time == pytest.approx(
                5.0 + (persisted_trigger.finished_at.replace(tzinfo=UTC) - start_time).total_seconds()
            )
            run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
            assert run is not None
            assert run.status == "failed"
            assert session.scalar(select(WorkflowPause)) is None

    @staticmethod
    def test_late_form_submission_is_refreshed_before_the_real_queue_closes(sqlite_session: Session) -> None:
        from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner
        from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent
        from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
        from core.workflow.nodes.human_input.enums import HumanInputFormStatus
        from graphon.engine_events import NodeRunStartedEvent
        from tests.unit_tests.core.app.apps.test_workflow_app_runner import (
            _make_paused_workflow,
            _resume_events,
            _save_form,
        )

        first = _save_form(sqlite_session, status=HumanInputFormStatus.SUBMITTED)
        late = _save_form(sqlite_session, status=HumanInputFormStatus.WAITING)
        queue = WorkflowAppQueueManager("form-refresh-task", "user", InvokeFrom.DEBUGGER, AppMode.WORKFLOW)
        runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")
        entry = _make_paused_workflow(runner, [first, late])
        for event in _resume_events(runner, entry):
            runner.handle_event(entry, event)
            if isinstance(event, NodeRunStartedEvent):
                HumanInputFormSubmissionRepository().mark_submitted(
                    form_id=late.id,
                    recipient_id=None,
                    selected_action_id="approve",
                    form_data={"answer": "late answer"},
                    submission_user_id=None,
                    submission_end_user_id=None,
                )
        events = [message.event for message in queue.listen()]
        completions = [event for event in events if isinstance(event, QueueHumanInputFormFilledEvent)]
        assert [event.form_id for event in completions] == [first.id, late.id]
        assert completions[-1].rendered_content == "Decision: late answer"
        assert isinstance(events[-1], QueueWorkflowSucceededEvent)

    @staticmethod
    @pytest.mark.usefixtures("_local_storage")
    def test_durable_pause_resumes_with_one_form_completion_and_continuing_indexes(sqlite_engine: Engine) -> None:
        from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent, QueueWorkflowStartedEvent
        from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
        from models.human_input import HumanInputForm

        original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        config = PauseStateLayerConfig(sqlite_engine, original.application_generate_entity.user_id)
        WorkflowRunAgg.run(original, config, workflow_repository, node_repository)
        before = [message.event for message in original._queue_manager.listen()]
        assert isinstance(before[-1], QueueWorkflowPausedEvent)
        with Session(sqlite_engine) as session:
            pause = session.scalar(select(WorkflowPause))
            form = session.scalar(select(HumanInputForm))
            assert pause is not None
            assert form is not None
            form_id = form.id
            snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
        HumanInputFormSubmissionRepository().mark_submitted(
            form_id=form_id,
            recipient_id=None,
            selected_action_id="approve",
            form_data={"answer": "approved result"},
            submission_user_id=None,
            submission_end_user_id=None,
        )
        entity = original.application_generate_entity.model_copy(update={"task_id": str(uuid4())})
        queue = WorkflowAppQueueManager(entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW)
        resumed = WorkflowAppRunner(
            application_generate_entity=entity,
            queue_manager=queue,
            variable_loader=DUMMY_VARIABLE_LOADER,
            workflow=original._workflow,
            system_user_id=entity.user_id,
            graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),
            response_stream_filter=snapshot.get_response_stream_filter(),
            execution_driver=WorkflowRunAgg.run,
        )
        WorkflowRunAgg.run(resumed, config, workflow_repository, node_repository)
        after = [message.event for message in queue.listen()]
        assert isinstance(after[0], QueueWorkflowStartedEvent)
        assert len(after[0].node_execution_snapshots) == 2
        completions = [event for event in after if isinstance(event, QueueHumanInputFormFilledEvent)]
        assert [event.form_id for event in completions] == [form_id]
        starts = [event for event in after if isinstance(event, QueueNodeStartedEvent)]
        assert [(event.node_id, event.node_run_index) for event in starts] == [("approval", 2), ("end", 3)]
        assert isinstance(after[-1], QueueWorkflowSucceededEvent)
        assert after[-1].outputs == {"answer": "approved result"}
        with Session(sqlite_engine) as session:
            executions = session.scalars(
                select(WorkflowNodeExecutionModel).order_by(WorkflowNodeExecutionModel.index)
            ).all()
            assert [(row.node_id, row.index) for row in executions] == [("start", 1), ("approval", 2), ("end", 3)]

    @staticmethod
    @pytest.mark.usefixtures("_local_storage")
    def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_engine: Engine) -> None:
        from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent
        from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
        from graphon.nodes.tool.entities import ToolNodeData
        from models.human_input import HumanInputForm
        from models.model import App
        from models.tools import WorkflowToolProvider

        original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        original_entity = original.application_generate_entity
        tenant_id = original_entity.app_config.tenant_id
        source_app_id = str(uuid4())
        source_workflow_ids = [str(uuid4()), str(uuid4())]
        with Session(sqlite_engine) as session, session.begin():
            session.add(
                App(
                    id=source_app_id,
                    tenant_id=tenant_id,
                    name="Source",
                    mode=AppMode.WORKFLOW,
                    enable_site=False,
                    enable_api=False,
                )
            )
            for workflow_id, version, marker in zip(
                source_workflow_ids, ("old", "new"), ("original", "republished"), strict=True
            ):
                session.add(
                    Workflow(
                        id=workflow_id,
                        app_id=source_app_id,
                        tenant_id=tenant_id,
                        created_by=original_entity.user_id,
                        type=ModelWorkflowType.WORKFLOW,
                        version=version,
                        _features="{}",
                        graph=json.dumps(
                            {
                                "nodes": [
                                    {
                                        "id": "source-start",
                                        "data": {
                                            "type": "start",
                                            "title": "Start",
                                            "variables": [
                                                {
                                                    "variable": "marker",
                                                    "type": "text-input",
                                                    "label": "Marker",
                                                    "default": marker,
                                                    "required": False,
                                                }
                                            ],
                                        },
                                    },
                                    {
                                        "id": "source-end",
                                        "data": {
                                            "type": "end",
                                            "title": "End",
                                            "outputs": [
                                                {"variable": "marker", "value_selector": ["source-start", "marker"]}
                                            ],
                                        },
                                    },
                                ],
                                "edges": [{"source": "source-start", "target": "source-end"}],
                            }
                        ),
                    )
                )
            provider = WorkflowToolProvider(
                tenant_id=tenant_id,
                app_id=source_app_id,
                version="old",
                name="source",
                label="Original source",
                icon="{}",
                user_id=original_entity.user_id,
                description="Source",
                parameter_configuration='[{"name":"marker","description":"Marker","form":"llm"}]',
            )
            session.add(provider)
            session.flush()
            provider_id = provider.id
        graph = dict(original._workflow.graph_dict)
        graph["nodes"].append(
            {
                "id": "tool",
                "data": {
                    "type": "tool",
                    **ToolNodeData(
                        title="Tool",
                        provider_id=provider_id,
                        provider_type="workflow",
                        provider_name="source",
                        tool_name="source",
                        tool_label="Source",
                        tool_configurations={},
                        tool_parameters={},
                    ).model_dump(mode="json"),
                },
            }
        )
        graph["nodes"][2]["data"]["outputs"] = [{"variable": "result", "value_selector": ["tool", "json"]}]
        graph["edges"][1]["target"] = "tool"
        graph["edges"].append({"source": "tool", "target": "end"})
        original._workflow.graph = json.dumps(graph)
        config = PauseStateLayerConfig(sqlite_engine, original_entity.user_id)
        WorkflowRunAgg.run(original, config, workflow_repository, node_repository)
        before = [message.event for message in original._queue_manager.listen()]
        assert isinstance(before[-1], QueueWorkflowPausedEvent)
        assert not any(isinstance(event, QueueNodeStartedEvent) and event.node_id == "tool" for event in before)
        with Session(sqlite_engine) as session, session.begin():
            pause = session.scalar(select(WorkflowPause))
            form = session.scalar(select(HumanInputForm))
            provider = session.get(WorkflowToolProvider, provider_id)
            assert pause is not None
            assert form is not None
            assert provider is not None
            form_id = form.id
            snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
            provider.version = "new"
            provider.label = "Republished source"
        HumanInputFormSubmissionRepository().mark_submitted(
            form_id=form_id,
            recipient_id=None,
            selected_action_id="approve",
            form_data={"answer": "approved"},
            submission_user_id=None,
            submission_end_user_id=None,
        )
        entity = snapshot.get_generate_entity().model_copy(update={"task_id": str(uuid4())})
        assert isinstance(entity, WorkflowAppGenerateEntity)
        queue = WorkflowAppQueueManager(entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW)
        resumed = WorkflowAppRunner(
            application_generate_entity=entity,
            queue_manager=queue,
            variable_loader=DUMMY_VARIABLE_LOADER,
            workflow=original._workflow,
            system_user_id=entity.user_id,
            graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),
            response_stream_filter=snapshot.get_response_stream_filter(),
            execution_driver=WorkflowRunAgg.run,
        )
        WorkflowRunAgg.run(resumed, config, workflow_repository, node_repository)
        after = [message.event for message in queue.listen()]
        assert [event.form_id for event in after if isinstance(event, QueueHumanInputFormFilledEvent)] == [form_id]
        assert isinstance(after[-1], QueueWorkflowSucceededEvent), after
        assert after[-1].outputs == {"result": [{"marker": "original"}]}
        assert entity.extras == original_entity.extras
        with Session(sqlite_engine) as session:
            child_workflows = session.scalars(
                select(WorkflowNodeExecutionModel.workflow_id).where(WorkflowNodeExecutionModel.app_id == source_app_id)
            ).all()
        assert child_workflows
        assert set(child_workflows) == {source_workflow_ids[0]}

        # A new execution resolves the newly published graph and parameter defaults.
        graph["nodes"] = [node for node in graph["nodes"] if node["id"] != "approval"]
        graph["edges"] = [{"source": "start", "target": "tool"}, {"source": "tool", "target": "end"}]
        original._workflow.graph = json.dumps(graph)
        fresh_entity = original_entity.model_copy(
            update={"task_id": str(uuid4()), "workflow_execution_id": str(uuid4()), "extras": {}}
        )
        fresh_queue = WorkflowAppQueueManager(
            fresh_entity.task_id, fresh_entity.user_id, fresh_entity.invoke_from, AppMode.WORKFLOW
        )
        fresh = WorkflowAppRunner(
            application_generate_entity=fresh_entity,
            queue_manager=fresh_queue,
            variable_loader=DUMMY_VARIABLE_LOADER,
            workflow=original._workflow,
            system_user_id=fresh_entity.user_id,
            execution_driver=WorkflowRunAgg.run,
        )
        WorkflowRunAgg.run(fresh, None, workflow_repository, node_repository)
        fresh_events = [message.event for message in fresh_queue.listen()]
        assert isinstance(fresh_events[-1], QueueWorkflowSucceededEvent), fresh_events
        assert fresh_events[-1].outputs == {"result": [{"marker": "republished"}]}

    @staticmethod
    def test_active_execution_snapshot_failure_is_not_published_as_a_pause(sqlite_engine: Engine) -> None:
        from graphon.engine_events import GraphRunFailedEvent, GraphRunPausedEvent

        runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        prepared = runner.prepare()
        assert prepared is not None
        aggregate = WorkflowRunAgg(
            prepared,
            runner,
            PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id),
            workflow_execution_repository=workflow_repository,
            workflow_node_execution_repository=node_repository,
        )
        # A participant that has not shut down must prevent a resume-ready snapshot.
        with prepared.entry.graph_engine.runtime_state.graph_execution.track_execution():
            events = list(aggregate.iter_events())
        assert isinstance(events[-1], GraphRunFailedEvent)
        assert "during active execution" in events[-1].error
        assert not any(isinstance(event, GraphRunPausedEvent) for event in events)
        with Session(sqlite_engine) as session:
            run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
            assert run is not None
            assert run.status == "failed"
            assert session.scalar(select(WorkflowPause)) is None

    @staticmethod
    @pytest.mark.usefixtures("_local_storage")
    @pytest.mark.parametrize(
        ("terminal_event", "status"),
        [
            (GraphRunSucceededEvent(), WorkflowExecutionStatus.SUCCEEDED),
            (GraphRunPartialSucceededEvent(exceptions_count=1), WorkflowExecutionStatus.PARTIAL_SUCCEEDED),
            (GraphRunFailedEvent(error="failed"), WorkflowExecutionStatus.FAILED),
            (GraphRunAbortedEvent(reason="stopped"), WorkflowExecutionStatus.STOPPED),
        ],
    )
    def test_elapsed_time_accumulates_across_pauses_and_matches_delayed_stream_responses(
        monkeypatch: pytest.MonkeyPatch,
        sqlite_engine: Engine,
        terminal_event: GraphRunSucceededEvent
        | GraphRunPartialSucceededEvent
        | GraphRunFailedEvent
        | GraphRunAbortedEvent,
        status: WorkflowExecutionStatus,
    ) -> None:
        from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
        from core.app.entities.task_entities import WorkflowPauseStreamResponse
        from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
        from core.workflow.nodes.human_input.enums import HumanInputFormStatus
        from core.workflow.system_variables import build_system_variables
        from graphon.entities import WorkflowStartReason
        from models.human_input import HumanInputForm

        clock = [100.0]
        monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])
        original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        graph = original._workflow.graph_dict
        graph["nodes"].append({"id": "second-approval", "data": graph["nodes"][1]["data"]})
        graph["edges"][1]["target"] = "second-approval"
        graph["edges"].append(
            {"id": "second-end", "source": "second-approval", "target": "end", "sourceHandle": "approve"}
        )
        original._workflow.graph = json.dumps(graph)
        runner = original
        run_id = runner.application_generate_entity.workflow_execution_id
        config = PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id)
        account = Account(name="Test", email="[email protected]")
        account.id = runner.application_generate_entity.user_id
        total = 0.0

        for index, duration in enumerate([2.0, 3.0, 4.0]):
            prepared = runner.prepare()
            assert prepared is not None
            aggregate = WorkflowRunAgg(
                prepared,
                runner,
                config,
                workflow_execution_repository=workflow_repository,
                workflow_node_execution_repository=node_repository,
            )
            is_paused = index < 2
            if is_paused:
                events = []
                for event in aggregate.iter_events():
                    if isinstance(event, GraphRunStartedEvent):
                        with Session(sqlite_engine) as session:
                            started = session.get(WorkflowRun, run_id)
                            assert started is not None
                            assert started.elapsed_time == total
                        clock[0] += duration
                    events.append(event)
                assert isinstance(events[-1], GraphRunPausedEvent), events
            else:
                aggregate._persistence_layer.on_event(GraphRunStartedEvent())
                clock[0] += duration
                aggregate._persistence_layer.on_event(terminal_event)
            total += duration
            with Session(sqlite_engine) as session:
                execution = session.get(WorkflowRun, run_id)
                assert execution is not None
                assert execution.elapsed_time == total
                assert execution.status == (WorkflowExecutionStatus.PAUSED if is_paused else status)
                assert (execution.finished_at is None) == is_paused

            # Queue processing and pause waits must not extend a settled segment.
            clock[0] += 1000
            entity = runner.application_generate_entity
            runtime = prepared.entry.graph_engine.runtime_state
            converter = WorkflowResponseConverter(
                application_generate_entity=entity,
                user=account,
                system_variables=build_system_variables(workflow_execution_id=run_id),
            )
            converter.workflow_start_to_stream_response(
                task_id=entity.task_id,
                workflow_run_id=run_id,
                workflow_id=original._workflow.id,
                reason=WorkflowStartReason.INITIAL if index == 0 else WorkflowStartReason.RESUMPTION,
            )
            if is_paused:
                responses = converter.workflow_pause_to_stream_response(
                    event=QueueWorkflowPausedEvent(), task_id=entity.task_id, graph_runtime_state=runtime
                )
                pause_response = responses[-1]
                assert isinstance(pause_response, WorkflowPauseStreamResponse)
                assert pause_response.data.elapsed_time == total
                with Session(sqlite_engine) as session:
                    pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run_id))
                    form = session.scalar(
                        select(HumanInputForm).where(
                            HumanInputForm.workflow_run_id == run_id,
                            HumanInputForm.status == HumanInputFormStatus.WAITING,
                        )
                    )
                    assert pause is not None
                    assert form is not None
                    form_id = form.id
                    snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
                entity = snapshot.get_generate_entity()
                assert isinstance(entity, WorkflowAppGenerateEntity)
                assert entity.workflow_execution_timing.elapsed_time == snapshot.elapsed_time == total
                HumanInputFormSubmissionRepository().mark_submitted(
                    form_id=form_id,
                    recipient_id=None,
                    selected_action_id="approve",
                    form_data={"answer": "approved result"},
                    submission_user_id=None,
                    submission_end_user_id=None,
                )
                entity.task_id = str(uuid4())
                runner = WorkflowAppRunner(
                    application_generate_entity=entity,
                    queue_manager=WorkflowAppQueueManager(
                        entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW
                    ),
                    variable_loader=DUMMY_VARIABLE_LOADER,
                    workflow=original._workflow,
                    system_user_id=entity.user_id,
                    graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),
                    response_stream_filter=snapshot.get_response_stream_filter(),
                    execution_driver=WorkflowRunAgg.run,
                )
            else:
                response = converter.workflow_finish_to_stream_response(
                    task_id=entity.task_id,
                    workflow_id=original._workflow.id,
                    status=status,
                    graph_runtime_state=runtime,
                )
                assert response.data.elapsed_time == 9.0
                assert entity.workflow_execution_timing.finish() == 9.0

    @staticmethod
    @pytest.mark.usefixtures("_local_storage")
    @pytest.mark.parametrize("stop_timing", ["before_summary", "during_upload"])
    def test_stop_wins_while_engine_is_persisting_a_pause(
        sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch, stop_timing: str
    ) -> None:
        from sqlalchemy.orm import sessionmaker

        from core.app.entities.queue_entities import QueueStopEvent
        from core.workflow.nodes.human_input.enums import HumanInputFormStatus
        from graphon.enums import WorkflowExecutionStatus
        from models.human_input import HumanInputForm
        from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository

        runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        entity = runner.application_generate_entity
        repository = DifyAPISQLAlchemyWorkflowRunRepository(sessionmaker(sqlite_engine))
        stop_calls = []

        def request_stop() -> None:
            admitted = repository.stop_paused_workflow_task(
                tenant_id=entity.app_config.tenant_id,
                app_id=entity.app_config.app_id,
                task_id=entity.task_id,
                owner=None,
            )
            assert admitted is not None
            stop_calls.append(admitted.status)

        if stop_timing == "during_upload":
            original_save = storage.save

            def save(key: str, data: bytes) -> None:
                if key.startswith("workflow-state-"):
                    request_stop()
                return original_save(key, data)

            monkeypatch.setattr(storage, "save", save)
        else:
            original_persist = WorkflowRunAgg._persist_pause

            def persist(aggregate: WorkflowRunAgg, event: GraphRunPausedEvent) -> None:
                request_stop()
                return original_persist(aggregate, event)

            monkeypatch.setattr(WorkflowRunAgg, "_persist_pause", persist)

        WorkflowRunAgg.run(
            runner, PauseStateLayerConfig(sqlite_engine, entity.user_id), workflow_repository, node_repository
        )
        events = [message.event for message in runner._queue_manager.listen()]
        assert stop_calls
        assert isinstance(events[-1], QueueStopEvent), events
        assert not any(isinstance(event, QueueWorkflowPausedEvent) for event in events)
        with Session(sqlite_engine) as session:
            run = session.get(WorkflowRun, entity.workflow_execution_id)
            assert run is not None
            assert run.status == WorkflowExecutionStatus.STOPPED
            assert run.stop_requested_at is not None
            assert run.task_id == entity.task_id
            assert session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run.id)) is None
            forms = session.scalars(select(HumanInputForm).where(HumanInputForm.workflow_run_id == run.id)).all()
            assert forms
            assert all(form.status == HumanInputFormStatus.EXPIRED for form in forms)

    @staticmethod
    def test_legacy_caller_rejects_human_input_before_creating_run_or_form(sqlite_engine: Engine) -> None:
        from models.human_input import HumanInputForm

        runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
        runner.application_generate_entity.allow_human_input = False
        with pytest.raises(ValueError, match="Human Input nodes are not supported"):
            WorkflowRunAgg.run(runner, None, workflow_repository, node_repository)
        with Session(sqlite_engine) as session:
            assert session.scalar(select(WorkflowRun)) is None
            assert session.scalar(select(HumanInputForm)) is None


v2_form = _v2_form_fixture


@pytest.mark.parametrize("token", ["initiator-token", None])
def test_v2_pause_persists_only_checkpoint_references_and_publishes_frozen_form(
    sqlite_engine: Engine, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, token: str | None, v2_form: Form
) -> None:
    from dataclasses import replace

    from pydantic import SecretStr

    from core.human_input_v2.shared.values import TenantId
    from core.workflow.nodes.human_input_v2.entities import HumanInputNodeData as V2NodeData
    from core.workflow.nodes.human_input_v2.runtime import PreparedForm
    from models.workflow import WorkflowPauseReason

    monkeypatch.setattr(storage, "storage_runner", OpenDALStorage(scheme="fs", root=str(tmp_path)), raising=False)
    runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
    entity = runner.application_generate_entity
    graph = runner._workflow.graph_dict
    graph["nodes"][1]["data"] = V2NodeData.model_validate(
        {
            "type": "human-input",
            "version": "2",
            "title": "Approval",
            "form_content": "Changed content",
            "recipients_spec": [],
            "user_actions": [],
            "message_template": {"subject": "Notice", "body": "[Request URL]"},
            "debug_mode": {"enabled": False, "channels": []},
        }
    ).model_dump(mode="json")
    runner._workflow.graph = json.dumps(graph)
    source_form = v2_form
    prepared_forms: list[PreparedForm] = []

    def prepare(_runtime: object, *, node_execution_id: str, **_kwargs: object) -> PreparedForm:
        prepared = PreparedForm(
            replace(
                source_form,
                id=node_execution_id,
                node_execution_id=node_execution_id,
                tenant_id=TenantId(entity.app_config.tenant_id),
                app_id=entity.app_config.app_id,
                workflow_run_id=entity.workflow_execution_id,
            ),
            SecretStr(token) if token is not None else None,
        )
        prepared_forms.append(prepared)
        return prepared

    monkeypatch.setattr("services.human_input_v2.composition.WorkflowHumanInputRuntime.prepare_form", prepare)

    def reject_legacy_lookup(*_args: object, **_kwargs: object) -> None:
        pytest.fail("V2 forms must not be loaded from the legacy repository")

    monkeypatch.setattr(
        "core.repositories.human_input_repository.HumanInputFormSubmissionRepository.get_by_form_ids",
        reject_legacy_lookup,
    )
    WorkflowRunAgg.run(
        runner, PauseStateLayerConfig(sqlite_engine, entity.user_id), workflow_repository, node_repository
    )
    events = [message.event for message in runner._queue_manager.listen()]
    paused = events[-1]
    assert isinstance(paused, QueueWorkflowPausedEvent), events
    assert isinstance(paused.reasons[0], HumanInputRequired)
    assert paused.reasons[0].form_version == "2"
    assert paused.reasons[0].form_content == source_form.resolved_form.legacy_form_content
    assert paused.human_input_v2_forms[prepared_forms[0].form.id] is prepared_forms[0]
    with Session(sqlite_engine) as session:
        pause = session.scalar(
            select(WorkflowPause).where(WorkflowPause.workflow_run_id == entity.workflow_execution_id)
        )
        assert pause is not None
        assert list(session.scalars(select(WorkflowPauseReason).where(WorkflowPauseReason.pause_id == pause.id))) == []
        saved_state = storage.load(pause.state_object_key).decode()
    context = WorkflowResumptionContext.loads(saved_state)
    restored = RuntimeState.from_snapshot(context.serialized_graph_runtime_state)
    assert isinstance(restored.graph_execution.pause_reasons[0], HitlRequired)
    assert restored.graph_execution.pause_reasons[0].session_id == f"hitlv2:{prepared_forms[0].form.id}"
    if token is not None:
        assert token not in saved_state