Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path
from uuid import uuid4
import pytest
from sqlalchemy import Engine, select
from sqlalchemy.orm import Session
from clients.agent_backend import FakeAgentBackendRunClient, FakeAgentBackendScenario
from core.app.apps.pipeline.pipeline_runner import PipelineRunner
from core.app.apps.workflow.app_config_manager import WorkflowAppConfig
from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager
from core.app.apps.workflow.app_runner import WorkflowAppRunner
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity
from core.app.entities.queue_entities import (
QueueNodeStartedEvent,
QueueWorkflowFailedEvent,
QueueWorkflowPausedEvent,
QueueWorkflowSucceededEvent,
)
from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext
from core.app.layers.trigger_post_layer import TriggerPostLayer
from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository
from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
from core.workflow.nodes.human_input.entities import HumanInputNodeData, ParagraphInputConfig, UserActionConfig
from core.workflow.nodes.human_input.pause_reason import HumanInputRequired
from extensions.ext_storage import storage
from extensions.storage.opendal_storage import OpenDALStorage
from graphon.engine_events import (
GraphRunAbortedEvent,
GraphRunFailedEvent,
GraphRunPartialSucceededEvent,
GraphRunPausedEvent,
GraphRunStartedEvent,
GraphRunSucceededEvent,
)
from graphon.entities.pause_reason import HitlRequired
from graphon.enums import WorkflowExecutionStatus
from graphon.nodes.start.entities import StartNodeData
from graphon.runtime import RuntimeState
from graphon.variable_loader import DUMMY_VARIABLE_LOADER
from models import Account
from models.agent import AgentWorkspace, AgentWorkspaceBinding, WorkflowAgentBindingType
from models.dataset import Document
from models.enums import AppTriggerType, CreatorUserRole, WorkflowRunTriggeredFrom, WorkflowTriggerStatus
from models.human_input import HumanInputForm
from models.model import App, AppMode
from models.tools import WorkflowToolProvider
from models.trigger import WorkflowTriggerLog
from models.workflow import (
Workflow,
WorkflowNodeExecutionModel,
WorkflowNodeExecutionTriggeredFrom,
WorkflowPause,
WorkflowRun,
)
from models.workflow import WorkflowType as ModelWorkflowType
from repositories.human_input_v2.form_repository import Form
from services.agent.workspace_service import AgentWorkspaceService
from services.workflow.entities import WorkflowScheduleCFSPlanEntity
from services.workflow_run_agg import WorkflowRunAgg
from tasks.workflow_cfs_scheduler.cfs_scheduler import AsyncWorkflowCFSPlanEntity
from tasks.workflow_cfs_scheduler.entities import AsyncWorkflowQueue
from tests.unit_tests.core.app.apps.pipeline.test_pipeline_runner import (
_build_app_generate_entity,
_build_runner,
_document,
_persist_scope,
_workflow,
)
from tests.unit_tests.core.workflow.nodes.agent_v2.test_binding_resolver import _agent, _binding, _snapshot
from tests.unit_tests.core.workflow.nodes.agent_v2.test_session_store import _install_backend_client
from tests.unit_tests.core.workflow.nodes.human_input_v2.test_node import form as _v2_form_fixture
"""Pipeline tool lifecycle through the real Engine and SQLAlchemy repositories."""
def _pipeline_tool_runner(
sqlite_engine: Engine, invoke_from: InvokeFrom, source_node_data: dict[str, object]
) -> tuple[PipelineRunner, SQLAlchemyWorkflowExecutionRepository, SQLAlchemyWorkflowNodeExecutionRepository]:
source_graph = {
"nodes": [
{"id": "source-start", "data": {"type": "start", "title": "Start", "variables": list[object]()}},
{"id": "source-node", "data": source_node_data},
{"id": "source-end", "data": {"type": "end", "title": "End", "outputs": list[object]()}},
],
"edges": [
{"source": "source-start", "target": "source-node"},
{"source": "source-node", "target": "source-end", "sourceHandle": "source"},
],
}
with Session(sqlite_engine, expire_on_commit=False) as session:
session.add(
App(
id="source-app",
tenant_id="tenant",
name="Source",
mode=AppMode.WORKFLOW,
enable_site=False,
enable_api=False,
)
)
session.add(
Workflow(
id="source-workflow",
tenant_id="tenant",
app_id="source-app",
type=ModelWorkflowType.WORKFLOW,
version="v1",
graph=json.dumps(source_graph),
_features="{}",
created_by="user",
)
)
provider = WorkflowToolProvider(
tenant_id="tenant",
app_id="source-app",
user_id="user",
version="v1",
name="source",
label="Source",
description="Source",
icon="{}",
)
session.add(provider)
session.flush()
graph = {
"nodes": [
{"id": "start", "data": {"type": "start", "title": "Start", "variables": list[object]()}},
{
"id": "tool",
"data": {
"type": "tool",
"title": "Tool",
"provider_id": provider.id,
"provider_type": "workflow",
"provider_name": "source",
"tool_name": "source",
"tool_label": "Source",
"tool_configurations": dict[str, object](),
"tool_parameters": dict[str, object](),
},
},
],
"edges": [{"source": "start", "target": "tool"}],
}
_, _, workflow = _persist_scope(session, workflow=_workflow(graph=graph), documents=(_document(),))
entity = _build_app_generate_entity().model_copy(
update={"invoke_from": invoke_from, "document_id": None if invoke_from == InvokeFrom.DEBUGGER else "doc"}
)
runner = _build_runner(entity, sqlite_engine, workflow)
user = Account(name="User", email="[email protected]")
user.id = entity.user_id
runs = SQLAlchemyWorkflowExecutionRepository(
sqlite_engine,
"tenant",
user,
"pipe",
WorkflowRunTriggeredFrom.RAG_PIPELINE_DEBUGGING
if invoke_from == InvokeFrom.DEBUGGER
else WorkflowRunTriggeredFrom.RAG_PIPELINE_RUN,
)
nodes = SQLAlchemyWorkflowNodeExecutionRepository(
sqlite_engine, "tenant", user, "pipe", WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN
)
return runner, runs, nodes
class TestPipelineWorkflowToolsCases:
@staticmethod
@pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.PUBLISHED_PIPELINE])
def test_pipeline_rejects_human_input_tool_before_creating_run_or_form(
sqlite_engine: Engine, invoke_from: InvokeFrom
) -> None:
human_input = HumanInputNodeData(
title="Approval", user_actions=[UserActionConfig(id="approve", title="Approve")]
)
runner, runs, nodes = _pipeline_tool_runner(sqlite_engine, invoke_from, human_input.model_dump(mode="json"))
with pytest.raises(ValueError, match="Human Input nodes are not supported"):
WorkflowRunAgg.run(runner, None, runs, nodes)
with Session(sqlite_engine) as session:
assert session.scalar(select(WorkflowRun)) is None
assert session.scalar(select(WorkflowNodeExecutionModel)) is None
assert session.scalar(select(HumanInputForm)) is None
document = session.get(Document, "doc")
assert document is not None
if invoke_from == InvokeFrom.PUBLISHED_PIPELINE:
assert document.indexing_status == "error"
assert document.error == "Human Input nodes are not supported in this execution context."
else:
assert document.indexing_status == "waiting"
assert document.error is None
@staticmethod
@pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.PUBLISHED_PIPELINE])
@pytest.mark.parametrize(
("scenario", "status"),
[(FakeAgentBackendScenario.SUCCESS, "succeeded"), (FakeAgentBackendScenario.FAILED, "failed")],
)
def test_pipeline_retires_nested_agent_workspace_when_run_ends(
sqlite_engine: Engine,
monkeypatch: pytest.MonkeyPatch,
invoke_from: InvokeFrom,
scenario: FakeAgentBackendScenario,
status: str,
) -> None:
runner, runs, nodes = _pipeline_tool_runner(
sqlite_engine,
invoke_from,
{"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Agent"},
)
with Session(sqlite_engine) as session:
agent = _agent(tenant_id="tenant")
session.add(agent)
session.flush()
snapshot = _snapshot(tenant_id="tenant", agent_id=agent.id)
session.add(snapshot)
session.flush()
session.add(
_binding(
ids={
"tenant_id": "tenant",
"app_id": "source-app",
"workflow_id": "source-workflow",
"node_id": "source-node",
},
agent_id=agent.id,
snapshot_id=snapshot.id,
binding_type=WorkflowAgentBindingType.INLINE_AGENT,
)
)
session.commit()
backend_client = _install_backend_client(monkeypatch)
monkeypatch.setattr(
"clients.agent_backend.factory.create_agent_backend_run_client",
lambda **_kwargs: FakeAgentBackendRunClient(scenario=scenario),
)
monkeypatch.setattr(
"core.workflow.nodes.agent_v2.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None
)
collections: list[dict[str, object]] = []
def collect_task(**kwargs: object) -> None:
collections.append(kwargs)
monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collect_task)
WorkflowRunAgg.run(runner, None, runs, nodes)
list(runner._queue_manager.listen())
with Session(sqlite_engine) as session:
run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
assert run is not None
assert run.status == status
agent_execution = session.scalars(
select(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.node_id == "source-node")
).one()
assert agent_execution.status == status
workspace = session.scalars(select(AgentWorkspace)).one()
binding = session.scalars(select(AgentWorkspaceBinding)).one()
assert (workspace.app_id, workspace.owner_id, workspace.status) == ("source-app", run.id, "retired")
assert binding.status == "retired"
workspace_id, binding_id = workspace.id, binding.id
assert collections == [
{
"tenant_id": "tenant",
"workspace_ids": [workspace_id],
"binding_ids": [],
"home_snapshot_ids": [],
"purge_agent_ids": [],
}
]
AgentWorkspaceService.collect_retired_workspace(tenant_id="tenant", workspace_id=workspace_id)
backend_client.destroy_execution_binding_sync.assert_called_once()
with Session(sqlite_engine) as session:
assert session.get(AgentWorkspace, workspace_id) is None
assert session.get(AgentWorkspaceBinding, binding_id) is None
"""Workflow orchestration against real Engine, SQLAlchemy, and filesystem storage."""
def make_workflow_runner(
sqlite_engine: Engine, *, human_input: bool = False
) -> tuple[WorkflowAppRunner, SQLAlchemyWorkflowExecutionRepository, SQLAlchemyWorkflowNodeExecutionRepository]:
"""Build the same runner/repositories/queue used by a workflow worker."""
tenant_id, app_id, workflow_id, user_id, run_id = (str(uuid4()) for _ in range(5))
graph: dict[str, list[dict[str, object]]] = {
"nodes": [{"id": "start", "data": {"type": "start", **StartNodeData(title="Start").model_dump(mode="json")}}],
"edges": [],
}
if human_input:
data = HumanInputNodeData(
title="Approval",
form_content="Decision: {{#$output.answer#}}",
inputs=[ParagraphInputConfig(output_variable_name="answer")],
user_actions=[UserActionConfig(id="approve", title="Approve")],
)
graph["nodes"].append({"id": "approval", "data": {"type": "human-input", **data.model_dump(mode="json")}})
graph["edges"].append({"id": "start-approval", "source": "start", "target": "approval"})
graph["nodes"].append(
{
"id": "end",
"data": {
"type": "end",
"title": "End",
"outputs": [
{"variable": "answer", "value_selector": ["approval", "answer"]},
],
},
}
)
graph["edges"].append({"id": "approval-end", "source": "approval", "target": "end", "sourceHandle": "approve"})
workflow = Workflow(
id=workflow_id,
tenant_id=tenant_id,
app_id=app_id,
type=ModelWorkflowType.WORKFLOW,
version="draft",
graph=json.dumps(graph),
_features="{}",
created_by=user_id,
)
entity = WorkflowAppGenerateEntity(
user_from=UserFrom.ACCOUNT,
task_id=str(uuid4()),
app_config=WorkflowAppConfig(
tenant_id=tenant_id, app_id=app_id, workflow_id=workflow_id, app_mode=AppMode.WORKFLOW
),
inputs={},
files=[],
user_id=user_id,
stream=True,
invoke_from=InvokeFrom.DEBUGGER,
workflow_execution_id=run_id,
)
user = Account(name="Workflow Tester", email=f"{user_id}@example.com")
user.id = user_id
queue = WorkflowAppQueueManager(entity.task_id, user_id, entity.invoke_from, AppMode.WORKFLOW)
workflow_repository = SQLAlchemyWorkflowExecutionRepository(
sqlite_engine,
tenant_id=tenant_id,
user=user,
app_id=app_id,
triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
)
node_repository = SQLAlchemyWorkflowNodeExecutionRepository(
sqlite_engine,
tenant_id=tenant_id,
user=user,
app_id=app_id,
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
)
runner = WorkflowAppRunner(
application_generate_entity=entity,
queue_manager=queue,
variable_loader=DUMMY_VARIABLE_LOADER,
workflow=workflow,
system_user_id=user_id,
execution_driver=WorkflowRunAgg.run,
)
return runner, workflow_repository, node_repository
class TestWorkflowRunAggCases:
@staticmethod
def test_driver_publishes_persisted_start_index_and_success(sqlite_engine: Engine) -> None:
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine)
WorkflowRunAgg.run(runner, None, workflow_repository, node_repository)
events = [message.event for message in runner._queue_manager.listen()]
starts = [event for event in events if isinstance(event, QueueNodeStartedEvent)]
assert len(starts) == 1
assert isinstance(events[-1], QueueWorkflowSucceededEvent)
with Session(sqlite_engine) as session:
execution = session.scalar(select(WorkflowNodeExecutionModel))
assert execution is not None
assert execution.index == starts[0].node_run_index == 1
run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
assert run is not None
assert run.status == "succeeded"
@staticmethod
@pytest.mark.parametrize("snapshot", [None, {}, {"tools": []}])
def test_invalid_saved_tool_snapshot_is_rejected(sqlite_engine: Engine, snapshot: object) -> None:
from pydantic import ValidationError
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine)
runner.application_generate_entity.extras["workflow_tool_snapshot"] = snapshot
with pytest.raises(ValidationError):
WorkflowRunAgg.run(runner, None, workflow_repository, node_repository)
with Session(sqlite_engine) as session:
assert session.scalar(select(WorkflowRun)) is None
@staticmethod
@pytest.fixture
def _local_storage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(storage, "storage_runner", OpenDALStorage(scheme="fs", root=str(tmp_path)), raising=False)
from core.app.apps import workflow_app_runner
monkeypatch.setattr(
workflow_app_runner.dispatch_human_input_form_delivery_task, "apply_async", lambda **_kwargs: None
)
@staticmethod
@pytest.mark.usefixtures("_local_storage")
@pytest.mark.parametrize("persist_snapshot", [False, True])
def test_pause_publication_persists_summary_and_optional_snapshot(
sqlite_engine: Engine, persist_snapshot: bool, caplog: pytest.LogCaptureFixture
) -> None:
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
config = (
PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id)
if persist_snapshot
else None
)
WorkflowRunAgg.run(runner, config, workflow_repository, node_repository)
assert "Failed to enqueue human input email task" not in caplog.text
events = [message.event for message in runner._queue_manager.listen()]
assert isinstance(events[-1], QueueWorkflowPausedEvent), events
with Session(sqlite_engine) as session:
run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
assert run is not None
pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run.id))
assert run.status == "paused"
assert run.finished_at is None
assert run.total_steps == 2
if not persist_snapshot:
assert pause is None
return
assert pause is not None
snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
restored = RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state)
assert restored.graph_execution.paused
assert snapshot.get_response_stream_filter().dumps() == snapshot.serialized_response_stream_filter_state
assert snapshot.get_generate_entity().model_dump(
mode="json"
) == runner.application_generate_entity.model_dump(mode="json")
@staticmethod
@pytest.mark.usefixtures("_local_storage")
def test_pause_database_failure_publishes_failure_instead_of_resume_ready(
sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch
) -> None:
def fail_pause(*_args: object, **_kwargs: object) -> None:
raise RuntimeError("pause-write-failed")
monkeypatch.setattr(
"repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository.create_workflow_pause",
fail_pause,
)
retirements: list[dict[str, object]] = []
collections: list[dict[str, object]] = []
def retire(_store: object, **kwargs: object) -> list[str]:
retirements.append(kwargs)
return ["workspace-1"]
monkeypatch.setattr(
"core.workflow.nodes.agent_v2.workspace_retirement_layer.WorkflowAgentWorkspaceStore.retire_workflow_run",
retire,
)
monkeypatch.setattr(
"core.workflow.nodes.agent_v2.workspace_retirement_layer.enqueue_agent_resource_collection",
lambda **kwargs: collections.append(kwargs),
)
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
trigger_log = WorkflowTriggerLog(
tenant_id=runner.application_generate_entity.app_config.tenant_id,
app_id=runner.application_generate_entity.app_config.app_id,
workflow_id=runner._workflow.id,
workflow_run_id=None,
root_node_id=None,
trigger_metadata="{}",
trigger_type=AppTriggerType.TRIGGER_SCHEDULE,
trigger_data="{}",
inputs="{}",
outputs=None,
status=WorkflowTriggerStatus.RUNNING,
error=None,
queue_name="workflow",
celery_task_id=None,
created_by_role=CreatorUserRole.ACCOUNT,
created_by=runner.application_generate_entity.user_id,
elapsed_time=5.0,
)
with Session(sqlite_engine, expire_on_commit=False) as session:
session.add(trigger_log)
session.commit()
start_time = datetime.now(UTC) - timedelta(seconds=10)
plan = AsyncWorkflowCFSPlanEntity(
schedule_strategy=WorkflowScheduleCFSPlanEntity.Strategy.Nop,
queue=AsyncWorkflowQueue.PROFESSIONAL_QUEUE,
)
runner._graph_engine_layers = (TriggerPostLayer(plan, start_time, trigger_log.id),)
WorkflowRunAgg.run(
runner,
PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id),
workflow_repository,
node_repository,
)
events = [message.event for message in runner._queue_manager.listen()]
assert isinstance(events[-1], QueueWorkflowFailedEvent), events
assert "pause-write-failed" in events[-1].error
assert not any(isinstance(event, QueueWorkflowPausedEvent) for event in events)
assert retirements == [
{
"tenant_id": runner.application_generate_entity.app_config.tenant_id,
"app_id": runner.application_generate_entity.app_config.app_id,
"workflow_run_id": runner.application_generate_entity.workflow_execution_id,
}
]
assert collections == [
{
"tenant_id": runner.application_generate_entity.app_config.tenant_id,
"workspace_ids": ["workspace-1"],
}
]
with Session(sqlite_engine) as session:
persisted_trigger = session.get(WorkflowTriggerLog, trigger_log.id)
assert persisted_trigger is not None
assert persisted_trigger.status == WorkflowTriggerStatus.FAILED
assert persisted_trigger.finished_at is not None
assert persisted_trigger.elapsed_time == pytest.approx(
5.0 + (persisted_trigger.finished_at.replace(tzinfo=UTC) - start_time).total_seconds()
)
run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
assert run is not None
assert run.status == "failed"
assert session.scalar(select(WorkflowPause)) is None
@staticmethod
def test_late_form_submission_is_refreshed_before_the_real_queue_closes(sqlite_session: Session) -> None:
from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner
from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent
from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
from core.workflow.nodes.human_input.enums import HumanInputFormStatus
from graphon.engine_events import NodeRunStartedEvent
from tests.unit_tests.core.app.apps.test_workflow_app_runner import (
_make_paused_workflow,
_resume_events,
_save_form,
)
first = _save_form(sqlite_session, status=HumanInputFormStatus.SUBMITTED)
late = _save_form(sqlite_session, status=HumanInputFormStatus.WAITING)
queue = WorkflowAppQueueManager("form-refresh-task", "user", InvokeFrom.DEBUGGER, AppMode.WORKFLOW)
runner = WorkflowBasedAppRunner(queue_manager=queue, app_id="app")
entry = _make_paused_workflow(runner, [first, late])
for event in _resume_events(runner, entry):
runner.handle_event(entry, event)
if isinstance(event, NodeRunStartedEvent):
HumanInputFormSubmissionRepository().mark_submitted(
form_id=late.id,
recipient_id=None,
selected_action_id="approve",
form_data={"answer": "late answer"},
submission_user_id=None,
submission_end_user_id=None,
)
events = [message.event for message in queue.listen()]
completions = [event for event in events if isinstance(event, QueueHumanInputFormFilledEvent)]
assert [event.form_id for event in completions] == [first.id, late.id]
assert completions[-1].rendered_content == "Decision: late answer"
assert isinstance(events[-1], QueueWorkflowSucceededEvent)
@staticmethod
@pytest.mark.usefixtures("_local_storage")
def test_durable_pause_resumes_with_one_form_completion_and_continuing_indexes(sqlite_engine: Engine) -> None:
from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent, QueueWorkflowStartedEvent
from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
from models.human_input import HumanInputForm
original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
config = PauseStateLayerConfig(sqlite_engine, original.application_generate_entity.user_id)
WorkflowRunAgg.run(original, config, workflow_repository, node_repository)
before = [message.event for message in original._queue_manager.listen()]
assert isinstance(before[-1], QueueWorkflowPausedEvent)
with Session(sqlite_engine) as session:
pause = session.scalar(select(WorkflowPause))
form = session.scalar(select(HumanInputForm))
assert pause is not None
assert form is not None
form_id = form.id
snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
HumanInputFormSubmissionRepository().mark_submitted(
form_id=form_id,
recipient_id=None,
selected_action_id="approve",
form_data={"answer": "approved result"},
submission_user_id=None,
submission_end_user_id=None,
)
entity = original.application_generate_entity.model_copy(update={"task_id": str(uuid4())})
queue = WorkflowAppQueueManager(entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW)
resumed = WorkflowAppRunner(
application_generate_entity=entity,
queue_manager=queue,
variable_loader=DUMMY_VARIABLE_LOADER,
workflow=original._workflow,
system_user_id=entity.user_id,
graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),
response_stream_filter=snapshot.get_response_stream_filter(),
execution_driver=WorkflowRunAgg.run,
)
WorkflowRunAgg.run(resumed, config, workflow_repository, node_repository)
after = [message.event for message in queue.listen()]
assert isinstance(after[0], QueueWorkflowStartedEvent)
assert len(after[0].node_execution_snapshots) == 2
completions = [event for event in after if isinstance(event, QueueHumanInputFormFilledEvent)]
assert [event.form_id for event in completions] == [form_id]
starts = [event for event in after if isinstance(event, QueueNodeStartedEvent)]
assert [(event.node_id, event.node_run_index) for event in starts] == [("approval", 2), ("end", 3)]
assert isinstance(after[-1], QueueWorkflowSucceededEvent)
assert after[-1].outputs == {"answer": "approved result"}
with Session(sqlite_engine) as session:
executions = session.scalars(
select(WorkflowNodeExecutionModel).order_by(WorkflowNodeExecutionModel.index)
).all()
assert [(row.node_id, row.index) for row in executions] == [("start", 1), ("approval", 2), ("end", 3)]
@staticmethod
@pytest.mark.usefixtures("_local_storage")
def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_engine: Engine) -> None:
from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent
from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
from graphon.nodes.tool.entities import ToolNodeData
from models.human_input import HumanInputForm
from models.model import App
from models.tools import WorkflowToolProvider
original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
original_entity = original.application_generate_entity
tenant_id = original_entity.app_config.tenant_id
source_app_id = str(uuid4())
source_workflow_ids = [str(uuid4()), str(uuid4())]
with Session(sqlite_engine) as session, session.begin():
session.add(
App(
id=source_app_id,
tenant_id=tenant_id,
name="Source",
mode=AppMode.WORKFLOW,
enable_site=False,
enable_api=False,
)
)
for workflow_id, version, marker in zip(
source_workflow_ids, ("old", "new"), ("original", "republished"), strict=True
):
session.add(
Workflow(
id=workflow_id,
app_id=source_app_id,
tenant_id=tenant_id,
created_by=original_entity.user_id,
type=ModelWorkflowType.WORKFLOW,
version=version,
_features="{}",
graph=json.dumps(
{
"nodes": [
{
"id": "source-start",
"data": {
"type": "start",
"title": "Start",
"variables": [
{
"variable": "marker",
"type": "text-input",
"label": "Marker",
"default": marker,
"required": False,
}
],
},
},
{
"id": "source-end",
"data": {
"type": "end",
"title": "End",
"outputs": [
{"variable": "marker", "value_selector": ["source-start", "marker"]}
],
},
},
],
"edges": [{"source": "source-start", "target": "source-end"}],
}
),
)
)
provider = WorkflowToolProvider(
tenant_id=tenant_id,
app_id=source_app_id,
version="old",
name="source",
label="Original source",
icon="{}",
user_id=original_entity.user_id,
description="Source",
parameter_configuration='[{"name":"marker","description":"Marker","form":"llm"}]',
)
session.add(provider)
session.flush()
provider_id = provider.id
graph = dict(original._workflow.graph_dict)
graph["nodes"].append(
{
"id": "tool",
"data": {
"type": "tool",
**ToolNodeData(
title="Tool",
provider_id=provider_id,
provider_type="workflow",
provider_name="source",
tool_name="source",
tool_label="Source",
tool_configurations={},
tool_parameters={},
).model_dump(mode="json"),
},
}
)
graph["nodes"][2]["data"]["outputs"] = [{"variable": "result", "value_selector": ["tool", "json"]}]
graph["edges"][1]["target"] = "tool"
graph["edges"].append({"source": "tool", "target": "end"})
original._workflow.graph = json.dumps(graph)
config = PauseStateLayerConfig(sqlite_engine, original_entity.user_id)
WorkflowRunAgg.run(original, config, workflow_repository, node_repository)
before = [message.event for message in original._queue_manager.listen()]
assert isinstance(before[-1], QueueWorkflowPausedEvent)
assert not any(isinstance(event, QueueNodeStartedEvent) and event.node_id == "tool" for event in before)
with Session(sqlite_engine) as session, session.begin():
pause = session.scalar(select(WorkflowPause))
form = session.scalar(select(HumanInputForm))
provider = session.get(WorkflowToolProvider, provider_id)
assert pause is not None
assert form is not None
assert provider is not None
form_id = form.id
snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
provider.version = "new"
provider.label = "Republished source"
HumanInputFormSubmissionRepository().mark_submitted(
form_id=form_id,
recipient_id=None,
selected_action_id="approve",
form_data={"answer": "approved"},
submission_user_id=None,
submission_end_user_id=None,
)
entity = snapshot.get_generate_entity().model_copy(update={"task_id": str(uuid4())})
assert isinstance(entity, WorkflowAppGenerateEntity)
queue = WorkflowAppQueueManager(entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW)
resumed = WorkflowAppRunner(
application_generate_entity=entity,
queue_manager=queue,
variable_loader=DUMMY_VARIABLE_LOADER,
workflow=original._workflow,
system_user_id=entity.user_id,
graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),
response_stream_filter=snapshot.get_response_stream_filter(),
execution_driver=WorkflowRunAgg.run,
)
WorkflowRunAgg.run(resumed, config, workflow_repository, node_repository)
after = [message.event for message in queue.listen()]
assert [event.form_id for event in after if isinstance(event, QueueHumanInputFormFilledEvent)] == [form_id]
assert isinstance(after[-1], QueueWorkflowSucceededEvent), after
assert after[-1].outputs == {"result": [{"marker": "original"}]}
assert entity.extras == original_entity.extras
with Session(sqlite_engine) as session:
child_workflows = session.scalars(
select(WorkflowNodeExecutionModel.workflow_id).where(WorkflowNodeExecutionModel.app_id == source_app_id)
).all()
assert child_workflows
assert set(child_workflows) == {source_workflow_ids[0]}
# A new execution resolves the newly published graph and parameter defaults.
graph["nodes"] = [node for node in graph["nodes"] if node["id"] != "approval"]
graph["edges"] = [{"source": "start", "target": "tool"}, {"source": "tool", "target": "end"}]
original._workflow.graph = json.dumps(graph)
fresh_entity = original_entity.model_copy(
update={"task_id": str(uuid4()), "workflow_execution_id": str(uuid4()), "extras": {}}
)
fresh_queue = WorkflowAppQueueManager(
fresh_entity.task_id, fresh_entity.user_id, fresh_entity.invoke_from, AppMode.WORKFLOW
)
fresh = WorkflowAppRunner(
application_generate_entity=fresh_entity,
queue_manager=fresh_queue,
variable_loader=DUMMY_VARIABLE_LOADER,
workflow=original._workflow,
system_user_id=fresh_entity.user_id,
execution_driver=WorkflowRunAgg.run,
)
WorkflowRunAgg.run(fresh, None, workflow_repository, node_repository)
fresh_events = [message.event for message in fresh_queue.listen()]
assert isinstance(fresh_events[-1], QueueWorkflowSucceededEvent), fresh_events
assert fresh_events[-1].outputs == {"result": [{"marker": "republished"}]}
@staticmethod
def test_active_execution_snapshot_failure_is_not_published_as_a_pause(sqlite_engine: Engine) -> None:
from graphon.engine_events import GraphRunFailedEvent, GraphRunPausedEvent
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
prepared = runner.prepare()
assert prepared is not None
aggregate = WorkflowRunAgg(
prepared,
runner,
PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id),
workflow_execution_repository=workflow_repository,
workflow_node_execution_repository=node_repository,
)
# A participant that has not shut down must prevent a resume-ready snapshot.
with prepared.entry.graph_engine.runtime_state.graph_execution.track_execution():
events = list(aggregate.iter_events())
assert isinstance(events[-1], GraphRunFailedEvent)
assert "during active execution" in events[-1].error
assert not any(isinstance(event, GraphRunPausedEvent) for event in events)
with Session(sqlite_engine) as session:
run = session.get(WorkflowRun, runner.application_generate_entity.workflow_execution_id)
assert run is not None
assert run.status == "failed"
assert session.scalar(select(WorkflowPause)) is None
@staticmethod
@pytest.mark.usefixtures("_local_storage")
@pytest.mark.parametrize(
("terminal_event", "status"),
[
(GraphRunSucceededEvent(), WorkflowExecutionStatus.SUCCEEDED),
(GraphRunPartialSucceededEvent(exceptions_count=1), WorkflowExecutionStatus.PARTIAL_SUCCEEDED),
(GraphRunFailedEvent(error="failed"), WorkflowExecutionStatus.FAILED),
(GraphRunAbortedEvent(reason="stopped"), WorkflowExecutionStatus.STOPPED),
],
)
def test_elapsed_time_accumulates_across_pauses_and_matches_delayed_stream_responses(
monkeypatch: pytest.MonkeyPatch,
sqlite_engine: Engine,
terminal_event: GraphRunSucceededEvent
| GraphRunPartialSucceededEvent
| GraphRunFailedEvent
| GraphRunAbortedEvent,
status: WorkflowExecutionStatus,
) -> None:
from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
from core.app.entities.task_entities import WorkflowPauseStreamResponse
from core.repositories.human_input_repository import HumanInputFormSubmissionRepository
from core.workflow.nodes.human_input.enums import HumanInputFormStatus
from core.workflow.system_variables import build_system_variables
from graphon.entities import WorkflowStartReason
from models.human_input import HumanInputForm
clock = [100.0]
monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])
original, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
graph = original._workflow.graph_dict
graph["nodes"].append({"id": "second-approval", "data": graph["nodes"][1]["data"]})
graph["edges"][1]["target"] = "second-approval"
graph["edges"].append(
{"id": "second-end", "source": "second-approval", "target": "end", "sourceHandle": "approve"}
)
original._workflow.graph = json.dumps(graph)
runner = original
run_id = runner.application_generate_entity.workflow_execution_id
config = PauseStateLayerConfig(sqlite_engine, runner.application_generate_entity.user_id)
account = Account(name="Test", email="[email protected]")
account.id = runner.application_generate_entity.user_id
total = 0.0
for index, duration in enumerate([2.0, 3.0, 4.0]):
prepared = runner.prepare()
assert prepared is not None
aggregate = WorkflowRunAgg(
prepared,
runner,
config,
workflow_execution_repository=workflow_repository,
workflow_node_execution_repository=node_repository,
)
is_paused = index < 2
if is_paused:
events = []
for event in aggregate.iter_events():
if isinstance(event, GraphRunStartedEvent):
with Session(sqlite_engine) as session:
started = session.get(WorkflowRun, run_id)
assert started is not None
assert started.elapsed_time == total
clock[0] += duration
events.append(event)
assert isinstance(events[-1], GraphRunPausedEvent), events
else:
aggregate._persistence_layer.on_event(GraphRunStartedEvent())
clock[0] += duration
aggregate._persistence_layer.on_event(terminal_event)
total += duration
with Session(sqlite_engine) as session:
execution = session.get(WorkflowRun, run_id)
assert execution is not None
assert execution.elapsed_time == total
assert execution.status == (WorkflowExecutionStatus.PAUSED if is_paused else status)
assert (execution.finished_at is None) == is_paused
# Queue processing and pause waits must not extend a settled segment.
clock[0] += 1000
entity = runner.application_generate_entity
runtime = prepared.entry.graph_engine.runtime_state
converter = WorkflowResponseConverter(
application_generate_entity=entity,
user=account,
system_variables=build_system_variables(workflow_execution_id=run_id),
)
converter.workflow_start_to_stream_response(
task_id=entity.task_id,
workflow_run_id=run_id,
workflow_id=original._workflow.id,
reason=WorkflowStartReason.INITIAL if index == 0 else WorkflowStartReason.RESUMPTION,
)
if is_paused:
responses = converter.workflow_pause_to_stream_response(
event=QueueWorkflowPausedEvent(), task_id=entity.task_id, graph_runtime_state=runtime
)
pause_response = responses[-1]
assert isinstance(pause_response, WorkflowPauseStreamResponse)
assert pause_response.data.elapsed_time == total
with Session(sqlite_engine) as session:
pause = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run_id))
form = session.scalar(
select(HumanInputForm).where(
HumanInputForm.workflow_run_id == run_id,
HumanInputForm.status == HumanInputFormStatus.WAITING,
)
)
assert pause is not None
assert form is not None
form_id = form.id
snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode())
entity = snapshot.get_generate_entity()
assert isinstance(entity, WorkflowAppGenerateEntity)
assert entity.workflow_execution_timing.elapsed_time == snapshot.elapsed_time == total
HumanInputFormSubmissionRepository().mark_submitted(
form_id=form_id,
recipient_id=None,
selected_action_id="approve",
form_data={"answer": "approved result"},
submission_user_id=None,
submission_end_user_id=None,
)
entity.task_id = str(uuid4())
runner = WorkflowAppRunner(
application_generate_entity=entity,
queue_manager=WorkflowAppQueueManager(
entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW
),
variable_loader=DUMMY_VARIABLE_LOADER,
workflow=original._workflow,
system_user_id=entity.user_id,
graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state),
response_stream_filter=snapshot.get_response_stream_filter(),
execution_driver=WorkflowRunAgg.run,
)
else:
response = converter.workflow_finish_to_stream_response(
task_id=entity.task_id,
workflow_id=original._workflow.id,
status=status,
graph_runtime_state=runtime,
)
assert response.data.elapsed_time == 9.0
assert entity.workflow_execution_timing.finish() == 9.0
@staticmethod
@pytest.mark.usefixtures("_local_storage")
@pytest.mark.parametrize("stop_timing", ["before_summary", "during_upload"])
def test_stop_wins_while_engine_is_persisting_a_pause(
sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch, stop_timing: str
) -> None:
from sqlalchemy.orm import sessionmaker
from core.app.entities.queue_entities import QueueStopEvent
from core.workflow.nodes.human_input.enums import HumanInputFormStatus
from graphon.enums import WorkflowExecutionStatus
from models.human_input import HumanInputForm
from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
entity = runner.application_generate_entity
repository = DifyAPISQLAlchemyWorkflowRunRepository(sessionmaker(sqlite_engine))
stop_calls = []
def request_stop() -> None:
admitted = repository.stop_paused_workflow_task(
tenant_id=entity.app_config.tenant_id,
app_id=entity.app_config.app_id,
task_id=entity.task_id,
owner=None,
)
assert admitted is not None
stop_calls.append(admitted.status)
if stop_timing == "during_upload":
original_save = storage.save
def save(key: str, data: bytes) -> None:
if key.startswith("workflow-state-"):
request_stop()
return original_save(key, data)
monkeypatch.setattr(storage, "save", save)
else:
original_persist = WorkflowRunAgg._persist_pause
def persist(aggregate: WorkflowRunAgg, event: GraphRunPausedEvent) -> None:
request_stop()
return original_persist(aggregate, event)
monkeypatch.setattr(WorkflowRunAgg, "_persist_pause", persist)
WorkflowRunAgg.run(
runner, PauseStateLayerConfig(sqlite_engine, entity.user_id), workflow_repository, node_repository
)
events = [message.event for message in runner._queue_manager.listen()]
assert stop_calls
assert isinstance(events[-1], QueueStopEvent), events
assert not any(isinstance(event, QueueWorkflowPausedEvent) for event in events)
with Session(sqlite_engine) as session:
run = session.get(WorkflowRun, entity.workflow_execution_id)
assert run is not None
assert run.status == WorkflowExecutionStatus.STOPPED
assert run.stop_requested_at is not None
assert run.task_id == entity.task_id
assert session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == run.id)) is None
forms = session.scalars(select(HumanInputForm).where(HumanInputForm.workflow_run_id == run.id)).all()
assert forms
assert all(form.status == HumanInputFormStatus.EXPIRED for form in forms)
@staticmethod
def test_legacy_caller_rejects_human_input_before_creating_run_or_form(sqlite_engine: Engine) -> None:
from models.human_input import HumanInputForm
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
runner.application_generate_entity.allow_human_input = False
with pytest.raises(ValueError, match="Human Input nodes are not supported"):
WorkflowRunAgg.run(runner, None, workflow_repository, node_repository)
with Session(sqlite_engine) as session:
assert session.scalar(select(WorkflowRun)) is None
assert session.scalar(select(HumanInputForm)) is None
v2_form = _v2_form_fixture
@pytest.mark.parametrize("token", ["initiator-token", None])
def test_v2_pause_persists_only_checkpoint_references_and_publishes_frozen_form(
sqlite_engine: Engine, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, token: str | None, v2_form: Form
) -> None:
from dataclasses import replace
from pydantic import SecretStr
from core.human_input_v2.shared.values import TenantId
from core.workflow.nodes.human_input_v2.entities import HumanInputNodeData as V2NodeData
from core.workflow.nodes.human_input_v2.runtime import PreparedForm
from models.workflow import WorkflowPauseReason
monkeypatch.setattr(storage, "storage_runner", OpenDALStorage(scheme="fs", root=str(tmp_path)), raising=False)
runner, workflow_repository, node_repository = make_workflow_runner(sqlite_engine, human_input=True)
entity = runner.application_generate_entity
graph = runner._workflow.graph_dict
graph["nodes"][1]["data"] = V2NodeData.model_validate(
{
"type": "human-input",
"version": "2",
"title": "Approval",
"form_content": "Changed content",
"recipients_spec": [],
"user_actions": [],
"message_template": {"subject": "Notice", "body": "[Request URL]"},
"debug_mode": {"enabled": False, "channels": []},
}
).model_dump(mode="json")
runner._workflow.graph = json.dumps(graph)
source_form = v2_form
prepared_forms: list[PreparedForm] = []
def prepare(_runtime: object, *, node_execution_id: str, **_kwargs: object) -> PreparedForm:
prepared = PreparedForm(
replace(
source_form,
id=node_execution_id,
node_execution_id=node_execution_id,
tenant_id=TenantId(entity.app_config.tenant_id),
app_id=entity.app_config.app_id,
workflow_run_id=entity.workflow_execution_id,
),
SecretStr(token) if token is not None else None,
)
prepared_forms.append(prepared)
return prepared
monkeypatch.setattr("services.human_input_v2.composition.WorkflowHumanInputRuntime.prepare_form", prepare)
def reject_legacy_lookup(*_args: object, **_kwargs: object) -> None:
pytest.fail("V2 forms must not be loaded from the legacy repository")
monkeypatch.setattr(
"core.repositories.human_input_repository.HumanInputFormSubmissionRepository.get_by_form_ids",
reject_legacy_lookup,
)
WorkflowRunAgg.run(
runner, PauseStateLayerConfig(sqlite_engine, entity.user_id), workflow_repository, node_repository
)
events = [message.event for message in runner._queue_manager.listen()]
paused = events[-1]
assert isinstance(paused, QueueWorkflowPausedEvent), events
assert isinstance(paused.reasons[0], HumanInputRequired)
assert paused.reasons[0].form_version == "2"
assert paused.reasons[0].form_content == source_form.resolved_form.legacy_form_content
assert paused.human_input_v2_forms[prepared_forms[0].form.id] is prepared_forms[0]
with Session(sqlite_engine) as session:
pause = session.scalar(
select(WorkflowPause).where(WorkflowPause.workflow_run_id == entity.workflow_execution_id)
)
assert pause is not None
assert list(session.scalars(select(WorkflowPauseReason).where(WorkflowPauseReason.pause_id == pause.id))) == []
saved_state = storage.load(pause.state_object_key).decode()
context = WorkflowResumptionContext.loads(saved_state)
restored = RuntimeState.from_snapshot(context.serialized_graph_runtime_state)
assert isinstance(restored.graph_execution.pause_reasons[0], HitlRequired)
assert restored.graph_execution.pause_reasons[0].session_id == f"hitlv2:{prepared_forms[0].form.id}"
if token is not None:
assert token not in saved_state