Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: accumulate workflow elapsed time across resumptions (#44161)

林玮 (Jade Lin)10285432d225
diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py
index 8497714f685..f837c9f8426 100644
--- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py
+++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py
@@ -320,7 +320,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
paused_nodes=paused_nodes,
reasons=reasons,
status=WorkflowExecutionStatus.PAUSED,
- elapsed_time=time.perf_counter() - self._base_task_pipeline.start_at,
+ elapsed_time=self._application_generate_entity.workflow_execution_timing.finish(),
total_tokens=runtime_state.total_tokens,
total_steps=runtime_state.node_run_steps,
),
diff --git a/api/core/app/apps/common/workflow_response_converter.py b/api/core/app/apps/common/workflow_response_converter.py
index 77ff812b2c9..205810adc9f 100644
--- a/api/core/app/apps/common/workflow_response_converter.py
+++ b/api/core/app/apps/common/workflow_response_converter.py
@@ -245,6 +245,7 @@ class WorkflowResponseConverter:
run_id = self._ensure_workflow_run_id(workflow_run_id)
started_at = naive_utc_now()
self._workflow_started_at = started_at
+ self._application_generate_entity.workflow_execution_timing.start()
return WorkflowStartStreamResponse(
task_id=task_id,
@@ -276,7 +277,7 @@ class WorkflowResponseConverter:
)
finished_at = naive_utc_now()
- elapsed_time = (finished_at - started_at).total_seconds()
+ elapsed_time = self._application_generate_entity.workflow_execution_timing.finish()
outputs_mapping = graph_runtime_state.outputs or {}
encoded_outputs = WorkflowRuntimeTypeConverter().to_json_encodable(outputs_mapping)
@@ -329,8 +330,7 @@ class WorkflowResponseConverter:
raise ValueError(
"workflow_pause_to_stream_response called before workflow_start_to_stream_response",
)
- paused_at = naive_utc_now()
- elapsed_time = (paused_at - started_at).total_seconds()
+ elapsed_time = self._application_generate_entity.workflow_execution_timing.finish()
encoded_outputs = self._encode_outputs(event.outputs) or {}
if self._application_generate_entity.invoke_from == InvokeFrom.SERVICE_API:
encoded_outputs = {}
diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py
index e5f9e68b2aa..b0065259d9e 100644
--- a/api/core/app/apps/workflow/generate_task_pipeline.py
+++ b/api/core/app/apps/workflow/generate_task_pipeline.py
@@ -1,5 +1,4 @@
import logging
-import time
from collections.abc import Callable, Generator
from contextlib import contextmanager
@@ -214,7 +213,7 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):
status=WorkflowExecutionStatus.PAUSED,
outputs={},
error=None,
- elapsed_time=time.perf_counter() - self._base_task_pipeline.start_at,
+ elapsed_time=self._application_generate_entity.workflow_execution_timing.finish(),
total_tokens=runtime_state.total_tokens,
total_steps=runtime_state.node_run_steps,
created_at=created_at,
diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.py
index cb3f52d12a0..a2557032f7c 100644
--- a/api/core/app/entities/app_invoke_entities.py
+++ b/api/core/app/entities/app_invoke_entities.py
@@ -19,6 +19,7 @@ from core.entities.provider_configuration import ProviderModelBundle
from graphon.file import File, FileUploadConfig
from graphon.model_runtime.entities.model_entities import AIModelEntity
from models.model import AppMode
+from repositories.entities.workflow_execution import WorkflowExecutionTiming
if TYPE_CHECKING:
from core.ops.ops_trace_manager import TraceQueueManager
@@ -307,6 +308,9 @@ class AdvancedChatAppGenerateEntity(ConversationAppGenerateEntity):
app_config: WorkflowUIBasedAppConfig = None # type: ignore
workflow_run_id: str | None = None
+ workflow_execution_timing: WorkflowExecutionTiming = Field(
+ default_factory=WorkflowExecutionTiming, exclude=True, repr=False
+ )
query: str
class SingleIterationRunEntity(BaseModel):
@@ -338,6 +342,9 @@ class WorkflowAppGenerateEntity(AppGenerateEntity):
# app config
app_config: WorkflowUIBasedAppConfig = None # type: ignore
workflow_execution_id: str
+ workflow_execution_timing: WorkflowExecutionTiming = Field(
+ default_factory=WorkflowExecutionTiming, exclude=True, repr=False
+ )
class SingleIterationRunEntity(BaseModel):
"""
diff --git a/api/core/app/layers/pause_state_persist_layer.py b/api/core/app/layers/pause_state_persist_layer.py
index 5d327adb065..b65728ab3b0 100644
--- a/api/core/app/layers/pause_state_persist_layer.py
+++ b/api/core/app/layers/pause_state_persist_layer.py
@@ -14,6 +14,7 @@ from graphon.graph_engine.layers import GraphEngineLayer
from graphon.graph_events import GraphEngineEvent, GraphRunPausedEvent
from models.model import AppMode
from repositories.api_workflow_run_repository import APIWorkflowRunRepository
+from repositories.entities.workflow_execution import WorkflowExecutionTiming
from repositories.factory import DifyAPIRepositoryFactory
@@ -44,6 +45,7 @@ class WorkflowResumptionContext(BaseModel):
# Only workflow / chatflow could be paused.
generate_entity: _GenerateEntityUnion
serialized_graph_runtime_state: str
+ elapsed_time: float = Field(default=0.0, ge=0, allow_inf_nan=False)
# Optional so that a workflow run paused before this field existed still
# loads: it just degrades to fresh-filter behavior on resume for that one
# stale run.
@@ -54,7 +56,11 @@ class WorkflowResumptionContext(BaseModel):
@classmethod
def loads(cls, value: str) -> Self:
- return cls.model_validate_json(value)
+ state = cls.model_validate_json(value)
+ state.generate_entity.entity.workflow_execution_timing = WorkflowExecutionTiming(
+ elapsed_time=state.elapsed_time,
+ )
+ return state
def get_generate_entity(self) -> WorkflowAppGenerateEntity | AdvancedChatAppGenerateEntity:
return self.generate_entity.entity
@@ -138,6 +144,7 @@ class PauseStatePersistenceLayer(GraphEngineLayer):
state = WorkflowResumptionContext(
serialized_graph_runtime_state=self.graph_runtime_state.dumps(),
+ elapsed_time=self._generate_entity.workflow_execution_timing.finish(),
generate_entity=entity_wrapper,
serialized_response_stream_filter_state=self._response_stream_filter.dumps(),
)
diff --git a/api/core/app/workflow/layers/persistence.py b/api/core/app/workflow/layers/persistence.py
index 6edc745701d..8b3119c1387 100644
--- a/api/core/app/workflow/layers/persistence.py
+++ b/api/core/app/workflow/layers/persistence.py
@@ -24,7 +24,7 @@ from core.workflow.node_execution_process_data import preserve_workflow_agent_bi
from core.workflow.system_variables import SystemVariableKey
from core.workflow.variable_prefixes import SYSTEM_VARIABLE_NODE_ID
from core.workflow.workflow_run_outputs import project_node_outputs_for_workflow_run
-from graphon.entities import WorkflowExecution, WorkflowNodeExecution, WorkflowStartReason
+from graphon.entities import WorkflowNodeExecution, WorkflowStartReason
from graphon.enums import (
BuiltinNodeTypes,
WorkflowExecutionStatus,
@@ -50,6 +50,7 @@ from graphon.graph_events import (
)
from graphon.node_events import NodeRunResult
from libs.datetime_utils import naive_utc_now
+from repositories.entities.workflow_execution import WorkflowExecution
from services.workflow.inspector_events import (
publish_node_changed as _inspector_publish_node_changed,
)
@@ -151,7 +152,9 @@ class WorkflowPersistenceLayer(GraphEngineLayer):
# ------------------------------------------------------------------
def _handle_graph_run_started(self, event: GraphRunStartedEvent | None = None) -> None:
execution_id = self._get_execution_id()
- workflow_execution = WorkflowExecution.new(
+ timing = self._application_generate_entity.workflow_execution_timing
+ timing.start()
+ workflow_execution = WorkflowExecution(
id_=execution_id,
workflow_id=self._workflow_info.workflow_id,
workflow_type=self._workflow_info.workflow_type,
@@ -159,6 +162,7 @@ class WorkflowPersistenceLayer(GraphEngineLayer):
graph=self._workflow_info.graph_data,
inputs=self._prepare_workflow_inputs(),
started_at=naive_utc_now(),
+ accumulated_elapsed_time=timing.elapsed_time,
)
self._workflow_execution_repository.save(workflow_execution)
@@ -413,6 +417,7 @@ class WorkflowPersistenceLayer(GraphEngineLayer):
return merged_process_data
def _populate_completion_statistics(self, execution: WorkflowExecution, *, update_finished: bool = True) -> None:
+ execution.accumulated_elapsed_time = self._application_generate_entity.workflow_execution_timing.finish()
if update_finished:
execution.finished_at = naive_utc_now()
runtime_state = self.graph_runtime_state
diff --git a/api/core/repositories/sqlalchemy_workflow_execution_repository.py b/api/core/repositories/sqlalchemy_workflow_execution_repository.py
index fa68dfa84a0..6be44a69394 100644
--- a/api/core/repositories/sqlalchemy_workflow_execution_repository.py
+++ b/api/core/repositories/sqlalchemy_workflow_execution_repository.py
@@ -20,6 +20,7 @@ from models import (
WorkflowRun,
)
from models.enums import WorkflowRunTriggeredFrom
+from repositories.entities.workflow_execution import WorkflowExecution as TimedWorkflowExecution
logger = logging.getLogger(__name__)
@@ -101,7 +102,7 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
# Convert status to domain enum
status = WorkflowExecutionStatus(db_model.status)
- return WorkflowExecution(
+ return TimedWorkflowExecution(
id_=db_model.id,
workflow_id=db_model.workflow_id,
workflow_type=WorkflowType(db_model.type),
@@ -116,6 +117,7 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
exceptions_count=db_model.exceptions_count,
started_at=db_model.created_at,
finished_at=db_model.finished_at,
+ accumulated_elapsed_time=db_model.elapsed_time,
)
def _to_db_model(self, domain_model: WorkflowExecution) -> WorkflowRun:
@@ -167,9 +169,10 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
db_model.created_at = domain_model.started_at
db_model.finished_at = domain_model.finished_at
- # Calculate elapsed time if finished_at is available
- if domain_model.finished_at:
- db_model.elapsed_time = (domain_model.finished_at - domain_model.started_at).total_seconds()
+ if domain_model.finished_at or (
+ isinstance(domain_model, TimedWorkflowExecution) and domain_model.accumulated_elapsed_time is not None
+ ):
+ db_model.elapsed_time = domain_model.elapsed_time
else:
db_model.elapsed_time = 0
diff --git a/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py b/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py
index 40c638f9d35..ac8a75d8890 100644
--- a/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py
+++ b/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py
@@ -130,9 +130,9 @@ def _dict_to_workflow_run(data: dict[str, Any]) -> WorkflowRun:
case _:
model.finished_at = ensure_naive_utc(finished_at)
- # Compute elapsed_time from started_at and finished_at
- # LogStore doesn't store elapsed_time, it's computed in WorkflowExecution domain entity
- if model.finished_at and model.created_at:
+ if data.get("elapsed_time") is not None:
+ model.elapsed_time = safe_float(data["elapsed_time"])
+ elif model.finished_at and model.created_at:
model.elapsed_time = (model.finished_at - model.created_at).total_seconds()
else:
# Use safe conversion to handle 'null' strings and None values
diff --git a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py
index 55d66aae680..c19eb8aa4fb 100644
--- a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py
+++ b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py
@@ -143,6 +143,7 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):
("total_tokens", str(domain_model.total_tokens)),
("total_steps", str(domain_model.total_steps)),
("exceptions_count", str(domain_model.exceptions_count)),
+ ("elapsed_time", str(domain_model.elapsed_time)),
(
"created_by_role",
self._creator_user_role.value
diff --git a/api/repositories/entities/workflow_execution.py b/api/repositories/entities/workflow_execution.py
new file mode 100644
index 00000000000..905b2cfb5d6
--- /dev/null
+++ b/api/repositories/entities/workflow_execution.py
@@ -0,0 +1,46 @@
+"""Workflow execution values shared across runtime and storage boundaries."""
+
+from dataclasses import dataclass, field
+from threading import Lock
+from time import perf_counter
+from typing import override
+
+from pydantic import Field
+
+from graphon.entities import WorkflowExecution as GraphonWorkflowExecution
+
+
+@dataclass
+class WorkflowExecutionTiming:
+ """Settle one execution segment once, excluding time spent paused."""
+
+ elapsed_time: float = 0.0
+ _started_at: float | None = field(default=None, init=False, repr=False, compare=False)
+ _finished: bool = field(default=False, init=False, repr=False, compare=False)
+ _lock: Lock = field(default_factory=Lock, init=False, repr=False, compare=False)
+
+ def start(self) -> None:
+ with self._lock:
+ if self._started_at is None:
+ self._started_at = perf_counter()
+
+ def finish(self) -> float:
+ with self._lock:
+ if self._started_at is not None and not self._finished:
+ self.elapsed_time += max(perf_counter() - self._started_at, 0.0)
+ self._finished = True
+ return self.elapsed_time
+
+
+class WorkflowExecution(GraphonWorkflowExecution):
+ """Carry settled active duration independently of wall-clock timestamps."""
+
+ accumulated_elapsed_time: float | None = Field(default=None, ge=0, allow_inf_nan=False)
+ """None preserves timestamp-based duration for older serialized executions."""
+
+ @property
+ @override
+ def elapsed_time(self) -> float:
+ if self.accumulated_elapsed_time is not None:
+ return self.accumulated_elapsed_time
+ return super().elapsed_time
diff --git a/api/tasks/workflow_execution_tasks.py b/api/tasks/workflow_execution_tasks.py
index d1eb24d860d..9e10dca1006 100644
--- a/api/tasks/workflow_execution_tasks.py
+++ b/api/tasks/workflow_execution_tasks.py
@@ -13,10 +13,10 @@ from celery import shared_task
from sqlalchemy import select
from core.db.session_factory import session_factory
-from graphon.entities import WorkflowExecution
from graphon.workflow_type_encoder import WorkflowRuntimeTypeConverter
from models import CreatorUserRole, WorkflowRun
from models.enums import WorkflowRunTriggeredFrom
+from repositories.entities.workflow_execution import WorkflowExecution
logger = logging.getLogger(__name__)
diff --git a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py
index f71c3694859..29a1f8722a1 100644
--- a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py
+++ b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py
@@ -150,11 +150,17 @@ def _persist_human_input_form(
def _build_service_api_pause_converter() -> WorkflowResponseConverter:
- application_generate_entity = SimpleNamespace(
+ application_generate_entity = WorkflowAppGenerateEntity(
+ task_id="task",
+ workflow_execution_id="run-id",
+ user_id="user",
+ stream=True,
inputs={},
files=[],
invoke_from=InvokeFrom.SERVICE_API,
- app_config=SimpleNamespace(app_id="app-id", tenant_id="tenant-id"),
+ app_config=WorkflowUIBasedAppConfig(
+ app_id="app-id", tenant_id="tenant-id", workflow_id="workflow-id", app_mode=AppMode.WORKFLOW
+ ),
)
system_variables = build_system_variables(
user_id="user",
@@ -569,8 +575,7 @@ class TestHitlServiceApi:
assert response.data.reasons[0]["form_id"] == "form-1"
assert response.data.reasons[0]["expiration_time"] == 123
- def test_workflow_blocking_pipeline_pause_payload_contract(self, monkeypatch: pytest.MonkeyPatch) -> None:
- from core.app.apps.workflow import generate_task_pipeline as workflow_pipeline_module
+ def test_workflow_blocking_pipeline_pause_payload_contract(self) -> None:
from core.app.apps.workflow.generate_task_pipeline import WorkflowAppGenerateTaskPipeline
app_config = WorkflowUIBasedAppConfig(
@@ -602,7 +607,6 @@ class TestHitlServiceApi:
stream=False,
draft_var_saver_factory=lambda **kwargs: None,
)
- monkeypatch.setattr(workflow_pipeline_module.time, "time", lambda: 1700000000)
def _gen():
yield HumanInputRequiredResponse(
diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py
index 8b898d8bd2f..4ecfa826809 100644
--- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py
+++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py
@@ -1,17 +1,17 @@
from datetime import UTC, datetime
-from types import SimpleNamespace
+from core.app.app_config.entities import WorkflowUIBasedAppConfig
from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
-from core.app.entities.app_invoke_entities import InvokeFrom
+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity
from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent, QueueHumanInputFormTimeoutEvent
from core.workflow.system_variables import build_system_variables
from graphon.entities import WorkflowStartReason
-from graphon.runtime import GraphRuntimeState, VariablePool
from graphon.variables.segments import StringSegment
from models.account import Account
+from models.model import AppMode
-def _build_converter():
+def _build_converter() -> WorkflowResponseConverter:
system_variables = build_system_variables(
files=[],
user_id="user-1",
@@ -19,15 +19,18 @@ def _build_converter():
workflow_id="wf-1",
workflow_execution_id="run-1",
)
- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)
- app_entity = SimpleNamespace(
+ app_entity = WorkflowAppGenerateEntity(
task_id="task-1",
- app_config=SimpleNamespace(app_id="app-1", tenant_id="tenant-1"),
+ app_config=WorkflowUIBasedAppConfig(
+ app_id="app-1", tenant_id="tenant-1", workflow_id="wf-1", app_mode=AppMode.WORKFLOW
+ ),
invoke_from=InvokeFrom.EXPLORE,
files=[],
inputs={},
workflow_execution_id="run-1",
call_depth=0,
+ user_id="user-1",
+ stream=True,
)
account = Account(name="tester", email="[email protected]")
account.id = "acc-1"
@@ -38,7 +41,7 @@ def _build_converter():
)
-def test_human_input_form_filled_stream_response_contains_rendered_content():
+def test_human_input_form_filled_stream_response_contains_rendered_content() -> None:
converter = _build_converter()
converter.workflow_start_to_stream_response(
task_id="task-1",
@@ -66,7 +69,7 @@ def test_human_input_form_filled_stream_response_contains_rendered_content():
assert resp.data.action_id == "Approve"
-def test_human_input_form_filled_stream_response_serializes_submitted_data():
+def test_human_input_form_filled_stream_response_serializes_submitted_data() -> None:
converter = _build_converter()
converter.workflow_start_to_stream_response(
task_id="task-1",
@@ -97,7 +100,7 @@ def test_human_input_form_filled_stream_response_serializes_submitted_data():
}
-def test_human_input_form_timeout_stream_response_contains_timeout_metadata():
+def test_human_input_form_timeout_stream_response_contains_timeout_metadata() -> None:
converter = _build_converter()
converter.workflow_start_to_stream_response(
task_id="task-1",
diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py
index 7e397ebc73f..f2167100fb2 100644
--- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py
+++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py
@@ -1,11 +1,10 @@
-from types import SimpleNamespace
-
+from core.app.app_config.entities import WorkflowUIBasedAppConfig
from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
-from core.app.entities.app_invoke_entities import InvokeFrom
+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity
from core.workflow.system_variables import build_system_variables
from graphon.entities import WorkflowStartReason
-from graphon.runtime import GraphRuntimeState, VariablePool
from models.account import Account
+from models.model import AppMode
def _build_converter() -> WorkflowResponseConverter:
@@ -17,15 +16,18 @@ def _build_converter() -> WorkflowResponseConverter:
workflow_id="wf-1",
workflow_execution_id="run-1",
)
- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)
- app_entity = SimpleNamespace(
+ app_entity = WorkflowAppGenerateEntity(
task_id="task-1",
- app_config=SimpleNamespace(app_id="app-1", tenant_id="tenant-1"),
+ app_config=WorkflowUIBasedAppConfig(
+ app_id="app-1", tenant_id="tenant-1", workflow_id="wf-1", app_mode=AppMode.WORKFLOW
+ ),
invoke_from=InvokeFrom.EXPLORE,
files=[],
inputs={},
workflow_execution_id="run-1",
call_depth=0,
+ user_id="user-1",
+ stream=True,
)
account = Account(name="tester", email="[email protected]")
account.id = "acc-1"
@@ -36,7 +38,7 @@ def _build_converter() -> WorkflowResponseConverter:
)
-def test_workflow_start_stream_response_carries_resumption_reason():
+def test_workflow_start_stream_response_carries_resumption_reason() -> None:
converter = _build_converter()
resp = converter.workflow_start_to_stream_response(
task_id="task-1",
@@ -47,7 +49,7 @@ def test_workflow_start_stream_response_carries_resumption_reason():
assert resp.data.reason is WorkflowStartReason.RESUMPTION
-def test_workflow_start_stream_response_carries_initial_reason():
+def test_workflow_start_stream_response_carries_initial_reason() -> None:
converter = _build_converter()
resp = converter.workflow_start_to_stream_response(
task_id="task-1",
diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py b/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py
index fb9b7f6a4b1..4fd4af28f38 100644
--- a/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py
+++ b/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py
@@ -6,10 +6,11 @@ from unittest.mock import MagicMock
import pytest
from sqlalchemy.orm import Session
+from core.app.app_config.entities import WorkflowUIBasedAppConfig
from core.app.apps.common import workflow_response_converter
from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
from core.app.apps.workflow.app_runner import WorkflowAppRunner
-from core.app.entities.app_invoke_entities import InvokeFrom
+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity
from core.app.entities.queue_entities import QueueWorkflowPausedEvent
from core.app.entities.task_entities import HumanInputRequiredResponse, WorkflowPauseStreamResponse
from core.workflow.nodes.human_input.entities import (
@@ -27,6 +28,7 @@ from graphon.graph_events import GraphRunPausedEvent
from graphon.runtime import GraphRuntimeState, VariablePool
from models.account import Account
from models.human_input import HumanInputForm, HumanInputFormRecipient, RecipientType
+from models.model import AppMode
from models.workflow import Workflow, WorkflowType
@@ -158,12 +160,18 @@ def test_graph_run_paused_event_emits_queue_pause_event(monkeypatch: pytest.Monk
assert queue_event.paused_nodes == ["node-human"]
-def _build_converter(*, invoke_from: InvokeFrom = InvokeFrom.SERVICE_API):
- application_generate_entity = SimpleNamespace(
+def _build_converter(*, invoke_from: InvokeFrom = InvokeFrom.SERVICE_API) -> WorkflowResponseConverter:
+ application_generate_entity = WorkflowAppGenerateEntity(
+ task_id="task",
+ workflow_execution_id="run-id",
+ user_id="user",
+ stream=True,
inputs={},
files=[],
invoke_from=invoke_from,
- app_config=SimpleNamespace(app_id="app-id", tenant_id="tenant-id"),
+ app_config=WorkflowUIBasedAppConfig(
+ app_id="app-id", tenant_id="tenant-id", workflow_id="workflow-id", app_mode=AppMode.WORKFLOW
+ ),
)
system_variables = build_system_variables(
user_id="user",
diff --git a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py
index 697cd531663..5b4cdd2b824 100644
--- a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py
+++ b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py
@@ -254,6 +254,11 @@ class TestPauseStatePersistenceLayer:
self, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]
):
generate_entity = self._create_generate_entity(workflow_execution_id="run-123")
+ clock = [10.0]
+ monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])
+ generate_entity.workflow_execution_timing.elapsed_time = 5.0
+ generate_entity.workflow_execution_timing.start()
+ clock[0] = 12.0
layer = PauseStatePersistenceLayer(
session_factory=sqlite_session_factory,
state_owner_user_id="owner-123",
@@ -285,6 +290,8 @@ class TestPauseStatePersistenceLayer:
assert call_kwargs["state_owner_user_id"] == "owner-123"
serialized_state = call_kwargs["state"]
resumption_context = WorkflowResumptionContext.loads(serialized_state)
+ assert resumption_context.elapsed_time == 7.0
+ assert resumption_context.get_generate_entity().workflow_execution_timing.elapsed_time == 7.0
assert resumption_context.serialized_graph_runtime_state == expected_state
assert resumption_context.get_generate_entity().model_dump() == generate_entity.model_dump()
pause_reasons = call_kwargs["pause_reasons"]
@@ -553,3 +560,14 @@ def test_get_response_stream_filter_defaults_when_state_missing() -> None:
restored = context.get_response_stream_filter()
assert isinstance(restored, ResponseStreamFilter)
+
+
+def test_resumption_context_defaults_elapsed_time_for_legacy_snapshots() -> None:
+ context = _build_workflow_generate_entity_for_roundtrip()
+ payload = context.model_dump()
+ payload.pop("elapsed_time")
+
+ restored = WorkflowResumptionContext.loads(json.dumps(payload))
+
+ assert restored.elapsed_time == 0.0
+ assert restored.get_generate_entity().workflow_execution_timing.elapsed_time == 0.0
diff --git a/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.py b/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.py
new file mode 100644
index 00000000000..61876bc05fb
--- /dev/null
+++ b/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.py
@@ -0,0 +1,144 @@
+import pytest
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.app.app_config.entities import WorkflowUIBasedAppConfig
+from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity
+from core.app.entities.queue_entities import QueueWorkflowPausedEvent
+from core.app.entities.task_entities import WorkflowPauseStreamResponse
+from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext, _WorkflowGenerateEntityWrapper
+from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer
+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository
+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
+from core.workflow.system_variables import build_system_variables
+from graphon.entities import WorkflowStartReason
+from graphon.enums import WorkflowExecutionStatus, WorkflowType
+from graphon.graph_engine.command_channels import InMemoryChannel
+from graphon.graph_events import (
+ GraphRunAbortedEvent,
+ GraphRunFailedEvent,
+ GraphRunPartialSucceededEvent,
+ GraphRunPausedEvent,
+ GraphRunStartedEvent,
+ GraphRunSucceededEvent,
+)
+from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool
+from models.account import Account
+from models.enums import WorkflowRunTriggeredFrom
+from models.model import AppMode
+from models.workflow import WorkflowNodeExecutionTriggeredFrom, WorkflowRun
+
+
+ ("terminal_event", "status"),
+ [
+ (GraphRunSucceededEvent(), WorkflowExecutionStatus.SUCCEEDED),
+ (GraphRunPartialSucceededEvent(exceptions_count=1), WorkflowExecutionStatus.PARTIAL_SUCCEEDED),
+ (GraphRunFailedEvent(error="failed"), WorkflowExecutionStatus.FAILED),
+ (GraphRunAbortedEvent(reason="stopped"), WorkflowExecutionStatus.STOPPED),
+ ],
+)
+def test_elapsed_time_accumulates_across_pauses_and_matches_delayed_stream_responses(
+ monkeypatch: pytest.MonkeyPatch,
+ sqlite_session_factory: sessionmaker[Session],
+ terminal_event: GraphRunSucceededEvent | GraphRunPartialSucceededEvent | GraphRunFailedEvent | GraphRunAbortedEvent,
+ status: WorkflowExecutionStatus,
+) -> None:
+ clock = [100.0]
+ monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])
+ monkeypatch.setattr("core.app.workflow.layers.persistence._inspector_publish_workflow_completed", lambda **_: None)
+ entity = WorkflowAppGenerateEntity(
+ task_id="task-1",
+ workflow_execution_id="run-1",
+ app_config=WorkflowUIBasedAppConfig(
+ tenant_id="tenant-1", app_id="app-1", workflow_id="workflow-1", app_mode=AppMode.WORKFLOW
+ ),
+ inputs={},
+ files=[],
+ user_id="user-1",
+ stream=True,
+ invoke_from=InvokeFrom.SERVICE_API,
+ )
+ system_variables = build_system_variables(workflow_execution_id="run-1")
+ total = 0.0
+ account = Account(name="Test", email="[email protected]")
+ account.id = "user-1"
+
+ for index, duration in enumerate([2.0, 3.0, 4.0]):
+ pool = VariablePool()
+ for variable in system_variables:
+ pool.add(variable.selector, variable)
+ runtime = GraphRuntimeState(variable_pool=pool, start_at=clock[0])
+ repository = SQLAlchemyWorkflowExecutionRepository(
+ session_factory=sqlite_session_factory,
+ tenant_id="tenant-1",
+ user=account,
+ app_id="app-1",
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ )
+ node_repository = SQLAlchemyWorkflowNodeExecutionRepository(
+ session_factory=sqlite_session_factory,
+ tenant_id="tenant-1",
+ user=account,
+ app_id="app-1",
+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
+ )
+ layer = WorkflowPersistenceLayer(
+ application_generate_entity=entity,
+ workflow_info=PersistenceWorkflowInfo(
+ workflow_id="workflow-1", workflow_type=WorkflowType.WORKFLOW, version="1", graph_data={}
+ ),
+ workflow_execution_repository=repository,
+ workflow_node_execution_repository=node_repository,
+ )
+ layer.initialize(ReadOnlyGraphRuntimeStateWrapper(runtime), InMemoryChannel())
+ layer.on_graph_start()
+ reason = WorkflowStartReason.INITIAL if index == 0 else WorkflowStartReason.RESUMPTION
+ layer.on_event(GraphRunStartedEvent(reason=reason))
+ with sqlite_session_factory() as session:
+ started_execution = session.get(WorkflowRun, "run-1")
+ assert started_execution is not None
+ assert started_execution.elapsed_time == total
+
+ clock[0] += duration
+ total += duration
+ is_paused = index < 2
+ layer.on_event(GraphRunPausedEvent() if is_paused else terminal_event)
+ with sqlite_session_factory() as session:
+ execution = session.get(WorkflowRun, "run-1")
+ assert execution is not None
+ assert execution.elapsed_time == total
+ assert execution.status == (WorkflowExecutionStatus.PAUSED if is_paused else status)
+ assert (execution.finished_at is None) == is_paused
+
+ # Queue processing and pause waits must not extend a settled segment.
+ clock[0] += 1000
+ converter = WorkflowResponseConverter(
+ application_generate_entity=entity,
+ user=account,
+ system_variables=system_variables,
+ )
+ converter.workflow_start_to_stream_response(
+ task_id="task-1", workflow_run_id="run-1", workflow_id="workflow-1", reason=reason
+ )
+ if is_paused:
+ responses = converter.workflow_pause_to_stream_response(
+ event=QueueWorkflowPausedEvent(), task_id="task-1", graph_runtime_state=runtime
+ )
+ pause_response = responses[-1]
+ assert isinstance(pause_response, WorkflowPauseStreamResponse)
+ assert pause_response.data.elapsed_time == total
+ context = WorkflowResumptionContext(
+ generate_entity=_WorkflowGenerateEntityWrapper(entity=entity),
+ serialized_graph_runtime_state=runtime.dumps(),
+ elapsed_time=total,
+ )
+ entity = WorkflowResumptionContext.loads(context.dumps()).get_generate_entity()
+ assert isinstance(entity, WorkflowAppGenerateEntity)
+ assert entity.workflow_execution_timing.elapsed_time == total
+ else:
+ response = converter.workflow_finish_to_stream_response(
+ task_id="task-1", workflow_id="workflow-1", status=status, graph_runtime_state=runtime
+ )
+ assert response.data.elapsed_time == 9.0
+ assert entity.workflow_execution_timing.finish() == 9.0
diff --git a/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.py b/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.py
index 01c4905e4cb..aff28b660aa 100644
--- a/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.py
+++ b/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.py
@@ -12,12 +12,48 @@ from graphon.enums import WorkflowExecutionStatus, WorkflowType
from models import Account, CreatorUserRole, EndUser, Tenant, WorkflowRun
from models.enums import EndUserType, WorkflowRunTriggeredFrom
from models.workflow import WorkflowType as ModelWorkflowType
+from repositories.entities.workflow_execution import WorkflowExecution as TimedWorkflowExecution
TABLES = (WorkflowRun,)
RESOURCE_TENANT_ID = "resource-tenant-id"
+def test_save_preserves_accumulated_duration_without_double_counting(
+ sqlite_session_factory: sessionmaker[Session], account: Account, sample_workflow_execution: WorkflowExecution
+) -> None:
+ repository = SQLAlchemyWorkflowExecutionRepository(
+ session_factory=sqlite_session_factory,
+ tenant_id=RESOURCE_TENANT_ID,
+ user=account,
+ app_id="app-1",
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ )
+ execution = TimedWorkflowExecution(
+ **(sample_workflow_execution.model_dump() | {"finished_at": None, "status": WorkflowExecutionStatus.RUNNING}),
+ accumulated_elapsed_time=5.0,
+ )
+ original_started_at = execution.started_at
+
+ for status, duration in [
+ (WorkflowExecutionStatus.RUNNING, 5.0),
+ (WorkflowExecutionStatus.PAUSED, 7.0),
+ (WorkflowExecutionStatus.SUCCEEDED, 9.0),
+ ]:
+ execution.status = status
+ execution.accumulated_elapsed_time = duration
+ if status == WorkflowExecutionStatus.SUCCEEDED:
+ execution.finished_at = datetime(2027, 1, 1, tzinfo=UTC)
+ repository.save(execution)
+ repository.save(execution)
+ with sqlite_session_factory() as session:
+ stored = session.get(WorkflowRun, execution.id_)
+ assert stored is not None
+ assert stored.elapsed_time == duration
+ assert stored.created_at == original_started_at.replace(tzinfo=None)
+ assert repository._to_domain_model(stored).elapsed_time == duration
+
+
def _make_account(*, tenant_id: str | None = None) -> Account:
account = Account(name="Repository User", email=f"{uuid4()}@example.com")
account.id = str(uuid4())
diff --git a/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.py b/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.py
index cb7bf043b8c..bf90bf1801f 100644
--- a/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.py
+++ b/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.py
@@ -52,3 +52,17 @@ def test_dict_to_workflow_run_defaults_missing_started_at_to_naive_utc_now() ->
assert model.created_at.tzinfo is None
# A naive local-time default would sit 5h30m ahead of UTC and drive elapsed_time negative.
assert abs((model.created_at - datetime.datetime.now(tz=datetime.UTC).replace(tzinfo=None)).total_seconds()) < 60
+
+
[email protected]("elapsed_time", ["0.0", "5.5"])
+def test_dict_to_workflow_run_prefers_stored_active_duration(elapsed_time: str) -> None:
+ model = _dict_to_workflow_run({**_BASE, "started_at": _START, "finished_at": _FINISH, "elapsed_time": elapsed_time})
+
+ assert model.elapsed_time == float(elapsed_time)
+
+
[email protected]("elapsed_time", [None, "null"])
+def test_dict_to_workflow_run_defaults_legacy_unfinished_duration_to_zero(elapsed_time: str | None) -> None:
+ model = _dict_to_workflow_run({**_BASE, "elapsed_time": elapsed_time})
+
+ assert model.elapsed_time == 0.0
diff --git a/api/tests/unit_tests/tasks/test_workflow_execution_tasks.py b/api/tests/unit_tests/tasks/test_workflow_execution_tasks.py
new file mode 100644
index 00000000000..16d634150e5
--- /dev/null
+++ b/api/tests/unit_tests/tasks/test_workflow_execution_tasks.py
@@ -0,0 +1,96 @@
+from datetime import datetime
+from unittest.mock import patch
+from uuid import uuid4
+
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.repositories.celery_workflow_execution_repository import CeleryWorkflowExecutionRepository
+from graphon.enums import WorkflowExecutionStatus, WorkflowType
+from models import Account, CreatorUserRole, WorkflowRun
+from models.enums import WorkflowRunTriggeredFrom
+from repositories.entities.workflow_execution import WorkflowExecution
+from tasks.workflow_execution_tasks import save_workflow_execution_task
+
+TABLES = (WorkflowRun,)
+
+
+def test_legacy_celery_payload_preserves_timestamp_based_duration(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ execution_id = str(uuid4())
+ graph: dict[str, object] = {}
+ inputs: dict[str, object] = {}
+ payload = {
+ "id_": execution_id,
+ "workflow_id": str(uuid4()),
+ "workflow_version": "1",
+ "workflow_type": WorkflowType.WORKFLOW.value,
+ "graph": graph,
+ "inputs": inputs,
+ "status": WorkflowExecutionStatus.SUCCEEDED.value,
+ "started_at": "2026-01-01T00:00:00",
+ "finished_at": "2026-01-01T00:00:15",
+ }
+
+ assert (
+ save_workflow_execution_task.run(
+ execution_data=payload,
+ tenant_id=str(uuid4()),
+ app_id=str(uuid4()),
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN.value,
+ creator_user_id=str(uuid4()),
+ creator_user_role=CreatorUserRole.ACCOUNT.value,
+ )
+ is True
+ )
+
+ with sqlite_session_factory() as session:
+ stored = session.get(WorkflowRun, execution_id)
+ assert stored is not None
+ assert stored.elapsed_time == 15.0
+
+
+def test_celery_roundtrip_preserves_accumulated_duration_on_create_update_and_redelivery(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ account = Account(name="Test", email="[email protected]")
+ account.id = str(uuid4())
+ repository = CeleryWorkflowExecutionRepository(
+ session_factory=sqlite_session_factory,
+ tenant_id=str(uuid4()),
+ user=account,
+ app_id=str(uuid4()),
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ )
+ execution = WorkflowExecution(
+ id_=str(uuid4()),
+ workflow_id=str(uuid4()),
+ workflow_version="1",
+ workflow_type=WorkflowType.WORKFLOW,
+ graph={},
+ inputs={},
+ started_at=datetime(2026, 1, 1),
+ accumulated_elapsed_time=5.0,
+ )
+
+ for status, duration in [
+ (WorkflowExecutionStatus.RUNNING, 5.0),
+ (WorkflowExecutionStatus.PAUSED, 7.0),
+ (WorkflowExecutionStatus.SUCCEEDED, 9.0),
+ ]:
+ execution.status = status
+ execution.accumulated_elapsed_time = duration
+ if status == WorkflowExecutionStatus.SUCCEEDED:
+ execution.finished_at = datetime(2026, 1, 2)
+ with patch("core.repositories.celery_workflow_execution_repository.save_workflow_execution_task") as task:
+ repository.save(execution)
+ payload = task.delay.call_args.kwargs
+ assert payload["execution_data"]["accumulated_elapsed_time"] == duration
+ assert payload["creator_user_role"] == CreatorUserRole.ACCOUNT
+ for _ in range(2):
+ assert save_workflow_execution_task.run(**payload) is True
+ with sqlite_session_factory() as session:
+ stored = session.get(WorkflowRun, execution.id_)
+ assert stored is not None
+ assert stored.elapsed_time == duration
+ assert stored.created_at == execution.started_at