Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix: accumulate workflow elapsed time across resumptions (#44161)
林玮 (Jade Lin)10285432d225
diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.pyindex 8497714f685..f837c9f8426 100644--- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py+++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py@@ -320,7 +320,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):paused_nodes=paused_nodes,reasons=reasons,status=WorkflowExecutionStatus.PAUSED,- elapsed_time=time.perf_counter() - self._base_task_pipeline.start_at,+ elapsed_time=self._application_generate_entity.workflow_execution_timing.finish(),total_tokens=runtime_state.total_tokens,total_steps=runtime_state.node_run_steps,),diff --git a/api/core/app/apps/common/workflow_response_converter.py b/api/core/app/apps/common/workflow_response_converter.pyindex 77ff812b2c9..205810adc9f 100644--- a/api/core/app/apps/common/workflow_response_converter.py+++ b/api/core/app/apps/common/workflow_response_converter.py@@ -245,6 +245,7 @@ class WorkflowResponseConverter:run_id = self._ensure_workflow_run_id(workflow_run_id)started_at = naive_utc_now()self._workflow_started_at = started_at+ self._application_generate_entity.workflow_execution_timing.start()return WorkflowStartStreamResponse(task_id=task_id,@@ -276,7 +277,7 @@ class WorkflowResponseConverter:)finished_at = naive_utc_now()- elapsed_time = (finished_at - started_at).total_seconds()+ elapsed_time = self._application_generate_entity.workflow_execution_timing.finish()outputs_mapping = graph_runtime_state.outputs or {}encoded_outputs = WorkflowRuntimeTypeConverter().to_json_encodable(outputs_mapping)@@ -329,8 +330,7 @@ class WorkflowResponseConverter:raise ValueError("workflow_pause_to_stream_response called before workflow_start_to_stream_response",)- paused_at = naive_utc_now()- elapsed_time = (paused_at - started_at).total_seconds()+ elapsed_time = self._application_generate_entity.workflow_execution_timing.finish()encoded_outputs = self._encode_outputs(event.outputs) or {}if self._application_generate_entity.invoke_from == InvokeFrom.SERVICE_API:encoded_outputs = {}diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.pyindex e5f9e68b2aa..b0065259d9e 100644--- a/api/core/app/apps/workflow/generate_task_pipeline.py+++ b/api/core/app/apps/workflow/generate_task_pipeline.py@@ -1,5 +1,4 @@import logging-import timefrom collections.abc import Callable, Generatorfrom contextlib import contextmanager@@ -214,7 +213,7 @@ class WorkflowAppGenerateTaskPipeline(GraphRuntimeStateSupport):status=WorkflowExecutionStatus.PAUSED,outputs={},error=None,- elapsed_time=time.perf_counter() - self._base_task_pipeline.start_at,+ elapsed_time=self._application_generate_entity.workflow_execution_timing.finish(),total_tokens=runtime_state.total_tokens,total_steps=runtime_state.node_run_steps,created_at=created_at,diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.pyindex cb3f52d12a0..a2557032f7c 100644--- a/api/core/app/entities/app_invoke_entities.py+++ b/api/core/app/entities/app_invoke_entities.py@@ -19,6 +19,7 @@ from core.entities.provider_configuration import ProviderModelBundlefrom graphon.file import File, FileUploadConfigfrom graphon.model_runtime.entities.model_entities import AIModelEntityfrom models.model import AppMode+from repositories.entities.workflow_execution import WorkflowExecutionTimingif TYPE_CHECKING:from core.ops.ops_trace_manager import TraceQueueManager@@ -307,6 +308,9 @@ class AdvancedChatAppGenerateEntity(ConversationAppGenerateEntity):app_config: WorkflowUIBasedAppConfig = None # type: ignoreworkflow_run_id: str | None = None+ workflow_execution_timing: WorkflowExecutionTiming = Field(+ default_factory=WorkflowExecutionTiming, exclude=True, repr=False+ )query: strclass SingleIterationRunEntity(BaseModel):@@ -338,6 +342,9 @@ class WorkflowAppGenerateEntity(AppGenerateEntity):# app configapp_config: WorkflowUIBasedAppConfig = None # type: ignoreworkflow_execution_id: str+ workflow_execution_timing: WorkflowExecutionTiming = Field(+ default_factory=WorkflowExecutionTiming, exclude=True, repr=False+ )class SingleIterationRunEntity(BaseModel):"""diff --git a/api/core/app/layers/pause_state_persist_layer.py b/api/core/app/layers/pause_state_persist_layer.pyindex 5d327adb065..b65728ab3b0 100644--- a/api/core/app/layers/pause_state_persist_layer.py+++ b/api/core/app/layers/pause_state_persist_layer.py@@ -14,6 +14,7 @@ from graphon.graph_engine.layers import GraphEngineLayerfrom graphon.graph_events import GraphEngineEvent, GraphRunPausedEventfrom models.model import AppModefrom repositories.api_workflow_run_repository import APIWorkflowRunRepository+from repositories.entities.workflow_execution import WorkflowExecutionTimingfrom repositories.factory import DifyAPIRepositoryFactory@@ -44,6 +45,7 @@ class WorkflowResumptionContext(BaseModel):# Only workflow / chatflow could be paused.generate_entity: _GenerateEntityUnionserialized_graph_runtime_state: str+ elapsed_time: float = Field(default=0.0, ge=0, allow_inf_nan=False)# Optional so that a workflow run paused before this field existed still# loads: it just degrades to fresh-filter behavior on resume for that one# stale run.@@ -54,7 +56,11 @@ class WorkflowResumptionContext(BaseModel):@classmethoddef loads(cls, value: str) -> Self:- return cls.model_validate_json(value)+ state = cls.model_validate_json(value)+ state.generate_entity.entity.workflow_execution_timing = WorkflowExecutionTiming(+ elapsed_time=state.elapsed_time,+ )+ return statedef get_generate_entity(self) -> WorkflowAppGenerateEntity | AdvancedChatAppGenerateEntity:return self.generate_entity.entity@@ -138,6 +144,7 @@ class PauseStatePersistenceLayer(GraphEngineLayer):state = WorkflowResumptionContext(serialized_graph_runtime_state=self.graph_runtime_state.dumps(),+ elapsed_time=self._generate_entity.workflow_execution_timing.finish(),generate_entity=entity_wrapper,serialized_response_stream_filter_state=self._response_stream_filter.dumps(),)diff --git a/api/core/app/workflow/layers/persistence.py b/api/core/app/workflow/layers/persistence.pyindex 6edc745701d..8b3119c1387 100644--- a/api/core/app/workflow/layers/persistence.py+++ b/api/core/app/workflow/layers/persistence.py@@ -24,7 +24,7 @@ from core.workflow.node_execution_process_data import preserve_workflow_agent_bifrom core.workflow.system_variables import SystemVariableKeyfrom core.workflow.variable_prefixes import SYSTEM_VARIABLE_NODE_IDfrom core.workflow.workflow_run_outputs import project_node_outputs_for_workflow_run-from graphon.entities import WorkflowExecution, WorkflowNodeExecution, WorkflowStartReason+from graphon.entities import WorkflowNodeExecution, WorkflowStartReasonfrom graphon.enums import (BuiltinNodeTypes,WorkflowExecutionStatus,@@ -50,6 +50,7 @@ from graphon.graph_events import ()from graphon.node_events import NodeRunResultfrom libs.datetime_utils import naive_utc_now+from repositories.entities.workflow_execution import WorkflowExecutionfrom services.workflow.inspector_events import (publish_node_changed as _inspector_publish_node_changed,)@@ -151,7 +152,9 @@ class WorkflowPersistenceLayer(GraphEngineLayer):# ------------------------------------------------------------------def _handle_graph_run_started(self, event: GraphRunStartedEvent | None = None) -> None:execution_id = self._get_execution_id()- workflow_execution = WorkflowExecution.new(+ timing = self._application_generate_entity.workflow_execution_timing+ timing.start()+ workflow_execution = WorkflowExecution(id_=execution_id,workflow_id=self._workflow_info.workflow_id,workflow_type=self._workflow_info.workflow_type,@@ -159,6 +162,7 @@ class WorkflowPersistenceLayer(GraphEngineLayer):graph=self._workflow_info.graph_data,inputs=self._prepare_workflow_inputs(),started_at=naive_utc_now(),+ accumulated_elapsed_time=timing.elapsed_time,)self._workflow_execution_repository.save(workflow_execution)@@ -413,6 +417,7 @@ class WorkflowPersistenceLayer(GraphEngineLayer):return merged_process_datadef _populate_completion_statistics(self, execution: WorkflowExecution, *, update_finished: bool = True) -> None:+ execution.accumulated_elapsed_time = self._application_generate_entity.workflow_execution_timing.finish()if update_finished:execution.finished_at = naive_utc_now()runtime_state = self.graph_runtime_statediff --git a/api/core/repositories/sqlalchemy_workflow_execution_repository.py b/api/core/repositories/sqlalchemy_workflow_execution_repository.pyindex fa68dfa84a0..6be44a69394 100644--- a/api/core/repositories/sqlalchemy_workflow_execution_repository.py+++ b/api/core/repositories/sqlalchemy_workflow_execution_repository.py@@ -20,6 +20,7 @@ from models import (WorkflowRun,)from models.enums import WorkflowRunTriggeredFrom+from repositories.entities.workflow_execution import WorkflowExecution as TimedWorkflowExecutionlogger = logging.getLogger(__name__)@@ -101,7 +102,7 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):# Convert status to domain enumstatus = WorkflowExecutionStatus(db_model.status)- return WorkflowExecution(+ return TimedWorkflowExecution(id_=db_model.id,workflow_id=db_model.workflow_id,workflow_type=WorkflowType(db_model.type),@@ -116,6 +117,7 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):exceptions_count=db_model.exceptions_count,started_at=db_model.created_at,finished_at=db_model.finished_at,+ accumulated_elapsed_time=db_model.elapsed_time,)def _to_db_model(self, domain_model: WorkflowExecution) -> WorkflowRun:@@ -167,9 +169,10 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):db_model.created_at = domain_model.started_atdb_model.finished_at = domain_model.finished_at- # Calculate elapsed time if finished_at is available- if domain_model.finished_at:- db_model.elapsed_time = (domain_model.finished_at - domain_model.started_at).total_seconds()+ if domain_model.finished_at or (+ isinstance(domain_model, TimedWorkflowExecution) and domain_model.accumulated_elapsed_time is not None+ ):+ db_model.elapsed_time = domain_model.elapsed_timeelse:db_model.elapsed_time = 0diff --git a/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py b/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.pyindex 40c638f9d35..ac8a75d8890 100644--- a/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py+++ b/api/extensions/logstore/repositories/logstore_api_workflow_run_repository.py@@ -130,9 +130,9 @@ def _dict_to_workflow_run(data: dict[str, Any]) -> WorkflowRun:case _:model.finished_at = ensure_naive_utc(finished_at)- # Compute elapsed_time from started_at and finished_at- # LogStore doesn't store elapsed_time, it's computed in WorkflowExecution domain entity- if model.finished_at and model.created_at:+ if data.get("elapsed_time") is not None:+ model.elapsed_time = safe_float(data["elapsed_time"])+ elif model.finished_at and model.created_at:model.elapsed_time = (model.finished_at - model.created_at).total_seconds()else:# Use safe conversion to handle 'null' strings and None valuesdiff --git a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.pyindex 55d66aae680..c19eb8aa4fb 100644--- a/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py+++ b/api/extensions/logstore/repositories/logstore_workflow_execution_repository.py@@ -143,6 +143,7 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):("total_tokens", str(domain_model.total_tokens)),("total_steps", str(domain_model.total_steps)),("exceptions_count", str(domain_model.exceptions_count)),+ ("elapsed_time", str(domain_model.elapsed_time)),("created_by_role",self._creator_user_role.valuediff --git a/api/repositories/entities/workflow_execution.py b/api/repositories/entities/workflow_execution.pynew file mode 100644index 00000000000..905b2cfb5d6--- /dev/null+++ b/api/repositories/entities/workflow_execution.py@@ -0,0 +1,46 @@+"""Workflow execution values shared across runtime and storage boundaries."""++from dataclasses import dataclass, field+from threading import Lock+from time import perf_counter+from typing import override++from pydantic import Field++from graphon.entities import WorkflowExecution as GraphonWorkflowExecution+++@dataclass+class WorkflowExecutionTiming:+ """Settle one execution segment once, excluding time spent paused."""++ elapsed_time: float = 0.0+ _started_at: float | None = field(default=None, init=False, repr=False, compare=False)+ _finished: bool = field(default=False, init=False, repr=False, compare=False)+ _lock: Lock = field(default_factory=Lock, init=False, repr=False, compare=False)++ def start(self) -> None:+ with self._lock:+ if self._started_at is None:+ self._started_at = perf_counter()++ def finish(self) -> float:+ with self._lock:+ if self._started_at is not None and not self._finished:+ self.elapsed_time += max(perf_counter() - self._started_at, 0.0)+ self._finished = True+ return self.elapsed_time+++class WorkflowExecution(GraphonWorkflowExecution):+ """Carry settled active duration independently of wall-clock timestamps."""++ accumulated_elapsed_time: float | None = Field(default=None, ge=0, allow_inf_nan=False)+ """None preserves timestamp-based duration for older serialized executions."""++ @property+ @override+ def elapsed_time(self) -> float:+ if self.accumulated_elapsed_time is not None:+ return self.accumulated_elapsed_time+ return super().elapsed_timediff --git a/api/tasks/workflow_execution_tasks.py b/api/tasks/workflow_execution_tasks.pyindex d1eb24d860d..9e10dca1006 100644--- a/api/tasks/workflow_execution_tasks.py+++ b/api/tasks/workflow_execution_tasks.py@@ -13,10 +13,10 @@ from celery import shared_taskfrom sqlalchemy import selectfrom core.db.session_factory import session_factory-from graphon.entities import WorkflowExecutionfrom graphon.workflow_type_encoder import WorkflowRuntimeTypeConverterfrom models import CreatorUserRole, WorkflowRunfrom models.enums import WorkflowRunTriggeredFrom+from repositories.entities.workflow_execution import WorkflowExecutionlogger = logging.getLogger(__name__)diff --git a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.pyindex f71c3694859..29a1f8722a1 100644--- a/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py+++ b/api/tests/unit_tests/controllers/service_api/app/test_hitl_service_api.py@@ -150,11 +150,17 @@ def _persist_human_input_form(def _build_service_api_pause_converter() -> WorkflowResponseConverter:- application_generate_entity = SimpleNamespace(+ application_generate_entity = WorkflowAppGenerateEntity(+ task_id="task",+ workflow_execution_id="run-id",+ user_id="user",+ stream=True,inputs={},files=[],invoke_from=InvokeFrom.SERVICE_API,- app_config=SimpleNamespace(app_id="app-id", tenant_id="tenant-id"),+ app_config=WorkflowUIBasedAppConfig(+ app_id="app-id", tenant_id="tenant-id", workflow_id="workflow-id", app_mode=AppMode.WORKFLOW+ ),)system_variables = build_system_variables(user_id="user",@@ -569,8 +575,7 @@ class TestHitlServiceApi:assert response.data.reasons[0]["form_id"] == "form-1"assert response.data.reasons[0]["expiration_time"] == 123- def test_workflow_blocking_pipeline_pause_payload_contract(self, monkeypatch: pytest.MonkeyPatch) -> None:- from core.app.apps.workflow import generate_task_pipeline as workflow_pipeline_module+ def test_workflow_blocking_pipeline_pause_payload_contract(self) -> None:from core.app.apps.workflow.generate_task_pipeline import WorkflowAppGenerateTaskPipelineapp_config = WorkflowUIBasedAppConfig(@@ -602,7 +607,6 @@ class TestHitlServiceApi:stream=False,draft_var_saver_factory=lambda **kwargs: None,)- monkeypatch.setattr(workflow_pipeline_module.time, "time", lambda: 1700000000)def _gen():yield HumanInputRequiredResponse(diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.pyindex 8b898d8bd2f..4ecfa826809 100644--- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py+++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_human_input.py@@ -1,17 +1,17 @@from datetime import UTC, datetime-from types import SimpleNamespace+from core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.apps.common.workflow_response_converter import WorkflowResponseConverter-from core.app.entities.app_invoke_entities import InvokeFrom+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntityfrom core.app.entities.queue_entities import QueueHumanInputFormFilledEvent, QueueHumanInputFormTimeoutEventfrom core.workflow.system_variables import build_system_variablesfrom graphon.entities import WorkflowStartReason-from graphon.runtime import GraphRuntimeState, VariablePoolfrom graphon.variables.segments import StringSegmentfrom models.account import Account+from models.model import AppMode-def _build_converter():+def _build_converter() -> WorkflowResponseConverter:system_variables = build_system_variables(files=[],user_id="user-1",@@ -19,15 +19,18 @@ def _build_converter():workflow_id="wf-1",workflow_execution_id="run-1",)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)- app_entity = SimpleNamespace(+ app_entity = WorkflowAppGenerateEntity(task_id="task-1",- app_config=SimpleNamespace(app_id="app-1", tenant_id="tenant-1"),+ app_config=WorkflowUIBasedAppConfig(+ app_id="app-1", tenant_id="tenant-1", workflow_id="wf-1", app_mode=AppMode.WORKFLOW+ ),invoke_from=InvokeFrom.EXPLORE,files=[],inputs={},workflow_execution_id="run-1",call_depth=0,+ user_id="user-1",+ stream=True,)account = Account(name="tester", email="[email protected]")account.id = "acc-1"@@ -38,7 +41,7 @@ def _build_converter():)-def test_human_input_form_filled_stream_response_contains_rendered_content():+def test_human_input_form_filled_stream_response_contains_rendered_content() -> None:converter = _build_converter()converter.workflow_start_to_stream_response(task_id="task-1",@@ -66,7 +69,7 @@ def test_human_input_form_filled_stream_response_contains_rendered_content():assert resp.data.action_id == "Approve"-def test_human_input_form_filled_stream_response_serializes_submitted_data():+def test_human_input_form_filled_stream_response_serializes_submitted_data() -> None:converter = _build_converter()converter.workflow_start_to_stream_response(task_id="task-1",@@ -97,7 +100,7 @@ def test_human_input_form_filled_stream_response_serializes_submitted_data():}-def test_human_input_form_timeout_stream_response_contains_timeout_metadata():+def test_human_input_form_timeout_stream_response_contains_timeout_metadata() -> None:converter = _build_converter()converter.workflow_start_to_stream_response(task_id="task-1",diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.pyindex 7e397ebc73f..f2167100fb2 100644--- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py+++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter_resumption.py@@ -1,11 +1,10 @@-from types import SimpleNamespace-+from core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.apps.common.workflow_response_converter import WorkflowResponseConverter-from core.app.entities.app_invoke_entities import InvokeFrom+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntityfrom core.workflow.system_variables import build_system_variablesfrom graphon.entities import WorkflowStartReason-from graphon.runtime import GraphRuntimeState, VariablePoolfrom models.account import Account+from models.model import AppModedef _build_converter() -> WorkflowResponseConverter:@@ -17,15 +16,18 @@ def _build_converter() -> WorkflowResponseConverter:workflow_id="wf-1",workflow_execution_id="run-1",)- runtime_state = GraphRuntimeState(variable_pool=VariablePool(), start_at=0.0)- app_entity = SimpleNamespace(+ app_entity = WorkflowAppGenerateEntity(task_id="task-1",- app_config=SimpleNamespace(app_id="app-1", tenant_id="tenant-1"),+ app_config=WorkflowUIBasedAppConfig(+ app_id="app-1", tenant_id="tenant-1", workflow_id="wf-1", app_mode=AppMode.WORKFLOW+ ),invoke_from=InvokeFrom.EXPLORE,files=[],inputs={},workflow_execution_id="run-1",call_depth=0,+ user_id="user-1",+ stream=True,)account = Account(name="tester", email="[email protected]")account.id = "acc-1"@@ -36,7 +38,7 @@ def _build_converter() -> WorkflowResponseConverter:)-def test_workflow_start_stream_response_carries_resumption_reason():+def test_workflow_start_stream_response_carries_resumption_reason() -> None:converter = _build_converter()resp = converter.workflow_start_to_stream_response(task_id="task-1",@@ -47,7 +49,7 @@ def test_workflow_start_stream_response_carries_resumption_reason():assert resp.data.reason is WorkflowStartReason.RESUMPTION-def test_workflow_start_stream_response_carries_initial_reason():+def test_workflow_start_stream_response_carries_initial_reason() -> None:converter = _build_converter()resp = converter.workflow_start_to_stream_response(task_id="task-1",diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py b/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.pyindex fb9b7f6a4b1..4fd4af28f38 100644--- a/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py+++ b/api/tests/unit_tests/core/app/apps/test_workflow_pause_events.py@@ -6,10 +6,11 @@ from unittest.mock import MagicMockimport pytestfrom sqlalchemy.orm import Session+from core.app.app_config.entities import WorkflowUIBasedAppConfigfrom core.app.apps.common import workflow_response_converterfrom core.app.apps.common.workflow_response_converter import WorkflowResponseConverterfrom core.app.apps.workflow.app_runner import WorkflowAppRunner-from core.app.entities.app_invoke_entities import InvokeFrom+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntityfrom core.app.entities.queue_entities import QueueWorkflowPausedEventfrom core.app.entities.task_entities import HumanInputRequiredResponse, WorkflowPauseStreamResponsefrom core.workflow.nodes.human_input.entities import (@@ -27,6 +28,7 @@ from graphon.graph_events import GraphRunPausedEventfrom graphon.runtime import GraphRuntimeState, VariablePoolfrom models.account import Accountfrom models.human_input import HumanInputForm, HumanInputFormRecipient, RecipientType+from models.model import AppModefrom models.workflow import Workflow, WorkflowType@@ -158,12 +160,18 @@ def test_graph_run_paused_event_emits_queue_pause_event(monkeypatch: pytest.Monkassert queue_event.paused_nodes == ["node-human"]-def _build_converter(*, invoke_from: InvokeFrom = InvokeFrom.SERVICE_API):- application_generate_entity = SimpleNamespace(+def _build_converter(*, invoke_from: InvokeFrom = InvokeFrom.SERVICE_API) -> WorkflowResponseConverter:+ application_generate_entity = WorkflowAppGenerateEntity(+ task_id="task",+ workflow_execution_id="run-id",+ user_id="user",+ stream=True,inputs={},files=[],invoke_from=invoke_from,- app_config=SimpleNamespace(app_id="app-id", tenant_id="tenant-id"),+ app_config=WorkflowUIBasedAppConfig(+ app_id="app-id", tenant_id="tenant-id", workflow_id="workflow-id", app_mode=AppMode.WORKFLOW+ ),)system_variables = build_system_variables(user_id="user",diff --git a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.pyindex 697cd531663..5b4cdd2b824 100644--- a/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py+++ b/api/tests/unit_tests/core/app/layers/test_pause_state_persist_layer.py@@ -254,6 +254,11 @@ class TestPauseStatePersistenceLayer:self, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]):generate_entity = self._create_generate_entity(workflow_execution_id="run-123")+ clock = [10.0]+ monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])+ generate_entity.workflow_execution_timing.elapsed_time = 5.0+ generate_entity.workflow_execution_timing.start()+ clock[0] = 12.0layer = PauseStatePersistenceLayer(session_factory=sqlite_session_factory,state_owner_user_id="owner-123",@@ -285,6 +290,8 @@ class TestPauseStatePersistenceLayer:assert call_kwargs["state_owner_user_id"] == "owner-123"serialized_state = call_kwargs["state"]resumption_context = WorkflowResumptionContext.loads(serialized_state)+ assert resumption_context.elapsed_time == 7.0+ assert resumption_context.get_generate_entity().workflow_execution_timing.elapsed_time == 7.0assert resumption_context.serialized_graph_runtime_state == expected_stateassert resumption_context.get_generate_entity().model_dump() == generate_entity.model_dump()pause_reasons = call_kwargs["pause_reasons"]@@ -553,3 +560,14 @@ def test_get_response_stream_filter_defaults_when_state_missing() -> None:restored = context.get_response_stream_filter()assert isinstance(restored, ResponseStreamFilter)+++def test_resumption_context_defaults_elapsed_time_for_legacy_snapshots() -> None:+ context = _build_workflow_generate_entity_for_roundtrip()+ payload = context.model_dump()+ payload.pop("elapsed_time")++ restored = WorkflowResumptionContext.loads(json.dumps(payload))++ assert restored.elapsed_time == 0.0+ assert restored.get_generate_entity().workflow_execution_timing.elapsed_time == 0.0diff --git a/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.py b/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.pynew file mode 100644index 00000000000..61876bc05fb--- /dev/null+++ b/api/tests/unit_tests/core/app/workflow/layers/test_execution_timing.py@@ -0,0 +1,144 @@+import pytest+from sqlalchemy.orm import Session, sessionmaker++from core.app.app_config.entities import WorkflowUIBasedAppConfig+from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter+from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity+from core.app.entities.queue_entities import QueueWorkflowPausedEvent+from core.app.entities.task_entities import WorkflowPauseStreamResponse+from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext, _WorkflowGenerateEntityWrapper+from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer+from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from core.workflow.system_variables import build_system_variables+from graphon.entities import WorkflowStartReason+from graphon.enums import WorkflowExecutionStatus, WorkflowType+from graphon.graph_engine.command_channels import InMemoryChannel+from graphon.graph_events import (+ GraphRunAbortedEvent,+ GraphRunFailedEvent,+ GraphRunPartialSucceededEvent,+ GraphRunPausedEvent,+ GraphRunStartedEvent,+ GraphRunSucceededEvent,+)+from graphon.runtime import GraphRuntimeState, ReadOnlyGraphRuntimeStateWrapper, VariablePool+from models.account import Account+from models.enums import WorkflowRunTriggeredFrom+from models.model import AppMode+from models.workflow import WorkflowNodeExecutionTriggeredFrom, WorkflowRun+++ ("terminal_event", "status"),+ [+ (GraphRunSucceededEvent(), WorkflowExecutionStatus.SUCCEEDED),+ (GraphRunPartialSucceededEvent(exceptions_count=1), WorkflowExecutionStatus.PARTIAL_SUCCEEDED),+ (GraphRunFailedEvent(error="failed"), WorkflowExecutionStatus.FAILED),+ (GraphRunAbortedEvent(reason="stopped"), WorkflowExecutionStatus.STOPPED),+ ],+)+def test_elapsed_time_accumulates_across_pauses_and_matches_delayed_stream_responses(+ monkeypatch: pytest.MonkeyPatch,+ sqlite_session_factory: sessionmaker[Session],+ terminal_event: GraphRunSucceededEvent | GraphRunPartialSucceededEvent | GraphRunFailedEvent | GraphRunAbortedEvent,+ status: WorkflowExecutionStatus,+) -> None:+ clock = [100.0]+ monkeypatch.setattr("repositories.entities.workflow_execution.perf_counter", lambda: clock[0])+ monkeypatch.setattr("core.app.workflow.layers.persistence._inspector_publish_workflow_completed", lambda **_: None)+ entity = WorkflowAppGenerateEntity(+ task_id="task-1",+ workflow_execution_id="run-1",+ app_config=WorkflowUIBasedAppConfig(+ tenant_id="tenant-1", app_id="app-1", workflow_id="workflow-1", app_mode=AppMode.WORKFLOW+ ),+ inputs={},+ files=[],+ user_id="user-1",+ stream=True,+ invoke_from=InvokeFrom.SERVICE_API,+ )+ system_variables = build_system_variables(workflow_execution_id="run-1")+ total = 0.0+ account = Account(name="Test", email="[email protected]")+ account.id = "user-1"++ for index, duration in enumerate([2.0, 3.0, 4.0]):+ pool = VariablePool()+ for variable in system_variables:+ pool.add(variable.selector, variable)+ runtime = GraphRuntimeState(variable_pool=pool, start_at=clock[0])+ repository = SQLAlchemyWorkflowExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=account,+ app_id="app-1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ )+ node_repository = SQLAlchemyWorkflowNodeExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id="tenant-1",+ user=account,+ app_id="app-1",+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ )+ layer = WorkflowPersistenceLayer(+ application_generate_entity=entity,+ workflow_info=PersistenceWorkflowInfo(+ workflow_id="workflow-1", workflow_type=WorkflowType.WORKFLOW, version="1", graph_data={}+ ),+ workflow_execution_repository=repository,+ workflow_node_execution_repository=node_repository,+ )+ layer.initialize(ReadOnlyGraphRuntimeStateWrapper(runtime), InMemoryChannel())+ layer.on_graph_start()+ reason = WorkflowStartReason.INITIAL if index == 0 else WorkflowStartReason.RESUMPTION+ layer.on_event(GraphRunStartedEvent(reason=reason))+ with sqlite_session_factory() as session:+ started_execution = session.get(WorkflowRun, "run-1")+ assert started_execution is not None+ assert started_execution.elapsed_time == total++ clock[0] += duration+ total += duration+ is_paused = index < 2+ layer.on_event(GraphRunPausedEvent() if is_paused else terminal_event)+ with sqlite_session_factory() as session:+ execution = session.get(WorkflowRun, "run-1")+ assert execution is not None+ assert execution.elapsed_time == total+ assert execution.status == (WorkflowExecutionStatus.PAUSED if is_paused else status)+ assert (execution.finished_at is None) == is_paused++ # Queue processing and pause waits must not extend a settled segment.+ clock[0] += 1000+ converter = WorkflowResponseConverter(+ application_generate_entity=entity,+ user=account,+ system_variables=system_variables,+ )+ converter.workflow_start_to_stream_response(+ task_id="task-1", workflow_run_id="run-1", workflow_id="workflow-1", reason=reason+ )+ if is_paused:+ responses = converter.workflow_pause_to_stream_response(+ event=QueueWorkflowPausedEvent(), task_id="task-1", graph_runtime_state=runtime+ )+ pause_response = responses[-1]+ assert isinstance(pause_response, WorkflowPauseStreamResponse)+ assert pause_response.data.elapsed_time == total+ context = WorkflowResumptionContext(+ generate_entity=_WorkflowGenerateEntityWrapper(entity=entity),+ serialized_graph_runtime_state=runtime.dumps(),+ elapsed_time=total,+ )+ entity = WorkflowResumptionContext.loads(context.dumps()).get_generate_entity()+ assert isinstance(entity, WorkflowAppGenerateEntity)+ assert entity.workflow_execution_timing.elapsed_time == total+ else:+ response = converter.workflow_finish_to_stream_response(+ task_id="task-1", workflow_id="workflow-1", status=status, graph_runtime_state=runtime+ )+ assert response.data.elapsed_time == 9.0+ assert entity.workflow_execution_timing.finish() == 9.0diff --git a/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.py b/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.pyindex 01c4905e4cb..aff28b660aa 100644--- a/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.py+++ b/api/tests/unit_tests/core/repositories/test_sqlalchemy_workflow_execution_repository.py@@ -12,12 +12,48 @@ from graphon.enums import WorkflowExecutionStatus, WorkflowTypefrom models import Account, CreatorUserRole, EndUser, Tenant, WorkflowRunfrom models.enums import EndUserType, WorkflowRunTriggeredFromfrom models.workflow import WorkflowType as ModelWorkflowType+from repositories.entities.workflow_execution import WorkflowExecution as TimedWorkflowExecutionTABLES = (WorkflowRun,)RESOURCE_TENANT_ID = "resource-tenant-id"+def test_save_preserves_accumulated_duration_without_double_counting(+ sqlite_session_factory: sessionmaker[Session], account: Account, sample_workflow_execution: WorkflowExecution+) -> None:+ repository = SQLAlchemyWorkflowExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id=RESOURCE_TENANT_ID,+ user=account,+ app_id="app-1",+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ )+ execution = TimedWorkflowExecution(+ **(sample_workflow_execution.model_dump() | {"finished_at": None, "status": WorkflowExecutionStatus.RUNNING}),+ accumulated_elapsed_time=5.0,+ )+ original_started_at = execution.started_at++ for status, duration in [+ (WorkflowExecutionStatus.RUNNING, 5.0),+ (WorkflowExecutionStatus.PAUSED, 7.0),+ (WorkflowExecutionStatus.SUCCEEDED, 9.0),+ ]:+ execution.status = status+ execution.accumulated_elapsed_time = duration+ if status == WorkflowExecutionStatus.SUCCEEDED:+ execution.finished_at = datetime(2027, 1, 1, tzinfo=UTC)+ repository.save(execution)+ repository.save(execution)+ with sqlite_session_factory() as session:+ stored = session.get(WorkflowRun, execution.id_)+ assert stored is not None+ assert stored.elapsed_time == duration+ assert stored.created_at == original_started_at.replace(tzinfo=None)+ assert repository._to_domain_model(stored).elapsed_time == duration++def _make_account(*, tenant_id: str | None = None) -> Account:account = Account(name="Repository User", email=f"{uuid4()}@example.com")account.id = str(uuid4())diff --git a/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.py b/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.pyindex cb7bf043b8c..bf90bf1801f 100644--- a/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.py+++ b/api/tests/unit_tests/extensions/logstore/repositories/test_logstore_api_workflow_run_repository.py@@ -52,3 +52,17 @@ def test_dict_to_workflow_run_defaults_missing_started_at_to_naive_utc_now() ->assert model.created_at.tzinfo is None# A naive local-time default would sit 5h30m ahead of UTC and drive elapsed_time negative.assert abs((model.created_at - datetime.datetime.now(tz=datetime.UTC).replace(tzinfo=None)).total_seconds()) < 60++[email protected]("elapsed_time", ["0.0", "5.5"])+def test_dict_to_workflow_run_prefers_stored_active_duration(elapsed_time: str) -> None:+ model = _dict_to_workflow_run({**_BASE, "started_at": _START, "finished_at": _FINISH, "elapsed_time": elapsed_time})++ assert model.elapsed_time == float(elapsed_time)++[email protected]("elapsed_time", [None, "null"])+def test_dict_to_workflow_run_defaults_legacy_unfinished_duration_to_zero(elapsed_time: str | None) -> None:+ model = _dict_to_workflow_run({**_BASE, "elapsed_time": elapsed_time})++ assert model.elapsed_time == 0.0diff --git a/api/tests/unit_tests/tasks/test_workflow_execution_tasks.py b/api/tests/unit_tests/tasks/test_workflow_execution_tasks.pynew file mode 100644index 00000000000..16d634150e5--- /dev/null+++ b/api/tests/unit_tests/tasks/test_workflow_execution_tasks.py@@ -0,0 +1,96 @@+from datetime import datetime+from unittest.mock import patch+from uuid import uuid4++from sqlalchemy.orm import Session, sessionmaker++from core.repositories.celery_workflow_execution_repository import CeleryWorkflowExecutionRepository+from graphon.enums import WorkflowExecutionStatus, WorkflowType+from models import Account, CreatorUserRole, WorkflowRun+from models.enums import WorkflowRunTriggeredFrom+from repositories.entities.workflow_execution import WorkflowExecution+from tasks.workflow_execution_tasks import save_workflow_execution_task++TABLES = (WorkflowRun,)+++def test_legacy_celery_payload_preserves_timestamp_based_duration(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ execution_id = str(uuid4())+ graph: dict[str, object] = {}+ inputs: dict[str, object] = {}+ payload = {+ "id_": execution_id,+ "workflow_id": str(uuid4()),+ "workflow_version": "1",+ "workflow_type": WorkflowType.WORKFLOW.value,+ "graph": graph,+ "inputs": inputs,+ "status": WorkflowExecutionStatus.SUCCEEDED.value,+ "started_at": "2026-01-01T00:00:00",+ "finished_at": "2026-01-01T00:00:15",+ }++ assert (+ save_workflow_execution_task.run(+ execution_data=payload,+ tenant_id=str(uuid4()),+ app_id=str(uuid4()),+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN.value,+ creator_user_id=str(uuid4()),+ creator_user_role=CreatorUserRole.ACCOUNT.value,+ )+ is True+ )++ with sqlite_session_factory() as session:+ stored = session.get(WorkflowRun, execution_id)+ assert stored is not None+ assert stored.elapsed_time == 15.0+++def test_celery_roundtrip_preserves_accumulated_duration_on_create_update_and_redelivery(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ account = Account(name="Test", email="[email protected]")+ account.id = str(uuid4())+ repository = CeleryWorkflowExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id=str(uuid4()),+ user=account,+ app_id=str(uuid4()),+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ )+ execution = WorkflowExecution(+ id_=str(uuid4()),+ workflow_id=str(uuid4()),+ workflow_version="1",+ workflow_type=WorkflowType.WORKFLOW,+ graph={},+ inputs={},+ started_at=datetime(2026, 1, 1),+ accumulated_elapsed_time=5.0,+ )++ for status, duration in [+ (WorkflowExecutionStatus.RUNNING, 5.0),+ (WorkflowExecutionStatus.PAUSED, 7.0),+ (WorkflowExecutionStatus.SUCCEEDED, 9.0),+ ]:+ execution.status = status+ execution.accumulated_elapsed_time = duration+ if status == WorkflowExecutionStatus.SUCCEEDED:+ execution.finished_at = datetime(2026, 1, 2)+ with patch("core.repositories.celery_workflow_execution_repository.save_workflow_execution_task") as task:+ repository.save(execution)+ payload = task.delay.call_args.kwargs+ assert payload["execution_data"]["accumulated_elapsed_time"] == duration+ assert payload["creator_user_role"] == CreatorUserRole.ACCOUNT+ for _ in range(2):+ assert save_workflow_execution_task.run(**payload) is True+ with sqlite_session_factory() as session:+ stored = session.get(WorkflowRun, execution.id_)+ assert stored is not None+ assert stored.elapsed_time == duration+ assert stored.created_at == execution.started_at