Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[SKY-8767] fix: persist ForLoopBlock output incrementally to survive activity timeouts (#5393)
Marc Kelechavab504bbd3bc98
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex befea3563..77dd3a6fd 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -159,6 +159,9 @@ TASKV2_TO_BLOCK_STATUS: dict[TaskV2Status, BlockStatus] = {# ForLoop constantsDEFAULT_MAX_LOOP_ITERATIONS = 100+# Persist accumulated loop output to DB every N iterations to survive timeouts.+# Trades up to N-1 iterations of data loss for O(N/K) writes instead of O(N).+PERSIST_LOOP_OUTPUT_INTERVAL = 10DEFAULT_MAX_STEPS_PER_ITERATION = 50@@ -1776,6 +1779,42 @@ class ForLoopBlock(Block):if isinstance(block, ForLoopBlock):block.validate_loop_blocks()+ async def _persist_partial_loop_output(+ self,+ workflow_run_id: str,+ outputs_with_loop_values: list[list[dict[str, Any]]],+ loop_idx: int,+ ) -> None:+ """Persist partial for-loop output to DB so data survives Temporal+ activity timeouts. The timeout handler runs on a different node and+ reads from DB — without this, accumulated iteration data is lost when+ the loop is killed mid-execution.++ Uses the DB UPSERT directly instead of record_output_parameter_value+ to avoid re-registering context parameters and emitting spurious+ 'already has a registered value' warnings on every call.++ On the normal iteration path, this is called every+ PERSIST_LOOP_OUTPUT_INTERVAL iterations and on the final iteration+ to balance durability vs DB load. Early-return paths (failure,+ cancellation) always persist since they are terminal."""+ if not self.output_parameter:+ return+ try:+ await app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter(+ workflow_run_id=workflow_run_id,+ output_parameter_id=self.output_parameter.output_parameter_id,+ value=outputs_with_loop_values,+ )+ except Exception:+ LOG.warning(+ "Failed to incrementally persist for-loop output",+ workflow_run_id=workflow_run_id,+ output_parameter_id=self.output_parameter.output_parameter_id,+ loop_idx=loop_idx,+ exc_info=True,+ )+async def execute_loop_helper(self,workflow_run_id: str,@@ -1809,6 +1848,7 @@ class ForLoopBlock(Block):organization_id=organization_id,)block_outputs.append(failure_block_result)+ await self._persist_partial_loop_output(workflow_run_id, outputs_with_loop_values, loop_idx)return LoopBlockExecutedResult(outputs_with_loop_values=outputs_with_loop_values,block_outputs=block_outputs,@@ -1880,6 +1920,7 @@ class ForLoopBlock(Block):)block_outputs.append(failure_block_result)outputs_with_loop_values.append(each_loop_output_values)+ await self._persist_partial_loop_output(workflow_run_id, outputs_with_loop_values, loop_idx)return LoopBlockExecutedResult(outputs_with_loop_values=outputs_with_loop_values,block_outputs=block_outputs,@@ -1981,6 +2022,7 @@ class ForLoopBlock(Block):# If next_loop_on_failure is False, stop the entire loopif not self.next_loop_on_failure:outputs_with_loop_values.append(each_loop_output_values)+ await self._persist_partial_loop_output(workflow_run_id, outputs_with_loop_values, loop_idx)return LoopBlockExecutedResult(outputs_with_loop_values=outputs_with_loop_values,block_outputs=block_outputs,@@ -1998,6 +2040,7 @@ class ForLoopBlock(Block):block_result=block_outputs,)outputs_with_loop_values.append(each_loop_output_values)+ await self._persist_partial_loop_output(workflow_run_id, outputs_with_loop_values, loop_idx)return LoopBlockExecutedResult(outputs_with_loop_values=outputs_with_loop_values,block_outputs=block_outputs,@@ -2021,6 +2064,7 @@ class ForLoopBlock(Block):next_loop_on_failure=loop_block.next_loop_on_failure or self.next_loop_on_failure,)outputs_with_loop_values.append(each_loop_output_values)+ await self._persist_partial_loop_output(workflow_run_id, outputs_with_loop_values, loop_idx)return LoopBlockExecutedResult(outputs_with_loop_values=outputs_with_loop_values,block_outputs=block_outputs,@@ -2052,6 +2096,7 @@ class ForLoopBlock(Block):)block_outputs.append(failure_block_result)outputs_with_loop_values.append(each_loop_output_values)+ await self._persist_partial_loop_output(workflow_run_id, outputs_with_loop_values, loop_idx)return LoopBlockExecutedResult(outputs_with_loop_values=outputs_with_loop_values,block_outputs=block_outputs,@@ -2076,6 +2121,9 @@ class ForLoopBlock(Block):breakoutputs_with_loop_values.append(each_loop_output_values)+ is_last_iteration = loop_idx == len(loop_over_values) - 1+ if loop_idx % PERSIST_LOOP_OUTPUT_INTERVAL == 0 or is_last_iteration:+ await self._persist_partial_loop_output(workflow_run_id, outputs_with_loop_values, loop_idx)return LoopBlockExecutedResult(outputs_with_loop_values=outputs_with_loop_values,diff --git a/tests/unit/test_forloop_incremental_persist.py b/tests/unit/test_forloop_incremental_persist.pynew file mode 100644index 000000000..843381cfd--- /dev/null+++ b/tests/unit/test_forloop_incremental_persist.py@@ -0,0 +1,401 @@+"""Tests for ForLoopBlock incremental output persistence.++Validates that:+1. Partial for-loop output is persisted to DB at intervals via direct DB call+2. Context registration (record_output_parameter_value) happens only once at final completion+3. Incremental persist failures don't break the loop+4. Accumulated output grows correctly across iterations+5. PERSIST_LOOP_OUTPUT_INTERVAL controls write frequency (O(N/K) not O(N))+"""++from datetime import UTC, datetime+from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.forge.sdk.workflow.models.block import Block, ForLoopBlock, LoopBlockExecutedResult, TaskBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter+from skyvern.schemas.workflows import BlockResult, BlockStatus++INTERVAL_PATCH = "skyvern.forge.sdk.workflow.models.block.PERSIST_LOOP_OUTPUT_INTERVAL"+++def _make_output_param(label: str) -> OutputParameter:+ now = datetime.now(UTC)+ return OutputParameter(+ output_parameter_id=f"op_{label}",+ key=f"{label}_output",+ workflow_id="wf_test",+ created_at=now,+ modified_at=now,+ )+++def _make_block_result(output_param: OutputParameter, value: dict[str, Any] | None = None) -> BlockResult:+ return BlockResult(+ success=True,+ output_parameter=output_param,+ output_parameter_value=value or {"extracted": "data"},+ status=BlockStatus.completed,+ )+++class TestExecuteCallsRecordOnceAtEnd:+ """ForLoopBlock.execute() must call record_output_parameter_value exactly once+ (at the end, after execute_loop_helper returns), not during iterations."""++ @pytest.mark.asyncio+ async def test_record_output_called_once_after_helper_returns(self) -> None:+ inner_task = TaskBlock(label="inner_task", output_parameter=_make_output_param("inner_task"))+ loop_block = ForLoopBlock(+ label="test_loop",+ output_parameter=_make_output_param("test_loop"),+ loop_blocks=[inner_task],+ )++ loop_result = LoopBlockExecutedResult(+ outputs_with_loop_values=[[{"loop_value": "a"}], [{"loop_value": "b"}], [{"loop_value": "c"}]],+ block_outputs=[_make_block_result(inner_task.output_parameter)] * 3,+ last_block=inner_task,+ )++ mock_context = MagicMock()+ final_result = _make_block_result(loop_block.output_parameter)++ with (+ patch.object(Block, "get_workflow_run_context", return_value=mock_context),+ patch.object(+ ForLoopBlock, "get_loop_over_parameter_values", new_callable=AsyncMock, return_value=["a", "b", "c"]+ ),+ patch.object(ForLoopBlock, "execute_loop_helper", new_callable=AsyncMock, return_value=loop_result),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock) as mock_record,+ patch.object(Block, "build_block_result", new_callable=AsyncMock, return_value=final_result),+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ ):+ mock_app.DATABASE.observer.update_workflow_run_block = AsyncMock()++ await loop_block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_loop",+ organization_id="org_test",+ )++ # record_output_parameter_value called exactly once — the final write+ assert mock_record.call_count == 1+ call_args = mock_record.call_args+ assert call_args.args[1] == "wr_test" # workflow_run_id+ assert call_args.args[2] == loop_result.outputs_with_loop_values # full accumulated output+++class TestExecuteLoopHelperPersistsToDbDirectly:+ """execute_loop_helper must persist partial output via direct DB UPSERT,+ not through record_output_parameter_value (which re-registers context)."""++ @pytest.mark.asyncio+ async def test_db_upsert_called_per_iteration_not_record_output(self) -> None:+ """With interval=1, each iteration triggers a DB UPSERT. record_output_parameter_value+ must NOT be called within the helper — that's execute()'s responsibility."""+ inner_task = TaskBlock(label="inner_task", output_parameter=_make_output_param("inner_task"))+ loop_block = ForLoopBlock(+ label="test_loop",+ output_parameter=_make_output_param("test_loop"),+ loop_blocks=[inner_task],+ )++ inner_result = _make_block_result(inner_task.output_parameter)+ mock_context = MagicMock()+ mock_context.has_value.return_value = False+ mock_context.set_value = MagicMock()+ mock_context.update_block_metadata = MagicMock()++ mock_db_upsert = AsyncMock()++ with (+ patch(INTERVAL_PATCH, 1),+ patch.object(Block, "execute_safe", new_callable=AsyncMock, return_value=inner_result),+ patch.object(ForLoopBlock, "get_loop_block_context_parameters", return_value=[]),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock) as mock_record,+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch("skyvern.forge.sdk.workflow.models.block.skyvern_context") as mock_skyvern_ctx,+ ):+ mock_skyvern_ctx.current.return_value = None+ mock_app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = mock_db_upsert++ result = await loop_block.execute_loop_helper(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_loop",+ workflow_run_context=mock_context,+ loop_over_values=["a", "b", "c"],+ organization_id="org_test",+ )++ # DB UPSERT called once per iteration (interval=1, 3 iterations)+ assert mock_db_upsert.call_count == 3+ # record_output_parameter_value NOT called during iterations+ assert mock_record.call_count == 0+ # All 3 iterations produced output+ assert len(result.outputs_with_loop_values) == 3++ @pytest.mark.asyncio+ async def test_accumulated_output_grows_per_iteration(self) -> None:+ """Each DB UPSERT should contain all iterations completed so far."""+ inner_task = TaskBlock(label="inner_task", output_parameter=_make_output_param("inner_task"))+ loop_block = ForLoopBlock(+ label="test_loop",+ output_parameter=_make_output_param("test_loop"),+ loop_blocks=[inner_task],+ )++ inner_result = _make_block_result(inner_task.output_parameter, {"med": "data"})+ mock_context = MagicMock()+ mock_context.has_value.return_value = False+ mock_context.set_value = MagicMock()+ mock_context.update_block_metadata = MagicMock()++ # Capture copies of the value at each call since the list is mutated in-place+ captured_values: list[list] = []++ async def capture_upsert(**kwargs: Any) -> None:+ captured_values.append(list(kwargs["value"]))++ mock_db_upsert = AsyncMock(side_effect=capture_upsert)++ with (+ patch(INTERVAL_PATCH, 1),+ patch.object(Block, "execute_safe", new_callable=AsyncMock, return_value=inner_result),+ patch.object(ForLoopBlock, "get_loop_block_context_parameters", return_value=[]),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch("skyvern.forge.sdk.workflow.models.block.skyvern_context") as mock_skyvern_ctx,+ ):+ mock_skyvern_ctx.current.return_value = None+ mock_app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = mock_db_upsert++ await loop_block.execute_loop_helper(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_loop",+ workflow_run_context=mock_context,+ loop_over_values=["a", "b", "c"],+ organization_id="org_test",+ )++ # Verify accumulation: snapshot i has i items (interval=1 → persist every iteration)+ assert len(captured_values) == 3+ for i, snapshot in enumerate(captured_values, start=1):+ assert len(snapshot) == i, f"Iteration {i}: expected {i} accumulated outputs, got {len(snapshot)}"++ @pytest.mark.asyncio+ async def test_upsert_uses_forloop_output_parameter_id(self) -> None:+ """The DB UPSERT must target the ForLoopBlock's own output_parameter_id."""+ inner_task = TaskBlock(label="inner_task", output_parameter=_make_output_param("inner_task"))+ loop_block = ForLoopBlock(+ label="test_loop",+ output_parameter=_make_output_param("test_loop"),+ loop_blocks=[inner_task],+ )++ inner_result = _make_block_result(inner_task.output_parameter)+ mock_context = MagicMock()+ mock_context.has_value.return_value = False+ mock_context.set_value = MagicMock()+ mock_context.update_block_metadata = MagicMock()++ mock_db_upsert = AsyncMock()++ with (+ patch.object(Block, "execute_safe", new_callable=AsyncMock, return_value=inner_result),+ patch.object(ForLoopBlock, "get_loop_block_context_parameters", return_value=[]),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch("skyvern.forge.sdk.workflow.models.block.skyvern_context") as mock_skyvern_ctx,+ ):+ mock_skyvern_ctx.current.return_value = None+ mock_app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = mock_db_upsert++ await loop_block.execute_loop_helper(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_loop",+ workflow_run_context=mock_context,+ loop_over_values=["x"],+ organization_id="org_test",+ )++ call_kwargs = mock_db_upsert.call_args_list[0].kwargs+ assert call_kwargs["workflow_run_id"] == "wr_test"+ assert call_kwargs["output_parameter_id"] == "op_test_loop"+++class TestIncrementalPersistFailureResilience:+ """Incremental DB persist failures must not break the loop."""++ @pytest.mark.asyncio+ async def test_db_error_does_not_stop_loop(self) -> None:+ """If the DB UPSERT fails on every iteration, the loop still completes."""+ inner_task = TaskBlock(label="inner_task", output_parameter=_make_output_param("inner_task"))+ loop_block = ForLoopBlock(+ label="test_loop",+ output_parameter=_make_output_param("test_loop"),+ loop_blocks=[inner_task],+ )++ inner_result = _make_block_result(inner_task.output_parameter)+ mock_context = MagicMock()+ mock_context.has_value.return_value = False+ mock_context.set_value = MagicMock()+ mock_context.update_block_metadata = MagicMock()++ mock_db_upsert = AsyncMock(side_effect=Exception("DB connection lost"))++ with (+ patch(INTERVAL_PATCH, 1),+ patch.object(Block, "execute_safe", new_callable=AsyncMock, return_value=inner_result),+ patch.object(ForLoopBlock, "get_loop_block_context_parameters", return_value=[]),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch("skyvern.forge.sdk.workflow.models.block.skyvern_context") as mock_skyvern_ctx,+ ):+ mock_skyvern_ctx.current.return_value = None+ mock_app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = mock_db_upsert++ result = await loop_block.execute_loop_helper(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_loop",+ workflow_run_context=mock_context,+ loop_over_values=["a", "b", "c"],+ organization_id="org_test",+ )++ # All 3 iterations attempted despite DB failures (interval=1)+ assert mock_db_upsert.call_count == 3+ # All 3 iterations produced output+ assert len(result.outputs_with_loop_values) == 3++ @pytest.mark.asyncio+ async def test_transient_db_failure_still_persists_later_iterations(self) -> None:+ """If DB fails on iteration 1 but succeeds on 2+3, those writes go through."""+ inner_task = TaskBlock(label="inner_task", output_parameter=_make_output_param("inner_task"))+ loop_block = ForLoopBlock(+ label="test_loop",+ output_parameter=_make_output_param("test_loop"),+ loop_blocks=[inner_task],+ )++ inner_result = _make_block_result(inner_task.output_parameter)+ mock_context = MagicMock()+ mock_context.has_value.return_value = False+ mock_context.set_value = MagicMock()+ mock_context.update_block_metadata = MagicMock()++ # Capture copies on successful calls to verify accumulation+ captured_values: list[list] = []+ call_count = 0++ async def transient_failure_then_succeed(**kwargs: Any) -> None:+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ raise Exception("transient")+ captured_values.append(list(kwargs["value"]))++ mock_db_upsert = AsyncMock(side_effect=transient_failure_then_succeed)++ with (+ patch(INTERVAL_PATCH, 1),+ patch.object(Block, "execute_safe", new_callable=AsyncMock, return_value=inner_result),+ patch.object(ForLoopBlock, "get_loop_block_context_parameters", return_value=[]),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch("skyvern.forge.sdk.workflow.models.block.skyvern_context") as mock_skyvern_ctx,+ ):+ mock_skyvern_ctx.current.return_value = None+ mock_app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = mock_db_upsert++ await loop_block.execute_loop_helper(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_loop",+ workflow_run_context=mock_context,+ loop_over_values=["a", "b", "c"],+ organization_id="org_test",+ )++ assert call_count == 3+ # Iteration 2 (first successful persist) has 2 accumulated outputs+ assert len(captured_values[0]) == 2+ # Iteration 3 has 3 accumulated outputs+ assert len(captured_values[1]) == 3+++class TestPersistIntervalBatching:+ """PERSIST_LOOP_OUTPUT_INTERVAL controls write frequency to reduce DB load."""++ @pytest.mark.asyncio+ async def test_default_interval_batches_writes(self) -> None:+ """With default interval=10, a 25-iteration loop persists at indices 0, 10, 20 (3 writes)."""+ inner_task = TaskBlock(label="inner_task", output_parameter=_make_output_param("inner_task"))+ loop_block = ForLoopBlock(+ label="test_loop",+ output_parameter=_make_output_param("test_loop"),+ loop_blocks=[inner_task],+ )++ inner_result = _make_block_result(inner_task.output_parameter)+ mock_context = MagicMock()+ mock_context.has_value.return_value = False+ mock_context.set_value = MagicMock()+ mock_context.update_block_metadata = MagicMock()++ captured_sizes: list[int] = []++ async def capture_upsert(**kwargs: Any) -> None:+ captured_sizes.append(len(kwargs["value"]))++ mock_db_upsert = AsyncMock(side_effect=capture_upsert)++ with (+ patch.object(Block, "execute_safe", new_callable=AsyncMock, return_value=inner_result),+ patch.object(ForLoopBlock, "get_loop_block_context_parameters", return_value=[]),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch("skyvern.forge.sdk.workflow.models.block.skyvern_context") as mock_skyvern_ctx,+ ):+ mock_skyvern_ctx.current.return_value = None+ mock_app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = mock_db_upsert++ result = await loop_block.execute_loop_helper(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_loop",+ workflow_run_context=mock_context,+ loop_over_values=list(range(25)),+ organization_id="org_test",+ )++ # 25 iterations, interval=10 → persists at idx 0, 10, 20, and 24 (last)+ assert mock_db_upsert.call_count == 4+ # idx 0: 1 item, idx 10: 11 items, idx 20: 21 items, idx 24: 25 items+ assert captured_sizes == [1, 11, 21, 25]+ # All 25 iterations still produce output+ assert len(result.outputs_with_loop_values) == 25++ @pytest.mark.asyncio+ async def test_no_persist_when_output_parameter_is_none(self) -> None:+ """If output_parameter is None, _persist_partial_loop_output is a no-op."""+ inner_task = TaskBlock(label="inner", output_parameter=_make_output_param("inner"))+ loop_block = ForLoopBlock.model_construct(+ label="test_loop",+ output_parameter=None,+ loop_blocks=[inner_task],+ )++ mock_db_upsert = AsyncMock()++ with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:+ mock_app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = mock_db_upsert++ await loop_block._persist_partial_loop_output(+ workflow_run_id="wr_test",+ outputs_with_loop_values=[[{"data": "val"}]],+ loop_idx=0,+ )++ assert mock_db_upsert.call_count == 0