Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
"""Tests for prompt-based conditional branch evaluation behavior."""
import copy
import json
from collections.abc import Iterator
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import skyvern.forge.sdk.workflow.models.block as block_module
from skyvern.config import settings
from skyvern.exceptions import BranchEvaluationContextTooLargeError, ConditionalBranchEvaluationError
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.exceptions import MissingJinjaVariables
from skyvern.forge.sdk.workflow.models.block import (
BranchCondition,
BranchEvaluationContext,
ConditionalBlock,
ExtractionBlock,
JinjaBranchCriteria,
PromptBranchCriteria,
_build_branch_evaluation_schema,
_coerce_condition_index,
_make_empty_params_explicit,
_neutralize_jinja_delimiters,
)
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter, WorkflowParameterType
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
from skyvern.schemas.run_enums import RunEngine
from skyvern.schemas.workflows import BlockResult, BlockStatus
from skyvern.utils.token_counter import count_tokens
from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext
BRANCH_CONTEXT_TOO_LARGE_FAILURE_REASON = (
"Workflow branch evaluation context is too large to process safely. "
"Reduce the workflow input or prior block output size, then retry."
)
def _output_parameter(key: str) -> OutputParameter:
now = datetime.now(UTC)
return OutputParameter(
output_parameter_id=f"{key}_id",
key=key,
workflow_id="wf",
created_at=now,
modified_at=now,
)
def _conditional_block() -> ConditionalBlock:
return ConditionalBlock(
label="cond",
output_parameter=_output_parameter("conditional_output"),
branch_conditions=[
BranchCondition(criteria=PromptBranchCriteria(expression="fallback"), next_block_label="next"),
BranchCondition(is_default=True, next_block_label=None),
],
)
def _extraction_result(output_parameter: OutputParameter, evaluations: list[dict]) -> BlockResult:
return BlockResult(
success=True,
output_parameter=output_parameter,
output_parameter_value={"evaluations": evaluations},
failure_reason=None,
)
@pytest.mark.asyncio
async def test_jinja_rendered_prompt_condition_keeps_browser_session() -> None:
"""When all expressions are fully Jinja-rendered, ExtractionBlock should still
receive the browser_session_id so that page-referencing conditions (e.g.
"the date on the page matches {{date}}") can see the screenshot. The prompt
template instructs the LLM to only use page content when the condition
explicitly references the page (SKY-8465)."""
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(expression='{{Single_or_Joint__c}} == "Joint"'),
next_block_label="joint",
)
evaluation_context = BranchEvaluationContext(
workflow_run_context=None,
template_renderer=lambda expr: expr.replace("{{Single_or_Joint__c}}", "Joint"),
)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"Single_or_Joint__c": "Joint"}) # type: ignore[method-assign]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal") as mock_prompt,
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_extraction_result(
block.output_parameter,
[{"reasoning": "ok", "result": True}],
)
)
mock_extraction_cls.return_value = mock_extraction
results, rendered_expressions, _, llm_response = await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
browser_session_id="bs_test",
)
assert results == [True]
assert rendered_expressions == ['Joint == "Joint"']
# ExtractionBlock should be called with the real browser_session_id
mock_extraction.execute.assert_awaited_once()
assert mock_extraction.execute.call_args.kwargs["browser_session_id"] == "bs_test"
# No context should be passed when all expressions are Jinja-rendered
evaluation_context.build_llm_safe_context_snapshot.assert_not_called() # type: ignore[attr-defined]
assert mock_prompt.call_args.kwargs["context_json"] is None
@pytest.mark.asyncio
async def test_pure_natlang_prompt_condition_uses_browser_session_and_context() -> None:
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"),
next_block_label="premium",
)
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"plan": "premium"}) # type: ignore[method-assign]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal") as mock_prompt,
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_extraction_result(
block.output_parameter,
[
{
"rendered_condition": "user selected premium plan",
"reasoning": "ok",
"result": True,
}
],
)
)
mock_extraction_cls.return_value = mock_extraction
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
browser_session_id="bs_test",
)
assert mock_extraction.execute.call_args.kwargs["browser_session_id"] == "bs_test"
evaluation_context.build_llm_safe_context_snapshot.assert_called_once() # type: ignore[attr-defined]
assert mock_prompt.call_args.kwargs["context_json"] is not None
@pytest.mark.asyncio
async def test_mixed_prompt_conditions_keep_browser_session() -> None:
block = _conditional_block()
branches = [
BranchCondition(
criteria=PromptBranchCriteria(expression="{{var}} == 'value'"),
next_block_label="jinja_branch",
),
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"),
next_block_label="natlang_branch",
),
]
evaluation_context = BranchEvaluationContext(
workflow_run_context=None,
template_renderer=lambda expr: expr.replace("{{var}}", "value"),
)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"var": "value"}) # type: ignore[method-assign]
with patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls:
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_extraction_result(
block.output_parameter,
[
{"rendered_condition": "value == 'value'", "reasoning": "ok", "result": True},
{
"rendered_condition": "user selected premium plan",
"reasoning": "ok",
"result": False,
},
],
)
)
mock_extraction_cls.return_value = mock_extraction
await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
browser_session_id="bs_test",
)
assert mock_extraction.execute.call_args.kwargs["browser_session_id"] == "bs_test"
evaluation_context.build_llm_safe_context_snapshot.assert_called_once() # type: ignore[attr-defined]
@pytest.mark.asyncio
async def test_oversized_branch_context_trips_before_extraction_and_returns_stable_failure() -> None:
# No default branch: with one defined, evaluation failures now route to it (SKY-14080)
# instead of surfacing this stable failure.
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("conditional_output"),
branch_conditions=[
BranchCondition(criteria=PromptBranchCriteria(expression="fallback"), next_block_label="next"),
],
)
oversized_context_value = "oversized-context-value " * 200_000
with (
patch.object(
BranchEvaluationContext,
"build_llm_safe_context_snapshot",
return_value={"sensitive_customer_key": oversized_context_value},
),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch("skyvern.forge.sdk.workflow.models.block.diagnostic_fingerprint", return_value="key-fingerprint"),
patch.object(block_module.LOG, "warning") as mock_warning,
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=None),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_failed_extraction_result(block.output_parameter, "downstream prompt failed")
)
mock_extraction_cls.return_value = mock_extraction
mock_build_result.return_value = BlockResult(
success=False,
output_parameter=block.output_parameter,
output_parameter_value=None,
failure_reason=BRANCH_CONTEXT_TOO_LARGE_FAILURE_REASON,
)
result = await block.execute(
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
assert result.failure_reason == BRANCH_CONTEXT_TOO_LARGE_FAILURE_REASON
assert mock_build_result.await_args.kwargs["failure_reason"] == BRANCH_CONTEXT_TOO_LARGE_FAILURE_REASON
mock_extraction_cls.assert_not_called()
circuit_logs = [
call
for call in mock_warning.call_args_list
if call.args[0] == "conditional_branch_context_circuit_breaker_tripped"
]
assert len(circuit_logs) == 1
circuit_log = circuit_logs[0]
assert circuit_log.kwargs["goal_token_count"] > 180_000
assert circuit_log.kwargs["max_goal_tokens"] == 150_000
assert circuit_log.kwargs["reserved_tokens"] == 30_000
assert circuit_log.kwargs["context_key_count"] == 1
assert circuit_log.kwargs["top_context_contributors"] == [
{
"key_fingerprint": "key-fingerprint",
"serialized_bytes": len(f'"{oversized_context_value}"'.encode()),
"token_count": circuit_log.kwargs["top_context_contributors"][0]["token_count"],
}
]
assert "sensitive_customer_key" not in str(circuit_log)
assert oversized_context_value not in str(circuit_log)
@pytest.mark.asyncio
async def test_branch_goal_at_reserved_budget_boundary_still_executes_unchanged() -> None:
block = _conditional_block()
branch = block.branch_conditions[0]
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"plan": "premium"}) # type: ignore[method-assign]
def _token_count(value: str) -> int:
return 150_000 if value == "goal-at-boundary" else 2
with (
patch(
"skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",
return_value="goal-at-boundary",
) as mock_prompt,
patch("skyvern.forge.sdk.workflow.models.block.count_tokens", side_effect=_token_count) as mock_count_tokens,
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_extraction_result(
block.output_parameter,
[{"condition_index": 1, "reasoning": "ok", "result": True}],
)
)
mock_extraction_cls.return_value = mock_extraction
results, _, extraction_goal, _ = await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
assert results == [True]
assert extraction_goal == "goal-at-boundary"
assert mock_extraction_cls.call_args.kwargs["data_extraction_goal"] == "goal-at-boundary"
assert mock_prompt.call_args.kwargs["context_json"] == '{"plan": "premium"}'
assert any(call.args == ("goal-at-boundary",) for call in mock_count_tokens.call_args_list)
@pytest.mark.asyncio
async def test_branch_goal_above_reserved_budget_trips_before_extraction() -> None:
block = _conditional_block()
branch = block.branch_conditions[0]
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"plan": "premium"}) # type: ignore[method-assign]
def _token_count(value: str) -> int:
return 150_001 if value == "goal-above-boundary" else 2
with (
patch(
"skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",
return_value="goal-above-boundary",
),
patch("skyvern.forge.sdk.workflow.models.block.count_tokens", side_effect=_token_count),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch("skyvern.forge.sdk.workflow.models.block.diagnostic_fingerprint", return_value="key-fingerprint"),
):
with pytest.raises(BranchEvaluationContextTooLargeError):
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
mock_extraction_cls.assert_not_called()
def _prompt_conditional(label: str) -> ConditionalBlock:
return ConditionalBlock(
label=label,
output_parameter=_output_parameter(f"{label}_output"),
branch_conditions=[
BranchCondition(
criteria=PromptBranchCriteria(expression="the customer is on the premium plan"),
next_block_label=f"{label}_matched",
),
BranchCondition(is_default=True, next_block_label=f"{label}_default"),
],
)
def _run_context(*blocks: ConditionalBlock) -> WorkflowRunContext:
now = datetime.now(UTC)
return WorkflowRunContext(
workflow_title="wf",
workflow_id="wf",
workflow_permanent_id="wpid",
workflow_run_id="wr",
aws_client=MagicMock(),
workflow=Workflow(
workflow_id="wf",
organization_id="org",
title="wf",
workflow_permanent_id="wpid",
version=1,
is_saved_task=False,
workflow_definition=WorkflowDefinition(parameters=[], blocks=list(blocks)),
created_at=now,
modified_at=now,
),
)
_EVALUATION_REASONING = "reasoning-marker: the stored plan is premium"
async def _execute_prompt_conditional(block: ConditionalBlock, run_context: WorkflowRunContext) -> dict:
"""Run the block against a real run context; only the model call and the database write are faked."""
async def _condition_is_true(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202
return _extraction_result(
self.output_parameter,
[{"condition_index": 1, "reasoning": _EVALUATION_REASONING, "result": True}],
)
with (
patch.object(ExtractionBlock, "execute", _condition_is_true),
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=run_context),
),
patch.object(
block_module.app.DATABASE.workflow_runs,
"create_or_update_workflow_run_output_parameter",
new=AsyncMock(),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
await block.execute(workflow_run_id="wr", workflow_run_block_id="wrb", organization_id="org")
return mock_build_result.await_args.kwargs
@pytest.mark.parametrize(
("output_share_of_budget", "expected_branch", "expected_evaluation_error"),
[
pytest.param(0.6, "cond_2_matched", None, id="one-copy-fits"),
pytest.param(1.2, "cond_2_default", BRANCH_CONTEXT_TOO_LARGE_FAILURE_REASON, id="one-copy-too-large"),
],
)
@pytest.mark.asyncio
async def test_prior_block_output_counts_once_against_the_branch_context_budget(
output_share_of_budget: float, expected_branch: str, expected_evaluation_error: str | None
) -> None:
cond_1, cond_2 = _prompt_conditional("cond_1"), _prompt_conditional("cond_2")
run_context = _run_context(cond_1, cond_2)
await _execute_prompt_conditional(cond_1, run_context)
row = "invoice 1042 paid in full; "
tokens_per_row = count_tokens(row * 100) / 100
row_count = int(output_share_of_budget * block_module.BRANCH_EVALUATION_GOAL_MAX_TOKENS / tokens_per_row)
await run_context.register_output_parameter_value_post_execution(
_output_parameter("invoices_output"), {"extracted_information": {"rows": row * row_count}}
)
result = await _execute_prompt_conditional(cond_2, run_context)
assert result["status"] == BlockStatus.completed
assert result["executed_branch_next_block"] == expected_branch
assert result["output_parameter_value"].get("evaluation_error") == expected_evaluation_error
@pytest.mark.asyncio
async def test_branch_context_snapshot_holds_each_output_once_and_no_earlier_evaluation_payload() -> None:
cond_1 = _prompt_conditional("cond_1")
run_context = _run_context(cond_1, _prompt_conditional("cond_2"))
# Present while cond_1 evaluates, so the prompt cond_1 stores embeds it.
run_context.values["plan"] = "plan-marker-premium"
await _execute_prompt_conditional(cond_1, run_context)
# Not a conditional: a field that shares a name with a conditional's debug field is this block's data.
lookup_output = {"status": "paid", "evaluations": ["manual review passed"]}
for label in ("first_lookup", "second_lookup"):
await run_context.register_output_parameter_value_post_execution(
_output_parameter(f"{label}_output"), lookup_output
)
values_before = copy.deepcopy(run_context.values)
snapshot = BranchEvaluationContext(
workflow_run_context=run_context, block_label="cond_2"
).build_llm_safe_context_snapshot()
# Two different blocks with equal outputs both stay; each is listed once, findable by its label.
assert {key: value for key, value in snapshot.items() if "lookup" in key} == {
"first_lookup_output": lookup_output,
"second_lookup_output": lookup_output,
}
assert [key for key in snapshot if key.startswith("cond_1")] == ["cond_1_output"]
# What a later condition can refer to survives: which branch the earlier conditional took.
assert snapshot["cond_1_output"]["branch_taken"] == "cond_1_matched"
assert snapshot["cond_1_output"]["branch_index"] == 0
assert snapshot["cond_1_output"]["criteria_expression"] == "the customer is on the premium plan"
# Its stored prompt, raw response and per-branch detail do not.
assert not {"llm_prompt", "llm_response", "evaluations"} & snapshot["cond_1_output"].keys()
serialized = json.dumps(snapshot, default=str)
assert serialized.count("plan-marker-premium") == 1
assert _EVALUATION_REASONING not in serialized
# Only the snapshot changes: templates still resolve both names against the untouched run context.
assert run_context.values == values_before
assert _EVALUATION_REASONING in json.dumps(run_context.values["cond_1"], default=str)
@pytest.mark.asyncio
async def test_branch_context_snapshot_deduplicates_carried_outputs_and_strips_carried_conditional_debug() -> None:
now = datetime.now(UTC)
conditional_output = {
"branch_taken": "prior_matched",
"llm_prompt": "stored prompt",
"llm_response": "stored response",
"evaluations": [{"reasoning": "stored detail"}],
}
plain_output = {"status": "paid", "evaluations": ["manual review passed"]}
workflow = _run_context(_prompt_conditional("prior")).workflow
assert workflow is not None
run_context = await WorkflowRunContext.init(
aws_client=MagicMock(),
organization=Organization(organization_id="org", organization_name="test", created_at=now, modified_at=now),
workflow_run_id="wr",
workflow_title="wf",
workflow_id="wf",
workflow_permanent_id="wpid",
workflow_parameter_tuples=[],
workflow_output_parameters=[],
context_parameters=[],
secret_parameters=[],
block_outputs={"prior": conditional_output, "lookup": plain_output},
workflow=workflow,
)
snapshot = BranchEvaluationContext(workflow_run_context=run_context).build_llm_safe_context_snapshot()
assert run_context.workflow_run_outputs == {}
assert run_context.carried_block_labels == {"prior", "lookup"}
assert snapshot["prior_output"] == {"branch_taken": "prior_matched"}
assert snapshot["lookup_output"] == plain_output
assert not {"prior", "lookup"} & snapshot.keys()
assert run_context.values["prior"] == conditional_output
@pytest.mark.asyncio
async def test_branch_context_snapshot_strips_only_recorded_conditional_outputs_and_keeps_declared_keys() -> None:
"""A conditional's output is known by its recorded output key and alias, not by a label a parameter may
share; a declared parameter is the author's whatever its name."""
now = datetime.now(UTC)
def _declare(key: str, value: object) -> None:
run_context.parameters[key] = WorkflowParameter(
key=key,
workflow_parameter_id=f"wp_{key}",
workflow_parameter_type=WorkflowParameterType.STRING,
workflow_id="wf",
created_at=now,
modified_at=now,
)
run_context.values[key] = value
conditional = _prompt_conditional("cond")
conditional.output_parameter = _output_parameter("decision_output")
run_context = _run_context(conditional, _prompt_conditional("pending"))
_declare("prompt_branch_eval_flag", "yes")
# A parameter that shares the not-yet-run conditional's label: author data, kept whole.
_declare("pending", {"llm_prompt": "author text", "answer": 1})
await _execute_prompt_conditional(conditional, run_context)
snapshot = BranchEvaluationContext(
workflow_run_context=run_context, block_label="next"
).build_llm_safe_context_snapshot()
assert snapshot["prompt_branch_eval_flag"] == "yes"
assert [key for key in snapshot if key.startswith("prompt_branch_eval_")] == ["prompt_branch_eval_flag"]
assert snapshot["pending"] == {"llm_prompt": "author text", "answer": 1}
assert "decision" not in snapshot
assert snapshot["decision_output"]["branch_taken"] == "cond_matched"
assert not {"llm_prompt", "llm_response", "evaluations"} & snapshot["decision_output"].keys()
@pytest.mark.asyncio
async def test_branch_context_snapshot_keeps_a_block_reference_that_differs_from_its_output() -> None:
"""A block that ran in two loop iterations: `<label>` merges both passes, `<label>_output` is the last."""
run_context = _run_context()
await run_context.register_output_parameter_value_post_execution(
_output_parameter("step_output"), {"first_pass": "a"}
)
await run_context.register_output_parameter_value_post_execution(
_output_parameter("step_output"), {"second_pass": "b"}
)
snapshot = BranchEvaluationContext(
workflow_run_context=run_context, block_label="cond"
).build_llm_safe_context_snapshot()
assert snapshot["step"] == {"first_pass": "a", "second_pass": "b"}
assert snapshot["step_output"] == {"second_pass": "b"}
@pytest.mark.asyncio
async def test_jinja_render_failure_falls_back_to_extraction_block() -> None:
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(expression='{{Single_or_Joint__c}} == "Joint"'),
next_block_label="joint",
)
def _raise_render_error(_: str) -> str:
raise RuntimeError("render failed")
evaluation_context = BranchEvaluationContext(
workflow_run_context=None,
template_renderer=_raise_render_error,
)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"Single_or_Joint__c": "Joint"}) # type: ignore[method-assign]
mock_llm_handler = AsyncMock()
with (
patch.dict(block_module.app.__dict__, {"LLM_API_HANDLER": mock_llm_handler}),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_extraction_result(
block.output_parameter,
[{"rendered_condition": '{{Single_or_Joint__c}} == "Joint"', "reasoning": "ok", "result": False}],
)
)
mock_extraction_cls.return_value = mock_extraction
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
browser_session_id="bs_test",
)
mock_extraction.execute.assert_awaited_once()
assert mock_extraction.execute.call_args.kwargs["browser_session_id"] == "bs_test"
mock_llm_handler.assert_not_called()
evaluation_context.build_llm_safe_context_snapshot.assert_called_once() # type: ignore[attr-defined]
# ---------------------------------------------------------------------------
# Tests for _make_empty_params_explicit (SKY-8073)
# ---------------------------------------------------------------------------
class TestMakeEmptyParamsExplicit:
"""Unit tests for _make_empty_params_explicit helper."""
def test_empty_param_is_patched(self) -> None:
"""When a single parameter resolves to empty string, it should be replaced with (empty value)."""
patched, was_patched = _make_empty_params_explicit(
original_expression="if {{test_parameter}} is not empty",
rendered_expression="if is not empty",
)
assert was_patched is True
assert patched == "if (empty value) is not empty"
def test_non_empty_param_is_not_patched(self) -> None:
"""Non-empty parameter values should pass through unchanged."""
patched, was_patched = _make_empty_params_explicit(
original_expression="if {{test_parameter}} is not empty",
rendered_expression="if hello is not empty",
)
assert was_patched is False
assert patched == "if hello is not empty"
def test_no_jinja_blocks(self) -> None:
"""Expressions without Jinja blocks should pass through unchanged."""
patched, was_patched = _make_empty_params_explicit(
original_expression="user selected premium plan",
rendered_expression="user selected premium plan",
)
assert was_patched is False
assert patched == "user selected premium plan"
def test_multiple_params_one_empty(self) -> None:
"""When one of multiple parameters is empty, only that one should be patched."""
patched, was_patched = _make_empty_params_explicit(
original_expression="{{a}} equals {{b}}",
rendered_expression=" equals hello",
)
assert was_patched is True
assert patched == "(empty value) equals hello"
def test_multiple_params_both_empty(self) -> None:
"""When all parameters are empty, all should be patched."""
patched, was_patched = _make_empty_params_explicit(
original_expression="{{a}} equals {{b}}",
rendered_expression=" equals ",
)
assert was_patched is True
assert patched == "(empty value) equals (empty value)"
def test_whitespace_only_param_is_patched(self) -> None:
"""A parameter that resolves to whitespace-only should be treated as empty."""
patched, was_patched = _make_empty_params_explicit(
original_expression="if {{test_parameter}} is not empty",
rendered_expression="if is not empty",
)
assert was_patched is True
assert patched == "if (empty value) is not empty"
def test_empty_original_expression(self) -> None:
"""Empty original expression should pass through."""
patched, was_patched = _make_empty_params_explicit(
original_expression="",
rendered_expression="",
)
assert was_patched is False
def test_adjacent_variables_are_skipped(self) -> None:
"""Adjacent Jinja variables (no separator) cannot be reliably split, so skip patching."""
patched, was_patched = _make_empty_params_explicit(
original_expression="{{a}}{{b}}",
rendered_expression="helloworld",
)
assert was_patched is False
assert patched == "helloworld"
def test_param_at_end_of_expression(self) -> None:
"""Parameter at the end of an expression should be handled correctly."""
patched, was_patched = _make_empty_params_explicit(
original_expression="check if empty: {{param}}",
rendered_expression="check if empty: ",
)
assert was_patched is True
assert patched == "check if empty: (empty value)"
def test_single_bare_variable_empty(self) -> None:
"""Entire expression is one variable that resolved to empty string."""
patched, was_patched = _make_empty_params_explicit(
original_expression="{{a}}",
rendered_expression="",
)
assert was_patched is True
assert patched == "(empty value)"
def test_rendered_value_containing_static_anchor(self) -> None:
"""When a rendered value contains static anchor text, regex may mis-split.
Verify we don't falsely detect an empty parameter."""
patched, was_patched = _make_empty_params_explicit(
original_expression="{{a}} equals {{b}}",
rendered_expression="x equals y equals z",
)
assert was_patched is False
assert patched == "x equals y equals z"
def test_multiline_rendered_value_passes_through(self) -> None:
"""Multiline rendered values (re.DOTALL path) should not be falsely patched."""
patched, was_patched = _make_empty_params_explicit(
original_expression="if {{data}} is valid",
rendered_expression="if line1\nline2 is valid",
)
assert was_patched is False
assert patched == "if line1\nline2 is valid"
@pytest.mark.asyncio
async def test_empty_param_produces_explicit_marker_in_prompt_evaluation() -> None:
"""Integration test: when a parameter resolves to empty string, the rendered
expression sent to the LLM should contain '(empty value)' so the LLM can
correctly evaluate the condition (SKY-8073)."""
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(expression="if {{test_parameter}} is not empty"),
next_block_label="not_empty_branch",
)
evaluation_context = BranchEvaluationContext(
workflow_run_context=None,
template_renderer=lambda expr: expr.replace("{{test_parameter}}", ""),
)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"test_parameter": ""}) # type: ignore[method-assign]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal") as mock_prompt,
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_extraction_result(
block.output_parameter,
[{"reasoning": "empty value is not empty -> false", "result": False}],
)
)
mock_extraction_cls.return_value = mock_extraction
results, rendered_expressions, _, _ = await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
assert results == [False]
# The rendered expression should contain the explicit marker, not a bare gap
assert rendered_expressions == ["if (empty value) is not empty"]
# The prompt should be loaded with the patched expression
assert mock_prompt.call_args.kwargs["conditions"] == ["if (empty value) is not empty"]
# ---------------------------------------------------------------------------
# Tests for None failure_reason guard in _evaluate_prompt_branches (SKY-8026)
# ---------------------------------------------------------------------------
def _failed_extraction_result(output_parameter: OutputParameter, failure_reason: str | None = None) -> BlockResult:
return BlockResult(
success=False,
output_parameter=output_parameter,
output_parameter_value=None,
failure_reason=failure_reason,
)
@pytest.mark.asyncio
async def test_extraction_failure_with_none_reason_produces_informative_error() -> None:
"""When ExtractionBlock fails with failure_reason=None, the raised ValueError
should NOT contain the literal string 'None' (SKY-8026)."""
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"),
next_block_label="premium",
)
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_failed_extraction_result(block.output_parameter, failure_reason=None)
)
mock_extraction_cls.return_value = mock_extraction
with pytest.raises(ConditionalBranchEvaluationError, match="Unknown error"):
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
@pytest.mark.asyncio
async def test_extraction_failure_with_reason_preserves_original_message() -> None:
"""When ExtractionBlock fails with a real failure_reason, that reason should
appear verbatim in the raised ValueError."""
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"),
next_block_label="premium",
)
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_failed_extraction_result(block.output_parameter, failure_reason="LLM rate limited")
)
mock_extraction_cls.return_value = mock_extraction
with pytest.raises(ConditionalBranchEvaluationError, match="LLM rate limited"):
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
@pytest.mark.asyncio
async def test_extra_placeholder_evals_recovered_when_well_formed_subset_matches() -> None:
"""LLM sometimes returns N+k evaluations for N branches, where the k extras have
reasoning=None. Matches the exact shape observed in production (wr_530455567744647688):
1 branch, 8 evaluations returned, entry 0 has real reasoning, entries 1-7 are
reasoning=None placeholders. The fix should strip the extras and return [False]."""
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(
expression="Does one of the accounts have both a reader_name not ending with 'pdf' and a purpose of 'Invoice'"
),
next_block_label="invoice_branch",
)
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
# One well-formed entry followed by 7 reasoning=None placeholders — exact production shape.
raw_evals = [
{"reasoning": "Neither account has purpose 'Invoice', so the condition is False.", "result": False},
{"reasoning": None, "result": False},
{"reasoning": None, "result": False},
{"reasoning": None, "result": False},
{"reasoning": None, "result": False},
{"reasoning": None, "result": False},
{"reasoning": None, "result": False},
{"reasoning": None, "result": False},
]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
results, _, _, _ = await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
assert results == [False]
@pytest.mark.asyncio
async def test_extra_evals_not_recovered_when_well_formed_count_does_not_match() -> None:
"""If stripping reasoning=None entries does NOT yield exactly len(branches) results,
the function should still raise ValueError rather than silently returning wrong data."""
block = _conditional_block()
branch = BranchCondition(
criteria=PromptBranchCriteria(expression="some condition"),
next_block_label="branch_a",
)
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
# 2 well-formed + 1 placeholder for 1 branch — filter yields 2, not 1, so no recovery.
raw_evals = [
{"reasoning": "sub-eval A", "result": True},
{"reasoning": "sub-eval B", "result": False},
{"reasoning": None, "result": False},
]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
with pytest.raises(ConditionalBranchEvaluationError, match="3 results for 1 branches"):
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
@pytest.mark.asyncio
async def test_extra_placeholder_evals_multi_branch_preserves_order() -> None:
"""With 2 branches and interleaved placeholders, the filter must preserve the order
of well-formed entries so results[0] maps to branch 0 and results[1] maps to branch 1."""
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(criteria=PromptBranchCriteria(expression="condition A"), next_block_label="a"),
BranchCondition(criteria=PromptBranchCriteria(expression="condition B"), next_block_label="b"),
],
)
branches = [c for c in block.branch_conditions if not c.is_default]
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
# Real evals for branch 0 (True) and branch 1 (False) interleaved with placeholders.
raw_evals = [
{"reasoning": "branch 0 reasoning", "result": True},
{"reasoning": None, "result": False},
{"reasoning": "branch 1 reasoning", "result": False},
{"reasoning": None, "result": False},
]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
results, _, _, _ = await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=evaluation_context,
workflow_run_id="wr_test",
workflow_run_block_id="wrb_test",
organization_id="org_test",
)
assert results == [True, False]
def test_prompt_template_includes_count_and_atomicity_for_compound_conditions() -> None:
rendered_one = prompt_engine.load_prompt(
"conditional-prompt-branch-evaluation",
conditions=["If A and if B"],
context_json=None,
)
rendered_two = prompt_engine.load_prompt(
"conditional-prompt-branch-evaluation",
conditions=["A", "B"],
context_json=None,
)
assert "exactly 1" in rendered_one
assert "exactly 2" in rendered_two
# Wording-coupled: if rephrased, confirm the replacement still conveys atomicity.
assert "split" in rendered_one.lower()
# ---------------------------------------------------------------------------
# Tests for condition_index alignment + malformed-batch retry (SKY-10682)
# ---------------------------------------------------------------------------
def _two_branch_block() -> ConditionalBlock:
return ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(criteria=PromptBranchCriteria(expression="condition A"), next_block_label="a"),
BranchCondition(criteria=PromptBranchCriteria(expression="condition B"), next_block_label="b"),
],
)
def _no_context() -> BranchEvaluationContext:
ctx = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
ctx.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
return ctx
@pytest.mark.asyncio
async def test_condition_index_alignment_is_order_independent() -> None:
"""Evaluations carrying condition_index must align by index, not position, so a
reversed-order LLM response still maps each result to the correct branch."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
# Returned out of order: condition 2 first (True), then condition 1 (False).
raw_evals = [
{"condition_index": 2, "reasoning": "B", "result": True},
{"condition_index": 1, "reasoning": "A", "result": False},
]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
results, _, _, _ = await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
assert results == [False, True] # branch 0 (index 1) -> False, branch 1 (index 2) -> True
@pytest.mark.asyncio
async def test_hallucinated_unindexed_entry_does_not_misroute() -> None:
"""A hallucinated extra entry WITHOUT a condition_index (the shape that shifted
positional alignment in SKY-10682) must be ignored; indexed entries align to the
correct branches instead of misrouting."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
raw_evals = [
{"reasoning": "hallucinated off-topic text", "result": True}, # junk, no condition_index
{"condition_index": 1, "reasoning": "A", "result": False},
{"condition_index": 2, "reasoning": "B", "result": True},
]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
results, _, _, _ = await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
assert results == [False, True]
@pytest.mark.asyncio
async def test_under_return_retries_then_succeeds() -> None:
"""The SKY-10682 failure shape: the LLM returns fewer results than branches on the
first attempt, then a clean response on retry. The batch must retry and succeed
rather than failing the whole run."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
first = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])
second = _extraction_result(
block.output_parameter,
[
{"condition_index": 1, "reasoning": "A", "result": False},
{"condition_index": 2, "reasoning": "B", "result": True},
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(side_effect=[first, second])
mock_extraction_cls.return_value = mock_extraction
results, _, _, _ = await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
assert results == [False, True]
assert mock_extraction.execute.await_count == 2
@pytest.mark.asyncio
async def test_under_return_fails_loudly_after_retries_exhausted() -> None:
"""If every attempt returns a malformed batch, the evaluation must fail loudly
(raise) rather than silently routing to a default/wrong branch."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
bad = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=bad)
mock_extraction_cls.return_value = mock_extraction
with pytest.raises(ConditionalBranchEvaluationError):
await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
assert mock_extraction.execute.await_count >= 2
@pytest.mark.asyncio
async def test_retry_varies_extraction_goal_for_true_reroll() -> None:
"""On retry the extraction goal must differ from the first attempt so the extraction
cache key (which includes data_extraction_goal) changes and we get a genuine re-roll
instead of replaying a cached malformed result (SKY-10682)."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
bad = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])
good = _extraction_result(
block.output_parameter,
[
{"condition_index": 1, "reasoning": "A", "result": False},
{"condition_index": 2, "reasoning": "B", "result": True},
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(side_effect=[bad, good])
mock_extraction_cls.return_value = mock_extraction
await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
goals = [call.kwargs["data_extraction_goal"] for call in mock_extraction_cls.call_args_list]
assert len(goals) == 2
assert goals[0] != goals[1]
@pytest.mark.asyncio
async def test_branch_eval_schema_is_strict_and_indexed() -> None:
"""The data_schema must require condition_index and forbid extra keys
(additionalProperties: false) so the LLM cannot inject hallucinated fields like the
off-topic `including` key seen in production (SKY-10682)."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
good = _extraction_result(
block.output_parameter,
[
{"condition_index": 1, "reasoning": "A", "result": False},
{"condition_index": 2, "reasoning": "B", "result": True},
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=good)
mock_extraction_cls.return_value = mock_extraction
await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
schema = mock_extraction_cls.call_args.kwargs["data_schema"]
item_schema = schema["properties"]["evaluations"]["items"]
assert item_schema["additionalProperties"] is False
assert "condition_index" in item_schema["properties"]
assert "condition_index" in item_schema["required"]
def test_prompt_template_requests_condition_index() -> None:
rendered = prompt_engine.load_prompt(
"conditional-prompt-branch-evaluation",
conditions=["A", "B"],
context_json=None,
)
assert "condition_index" in rendered
def test_build_branch_evaluation_schema_is_strict_and_indexed() -> None:
schema = _build_branch_evaluation_schema(3)
evaluations = schema["properties"]["evaluations"]
assert evaluations["minItems"] == 3
assert evaluations["maxItems"] == 3
item = evaluations["items"]
assert item["additionalProperties"] is False
assert item["required"] == ["condition_index", "reasoning", "result"]
assert schema["additionalProperties"] is False
@pytest.mark.asyncio
async def test_string_condition_index_out_of_order_aligns_by_index() -> None:
"""The schema requests an integer condition_index but isn't provider-enforced, so the model
may type it as a string. Out of order, positional alignment would misroute; digit strings must
coerce to int and stay on the index-aligned path (SKY-10682)."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
raw_evals = [
{"condition_index": "2", "reasoning": "B", "result": True},
{"condition_index": "1", "reasoning": "A", "result": False},
]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
results, _, _, _ = await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
assert results == [False, True]
@pytest.mark.asyncio
async def test_float_condition_index_out_of_order_aligns_by_index() -> None:
"""An integral-float condition_index (e.g. 2.0) must align by index too, not fall back to
positional ordering, which would misroute on a reversed batch (SKY-10682)."""
block = _two_branch_block()
branches = [c for c in block.branch_conditions if not c.is_default]
raw_evals = [
{"condition_index": 2.0, "reasoning": "B", "result": True},
{"condition_index": 1.0, "reasoning": "A", "result": False},
]
with (
patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
results, _, _, _ = await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=_no_context(),
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
assert results == [False, True]
def test_coerce_condition_index_handles_loose_types() -> None:
assert _coerce_condition_index(2) == 2
assert _coerce_condition_index(2.0) == 2
assert _coerce_condition_index("2") == 2
assert _coerce_condition_index(" 3 ") == 3
# bool is an int subclass but is never a valid index
assert _coerce_condition_index(True) is None
assert _coerce_condition_index(False) is None
# non-integral / unparseable values are rejected
assert _coerce_condition_index(2.5) is None
assert _coerce_condition_index("two") is None
assert _coerce_condition_index("") is None
assert _coerce_condition_index(None) is None
# ---------------------------------------------------------------------------
# Tests for stored-Jinja injection into branch-eval prompts + default-branch
# fallback on evaluation failure (SKY-14080)
# ---------------------------------------------------------------------------
_ALL_JINJA_DELIMITERS = ("{{", "}}", "{%", "%}", "{#", "#}")
# JSON structure itself produces adjacent `}}` when nested objects close together; without an
# opening delimiter that text is inert in Jinja, so only openers must never appear in full text.
_JINJA_OPENING_DELIMITERS = ("{{", "{%", "{#")
def _iter_strings(value): # noqa: ANN001, ANN202
if isinstance(value, str):
yield value
elif isinstance(value, dict):
for key, item in value.items():
yield from _iter_strings(key)
yield from _iter_strings(item)
elif isinstance(value, (list, tuple)):
for item in value:
yield from _iter_strings(item)
@pytest.mark.asyncio
async def test_stored_jinja_in_prior_output_does_not_break_branch_evaluation(monkeypatch) -> None:
"""Regression: a prior block's stored output containing a literal ``{{current_value.x}}``
used to be embedded verbatim in the branch-eval goal and re-rendered by the synthetic
ExtractionBlock, raising UndefinedError/MissingJinjaVariables under strict templating."""
monkeypatch.setattr(settings, "WORKFLOW_TEMPLATING_STRICTNESS", "strict")
fake_ctx = FakeWorkflowRunContext(
values={
"prior_conditional_output": {
"criteria_expression": "{{ current_value.account_number }} == '123'",
"original_expression": "{{ current_value.account_number }} == '123'",
"branch_taken": "path_a",
},
},
)
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(criteria=PromptBranchCriteria(expression="the account is active"), next_block_label="a"),
BranchCondition(criteria=PromptBranchCriteria(expression="the account is closed"), next_block_label="b"),
BranchCondition(is_default=True, next_block_label="fallback"),
],
)
branches = [c for c in block.branch_conditions if not c.is_default]
evaluation_context = BranchEvaluationContext(
workflow_run_context=fake_ctx,
block_label="cond",
template_renderer=lambda expr: expr,
)
captured: dict = {}
async def _execute_after_real_format(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202
# Run the real second-render step the production ExtractionBlock.execute performs; the
# bug fired here as an UndefinedError on the embedded `{{current_value...}}` text.
self.format_potential_template_parameters(workflow_run_context=fake_ctx)
captured["goal"] = self.data_extraction_goal
return _extraction_result(
self.output_parameter,
[
{"condition_index": 1, "reasoning": "ok", "result": False},
{"condition_index": 2, "reasoning": "ok", "result": True},
],
)
with patch.object(ExtractionBlock, "execute", _execute_after_real_format):
results, _, _, _ = await block._evaluate_prompt_branches(
branches=branches,
evaluation_context=evaluation_context,
workflow_run_id="wr",
workflow_run_block_id="wrb",
organization_id="org",
)
assert results == [False, True]
goal = captured["goal"]
# The stored expression text survives for the LLM but neutralized, with no live Jinja opener.
assert "{ { current_value.account_number } }" in goal
for delimiter in _JINJA_OPENING_DELIMITERS:
assert delimiter not in goal
def test_prerendered_extraction_goal_skips_second_jinja_render(monkeypatch) -> None:
monkeypatch.setattr(settings, "WORKFLOW_TEMPLATING_STRICTNESS", "strict")
goal = "Evaluate: {{ current_value.account_number }} == '123'"
block = ExtractionBlock(label="eval", data_extraction_goal=goal, output_parameter=_output_parameter("out"))
block.mark_data_extraction_goal_prerendered()
block.format_potential_template_parameters(workflow_run_context=FakeWorkflowRunContext(values={}))
assert block.data_extraction_goal == goal
def test_unmarked_extraction_goal_still_renders(monkeypatch) -> None:
monkeypatch.setattr(settings, "WORKFLOW_TEMPLATING_STRICTNESS", "strict")
block = ExtractionBlock(
label="eval",
data_extraction_goal="Evaluate: {{ foo }}",
output_parameter=_output_parameter("out"),
)
block.format_potential_template_parameters(workflow_run_context=FakeWorkflowRunContext(values={"foo": "bar"}))
assert block.data_extraction_goal == "Evaluate: bar"
unresolvable = ExtractionBlock(
label="eval",
data_extraction_goal="Evaluate: {{ current_value.account_number }}",
output_parameter=_output_parameter("out"),
)
with pytest.raises(MissingJinjaVariables):
unresolvable.format_potential_template_parameters(workflow_run_context=FakeWorkflowRunContext(values={}))
def test_llm_safe_context_snapshot_neutralizes_jinja_delimiters() -> None:
fake_ctx = FakeWorkflowRunContext(
values={
"prior_output": {
"criteria_expression": "{{ current_value.x }} == 1",
"nested": ["{% if x %}", {"deep": "{# comment #}"}],
},
"extracted_output": {"extracted_information": {"note": "{{ injected }}"}},
"plain": "no jinja here",
"count": 7,
},
)
snapshot = BranchEvaluationContext(
workflow_run_context=fake_ctx, block_label="cond"
).build_llm_safe_context_snapshot()
for embedded_string in _iter_strings(snapshot):
for delimiter in _ALL_JINJA_DELIMITERS:
assert delimiter not in embedded_string
serialized = json.dumps(snapshot, default=str)
for delimiter in _JINJA_OPENING_DELIMITERS:
assert delimiter not in serialized
# Values stay readable for the LLM and non-strings pass through untouched.
assert "current_value.x" in serialized
assert snapshot["plain"] == "no jinja here"
assert snapshot["count"] == 7
def test_neutralize_jinja_delimiters_cannot_reform_delimiters_from_brace_runs() -> None:
neutralized = _neutralize_jinja_delimiters({"k": ["{{{x}}}", "{{{{y}}}}", "a{%b%}c", "{#c#}"]})
for embedded_string in _iter_strings(neutralized):
for delimiter in _ALL_JINJA_DELIMITERS:
assert delimiter not in embedded_string
def test_neutralize_jinja_delimiters_keeps_keys_that_would_collide() -> None:
"""Neutralization is not injective, so it is applied to values only: rewriting these two
distinct keys would map both to ``{ {a} }`` and silently drop one of the values."""
neutralized = _neutralize_jinja_delimiters({"{{a}}": "{{ x }}", "{ {a} }": "{{ y }}"})
assert len(neutralized) == 2
assert neutralized["{{a}}"] == "{ { x } }"
assert neutralized["{ {a} }"] == "{ { y } }"
@pytest.mark.asyncio
async def test_branch_evaluation_failure_with_default_branch_falls_back() -> None:
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="premium"
),
BranchCondition(is_default=True, next_block_label="fallback_block"),
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=None),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_failed_extraction_result(block.output_parameter, "LLM exploded")
)
mock_extraction_cls.return_value = mock_extraction
await block.execute(workflow_run_id="wr", workflow_run_block_id="wrb", organization_id="org")
kwargs = mock_build_result.await_args.kwargs
assert kwargs["success"] is True
assert kwargs["failure_reason"] is None
assert kwargs["status"] == BlockStatus.completed
assert kwargs["executed_branch_next_block"] == "fallback_block"
metadata = kwargs["output_parameter_value"]
assert metadata["branch_taken"] == "fallback_block"
assert metadata["next_block_label"] == "fallback_block"
assert "LLM exploded" in metadata["evaluation_error"]
default_eval = next(entry for entry in metadata["evaluations"] if entry["is_default"])
assert default_eval["is_matched"] is True
failed_eval = next(entry for entry in metadata["evaluations"] if not entry["is_default"])
assert "LLM exploded" in failed_eval["error"]
@pytest.mark.asyncio
async def test_branch_evaluation_failure_without_default_branch_still_fails() -> None:
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="premium"
),
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=None),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_failed_extraction_result(block.output_parameter, "LLM exploded")
)
mock_extraction_cls.return_value = mock_extraction
await block.execute(workflow_run_id="wr", workflow_run_block_id="wrb", organization_id="org")
kwargs = mock_build_result.await_args.kwargs
assert kwargs["success"] is False
assert kwargs["status"] == BlockStatus.failed
assert "LLM exploded" in kwargs["failure_reason"]
metadata = kwargs["output_parameter_value"]
assert metadata["branch_taken"] is None
assert metadata["next_block_label"] is None
assert "evaluation_error" not in metadata
@pytest.mark.asyncio
async def test_no_branch_matched_without_failure_still_takes_default_branch() -> None:
"""Guard the pre-existing no-match path through the restructured fallback: a clean False
evaluation (no error) must take the default branch without recording an evaluation_error."""
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="premium"
),
BranchCondition(is_default=True, next_block_label="fallback_block"),
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=None),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_extraction_result(
block.output_parameter,
[{"condition_index": 1, "reasoning": "not premium", "result": False}],
)
)
mock_extraction_cls.return_value = mock_extraction
await block.execute(workflow_run_id="wr", workflow_run_block_id="wrb", organization_id="org")
kwargs = mock_build_result.await_args.kwargs
assert kwargs["success"] is True
assert kwargs["failure_reason"] is None
metadata = kwargs["output_parameter_value"]
assert metadata["branch_taken"] == "fallback_block"
assert "evaluation_error" not in metadata
default_eval = next(entry for entry in metadata["evaluations"] if entry["is_default"])
assert default_eval["is_matched"] is True
@pytest.mark.asyncio
async def test_failed_branch_does_not_hide_a_later_matching_branch() -> None:
"""A branch that cannot be evaluated must not short-circuit the loop into the default: the
remaining branches are still evaluated, and a later match wins over the default."""
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="premium"
),
BranchCondition(criteria=JinjaBranchCriteria(expression="{{ 1 == 1 }}"), next_block_label="jinja_match"),
BranchCondition(is_default=True, next_block_label="fallback_block"),
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=None),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_failed_extraction_result(block.output_parameter, "LLM exploded")
)
mock_extraction_cls.return_value = mock_extraction
await block.execute(workflow_run_id="wr", workflow_run_block_id="wrb", organization_id="org")
kwargs = mock_build_result.await_args.kwargs
assert kwargs["success"] is True
assert kwargs["failure_reason"] is None
assert kwargs["status"] == BlockStatus.completed
assert kwargs["executed_branch_next_block"] == "jinja_match"
metadata = kwargs["output_parameter_value"]
assert metadata["branch_taken"] == "jinja_match"
assert metadata["branch_index"] == 1
# The unevaluable branch stays visible even though routing succeeded.
assert "LLM exploded" in metadata["evaluation_error"]
assert "LLM exploded" in metadata["evaluations"][0]["error"]
assert [entry["next_block_label"] for entry in metadata["evaluations"] if entry["is_matched"]] == ["jinja_match"]
@pytest.mark.asyncio
async def test_failed_branch_does_not_reorder_later_matching_branches() -> None:
"""Among branches that do evaluate, author order still decides: an earlier failure must not
let a lower-priority branch jump ahead of the first branch that actually matched."""
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="premium"
),
BranchCondition(criteria=JinjaBranchCriteria(expression="{{ 1 == 1 }}"), next_block_label="first_true"),
BranchCondition(criteria=JinjaBranchCriteria(expression="{{ 2 == 2 }}"), next_block_label="second_true"),
BranchCondition(is_default=True, next_block_label="fallback_block"),
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=None),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
mock_extraction = MagicMock()
mock_extraction.execute = AsyncMock(
return_value=_failed_extraction_result(block.output_parameter, "LLM exploded")
)
mock_extraction_cls.return_value = mock_extraction
await block.execute(workflow_run_id="wr", workflow_run_block_id="wrb", organization_id="org")
kwargs = mock_build_result.await_args.kwargs
metadata = kwargs["output_parameter_value"]
assert metadata["branch_taken"] == "first_true"
assert metadata["branch_index"] == 1
# The third branch is never reached, so it must not appear as evaluated or matched.
assert [entry["next_block_label"] for entry in metadata["evaluations"]] == ["premium", "first_true"]
# Engine A/B parity contracts (SKY-15494): the synthetic branch-eval ExtractionBlock follows the
# run's resolved engine, so these pin that rendering/alignment never branch on engine, a failed
# extraction raises rather than retrying on another engine, and jinja-only conditionals build no
# ExtractionBlock at all. Whether the block honors the run override lives in
# tests/unit/test_workflow_block_engine.py, which owns engine resolution/dispatch coverage.
@pytest.fixture
def scoped_context() -> Iterator[SkyvernContext]:
context = SkyvernContext()
skyvern_context.set(context)
try:
yield context
finally:
skyvern_context.reset()
def _pin_engine_override(context: SkyvernContext, workflow_run_id: str, engine: RunEngine | None) -> None:
context.workflow_block_engine_resolved_run_id = workflow_run_id
context.workflow_block_engine_override = engine
@pytest.mark.asyncio
@pytest.mark.parametrize("engine_override", [None, RunEngine.skyvern_v3], ids=["control", "v3_override"])
async def test_multi_branch_batch_parity_regardless_of_resolved_engine(
scoped_context: SkyvernContext, engine_override: RunEngine | None
) -> None:
"""Contract A: for a multi-branch, all-prompt batch, the (results, rendered_expressions) the
batch function returns must not depend on which engine the synthetic ExtractionBlock
resolves to."""
_pin_engine_override(scoped_context, "wr_multi", engine_override)
branches = [
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="premium"
),
BranchCondition(
criteria=PromptBranchCriteria(expression="user is a returning customer"), next_block_label="returning"
),
BranchCondition(
criteria=PromptBranchCriteria(expression="cart total exceeds $100"), next_block_label="big_cart"
),
]
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"plan": "premium"}) # type: ignore[method-assign]
async def _execute(self: ExtractionBlock, *args: object, **kwargs: object) -> BlockResult:
return _extraction_result(
self.output_parameter,
[
{"condition_index": 1, "reasoning": "ok", "result": True},
{"condition_index": 2, "reasoning": "ok", "result": False},
{"condition_index": 3, "reasoning": "ok", "result": True},
],
)
with patch.object(ExtractionBlock, "execute", _execute):
results, rendered_expressions, _, _ = await block_module._evaluate_prompt_branch_conditions_batch(
log_label="cond",
branches=branches,
evaluation_context=evaluation_context,
workflow_run_id="wr_multi",
workflow_run_block_id="wrb",
organization_id="org_1",
browser_session_id=None,
workflow_id="wf_1",
)
assert results == [True, False, True]
assert rendered_expressions == [
"user selected premium plan",
"user is a returning customer",
"cart total exceeds $100",
]
@pytest.mark.asyncio
@pytest.mark.parametrize("engine_override", [None, RunEngine.skyvern_v3], ids=["control", "v3_override"])
async def test_jinja_prerendered_mix_batch_parity_regardless_of_resolved_engine(
scoped_context: SkyvernContext, engine_override: RunEngine | None
) -> None:
"""Contract A: a batch mixing a fully Jinja-rendered branch with a pure natural-language
branch returns the same rendered expressions and results regardless of the resolved engine."""
_pin_engine_override(scoped_context, "wr_mixed_parity", engine_override)
branches = [
BranchCondition(criteria=PromptBranchCriteria(expression='{{tier}} == "gold"'), next_block_label="gold"),
BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="premium"
),
]
evaluation_context = BranchEvaluationContext(
workflow_run_context=None,
template_renderer=lambda expr: expr.replace("{{tier}}", "gold"),
)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={"plan": "premium"}) # type: ignore[method-assign]
async def _execute(self: ExtractionBlock, *args: object, **kwargs: object) -> BlockResult:
return _extraction_result(
self.output_parameter,
[
{"condition_index": 1, "reasoning": "ok", "result": True},
{"condition_index": 2, "reasoning": "ok", "result": False},
],
)
with patch.object(ExtractionBlock, "execute", _execute):
results, rendered_expressions, _, _ = await block_module._evaluate_prompt_branch_conditions_batch(
log_label="cond",
branches=branches,
evaluation_context=evaluation_context,
workflow_run_id="wr_mixed_parity",
workflow_run_block_id="wrb",
organization_id="org_1",
browser_session_id=None,
workflow_id="wf_1",
)
assert results == [True, False]
assert rendered_expressions == ['gold == "gold"', "user selected premium plan"]
@pytest.mark.asyncio
async def test_failed_extraction_raises_without_a_different_engine_retry(scoped_context: SkyvernContext) -> None:
"""Contract C: a failing extraction raises ConditionalBranchEvaluationError immediately, with
no second execute attempt on a different engine. Extraction-level failures already retry at
the step level inside the task (block.py:14703-14714), so the batch function must not
silently re-roll on a different engine as a fallback."""
_pin_engine_override(scoped_context, "wr_no_fallback", RunEngine.skyvern_v3)
branch = BranchCondition(
criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="x"
)
evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
captured_engines: list[RunEngine] = []
async def _execute(self: ExtractionBlock, *args: object, **kwargs: object) -> BlockResult:
captured_engines.append(self.resolve_engine("wr_no_fallback"))
return _failed_extraction_result(self.output_parameter, "LLM rate limited")
with (
patch.object(ExtractionBlock, "execute", _execute),
pytest.raises(ConditionalBranchEvaluationError, match="LLM rate limited"),
):
await block_module._evaluate_prompt_branch_conditions_batch(
log_label="cond",
branches=[branch],
evaluation_context=evaluation_context,
workflow_run_id="wr_no_fallback",
workflow_run_block_id="wrb",
organization_id="org_1",
browser_session_id=None,
workflow_id="wf_1",
)
assert len(captured_engines) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("engine_override", [None, RunEngine.skyvern_v3], ids=["no_override", "v3_override"])
async def test_jinja_only_conditional_never_constructs_extraction_block(
scoped_context: SkyvernContext, engine_override: RunEngine | None
) -> None:
"""Contract D: a conditional whose branches are all JinjaBranchCriteria never builds or
executes a synthetic ExtractionBlock, whether or not the run is pinned into the v3 A/B."""
_pin_engine_override(scoped_context, "wr_jinja_only", engine_override)
block = ConditionalBlock(
label="cond",
output_parameter=_output_parameter("out"),
branch_conditions=[
BranchCondition(criteria=JinjaBranchCriteria(expression="{{ 1 == 2 }}"), next_block_label="a"),
BranchCondition(is_default=True, next_block_label="fallback_block"),
],
)
with (
patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
patch.object(
block_module.app.WORKFLOW_CONTEXT_MANAGER,
"get_workflow_run_context",
new=MagicMock(return_value=None),
),
patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
):
await block.execute(workflow_run_id="wr_jinja_only", workflow_run_block_id="wrb", organization_id="org")
mock_extraction_cls.assert_not_called()
assert mock_build_result.await_args.kwargs["executed_branch_next_block"] == "fallback_block"