Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9962): recover conditional block when LLM returns extra placeholder evaluations (#6058)
Trevor Cheung1157fa5b45cb
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex dd681c3fe..a39dd081e 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -7555,6 +7555,21 @@ async def _evaluate_prompt_branch_conditions_batch(raw_evaluations = _find_evaluations_array(output_value)+ # LLM sometimes splits compound criteria into per-clause sub-evaluations, emitting+ # reasoning=None placeholder entries for each. Strip them and recover if the remainder+ # matches len(branches); otherwise fall through to the existing hard-fail.+ if len(raw_evaluations) > len(branches):+ well_formed = [e for e in raw_evaluations if not (isinstance(e, dict) and e.get("reasoning") is None)]+ if len(well_formed) == len(branches):+ LOG.warning(+ "LLM returned extra placeholder evaluations; using well-formed subset",+ block_label=log_label,+ total_returned=len(raw_evaluations),+ well_formed_count=len(well_formed),+ num_branches=len(branches),+ )+ raw_evaluations = well_formed+for idx, evaluation in enumerate(raw_evaluations):bool_result, rendered_expr = _parse_single_evaluation(evaluation=evaluation,diff --git a/tests/unit/workflow/test_conditional_branch_evaluation.py b/tests/unit/workflow/test_conditional_branch_evaluation.pyindex 8830755a2..4d27a863b 100644--- a/tests/unit/workflow/test_conditional_branch_evaluation.py+++ b/tests/unit/workflow/test_conditional_branch_evaluation.py@@ -482,6 +482,136 @@ async def test_extraction_failure_with_reason_preserves_original_message() -> No)+async def test_extra_placeholder_evals_recovered_when_well_formed_subset_matches() -> None:+ """LLM sometimes returns N+k evaluations for N branches, where the k extras have+ reasoning=None. Matches the exact shape observed in production (wr_530455567744647688):+ 1 branch, 8 evaluations returned, entry 0 has real reasoning, entries 1-7 are+ reasoning=None placeholders. The fix should strip the extras and return [False]."""+ block = _conditional_block()+ branch = BranchCondition(+ criteria=PromptBranchCriteria(+ expression="Does one of the accounts have both a reader_name not ending with 'pdf' and a purpose of 'Invoice'"+ ),+ next_block_label="invoice_branch",+ )++ evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)+ evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]++ # One well-formed entry followed by 7 reasoning=None placeholders — exact production shape.+ raw_evals = [+ {"reasoning": "Neither account has purpose 'Invoice', so the condition is False.", "result": False},+ {"reasoning": None, "result": False},+ {"reasoning": None, "result": False},+ {"reasoning": None, "result": False},+ {"reasoning": None, "result": False},+ {"reasoning": None, "result": False},+ {"reasoning": None, "result": False},+ {"reasoning": None, "result": False},+ ]++ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))+ mock_extraction_cls.return_value = mock_extraction++ results, _, _, _ = await block._evaluate_prompt_branches(+ branches=[branch],+ evaluation_context=evaluation_context,+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert results == [False]+++async def test_extra_evals_not_recovered_when_well_formed_count_does_not_match() -> None:+ """If stripping reasoning=None entries does NOT yield exactly len(branches) results,+ the function should still raise ValueError rather than silently returning wrong data."""+ block = _conditional_block()+ branch = BranchCondition(+ criteria=PromptBranchCriteria(expression="some condition"),+ next_block_label="branch_a",+ )++ evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)+ evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]++ # 2 well-formed + 1 placeholder for 1 branch — filter yields 2, not 1, so no recovery.+ raw_evals = [+ {"reasoning": "sub-eval A", "result": True},+ {"reasoning": "sub-eval B", "result": False},+ {"reasoning": None, "result": False},+ ]++ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))+ mock_extraction_cls.return_value = mock_extraction++ with pytest.raises(ValueError, match="3 results for 1 branches"):+ await block._evaluate_prompt_branches(+ branches=[branch],+ evaluation_context=evaluation_context,+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )+++async def test_extra_placeholder_evals_multi_branch_preserves_order() -> None:+ """With 2 branches and interleaved placeholders, the filter must preserve the order+ of well-formed entries so results[0] maps to branch 0 and results[1] maps to branch 1."""+ block = ConditionalBlock(+ label="cond",+ output_parameter=_output_parameter("out"),+ branch_conditions=[+ BranchCondition(criteria=PromptBranchCriteria(expression="condition A"), next_block_label="a"),+ BranchCondition(criteria=PromptBranchCriteria(expression="condition B"), next_block_label="b"),+ ],+ )+ branches = [c for c in block.branch_conditions if not c.is_default]++ evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)+ evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]++ # Real evals for branch 0 (True) and branch 1 (False) interleaved with placeholders.+ raw_evals = [+ {"reasoning": "branch 0 reasoning", "result": True},+ {"reasoning": None, "result": False},+ {"reasoning": "branch 1 reasoning", "result": False},+ {"reasoning": None, "result": False},+ ]++ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))+ mock_extraction_cls.return_value = mock_extraction++ results, _, _, _ = await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=evaluation_context,+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert results == [True, False]++def test_prompt_template_includes_count_and_atomicity_for_compound_conditions() -> None:rendered_one = prompt_engine.load_prompt("conditional-prompt-branch-evaluation",