Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10682): harden conditional branch evaluation against malformed LLM batches (#6400)
Shuchang Zhengf9289402f2fc
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex c89854c44..f16e75555 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -161,6 +161,19 @@ class InvalidOpenAIResponseFormat(SkyvernException):super().__init__(f"Invalid response format: {message}")+class ConditionalBranchEvaluationError(SkyvernException):+ """A conditional block could not resolve which branch to take."""+++class MalformedBranchEvaluationError(ConditionalBranchEvaluationError):+ """The LLM's branch-evaluation output could not be safely aligned to the branches.++ Raised for a wrong result count, a missing/duplicate condition_index, or an+ unparseable shape. A distinct type lets the batch evaluator re-roll a fresh LLM call+ instead of hard-failing immediately or silently routing to the wrong branch.+ """++class FailedToSendWebhook(SkyvernException):def __init__(self,diff --git a/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2 b/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2index e6b9d4f66..61a0fced5 100644--- a/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2+++ b/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2@@ -17,6 +17,7 @@ Use this context to understand variable values:{{ context_json }}{% endif %}-For each condition, provide IN THIS ORDER:-1. "reasoning": Explain your reasoning for evaluating the condition. Compare the actual values.-2. "result": The boolean result (true/false) based on the condition.+Return exactly {{ conditions|length }} evaluation object{{ '' if conditions|length == 1 else 's' }} — one per numbered condition, covering every condition_index from 1 to {{ conditions|length }} exactly once. Do not merge, skip, or add conditions. For each condition, provide IN THIS ORDER:+1. "condition_index": The 1-based number of the condition this object evaluates (Condition 1 -> 1, Condition 2 -> 2, ...).+2. "reasoning": Explain your reasoning for evaluating the condition. Compare the actual values.+3. "result": The boolean result (true/false) based on the condition.diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex b949d3377..071cac752 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -45,8 +45,10 @@ from skyvern.constants import ()from skyvern.exceptions import (AzureConfigurationError,+ ConditionalBranchEvaluationError,ContextParameterValueNotFound,DownloadFileMaxSizeExceeded,+ MalformedBranchEvaluationError,MissingBrowserState,MissingBrowserStatePage,MissingStarterUrl,@@ -7293,6 +7295,145 @@ def _parse_single_evaluation(return (bool_result, rendered_expression)+# Number of times to evaluate the prompt-based conditional branches before giving up.+# The dominant failure is a transient malformed/under-returned LLM batch; one re-roll+# recovers most of them while still failing loudly when the model is persistently wrong.+MAX_PROMPT_BRANCH_EVAL_ATTEMPTS = 2+++def _build_branch_evaluation_schema(num_branches: int) -> dict[str, Any]:+ """Strict JSON schema for the batched branch-evaluation LLM call.++ ``additionalProperties: false`` plus a required ``condition_index`` stop the model from+ injecting hallucinated fields and force one self-identifying result per condition.+ """+ return {+ "type": "object",+ "additionalProperties": False,+ "properties": {+ "evaluations": {+ "type": "array",+ "items": {+ "type": "object",+ "additionalProperties": False,+ "properties": {+ "condition_index": {+ "type": "integer",+ "description": (+ "The 1-based index of the condition this object evaluates "+ "(Condition 1 -> 1, Condition 2 -> 2, ...)."+ ),+ },+ "reasoning": {+ "type": "string",+ "description": "Explanation of the reasoning behind evaluating the condition.",+ },+ "result": {+ "type": "boolean",+ "description": "TRUE if the condition is satisfied, FALSE otherwise.",+ },+ },+ "required": ["condition_index", "reasoning", "result"],+ },+ "description": "Exactly one evaluation per condition, covering condition_index 1..N.",+ "minItems": num_branches,+ "maxItems": num_branches,+ }+ },+ "required": ["evaluations"],+ }+++def _coerce_condition_index(raw: Any) -> int | None:+ """Read a 1-based ``condition_index`` the LLM may have typed loosely.++ Accepts ints, integral floats (``2.0``), and digit strings (``"2"``); returns ``None`` for+ bools (an int subclass that must not read as 0/1) and non-integral/garbage values. The schema+ is a prompt-level hint, not provider-enforced, so a model that under-returns can equally+ mistype the index; coercing keeps a loosely-typed index on the order-safe alignment path+ instead of misrouting via positional fallback.+ """+ if isinstance(raw, bool):+ return None+ if isinstance(raw, int):+ return raw+ if isinstance(raw, float):+ return int(raw) if raw.is_integer() else None+ if isinstance(raw, str):+ try:+ return int(raw.strip())+ except ValueError:+ return None+ return None+++def _align_branch_evaluations(+ *,+ output_value: Any,+ branches: list[BranchCondition],+ rendered_expressions: list[str],+) -> tuple[list[bool], list[str], dict[str, Any]]:+ """Align the LLM evaluations to ``branches``.++ Returns ``(results, rendered_expressions, normalized_output)``, where ``normalized_output``+ is the evaluations wrapped in a dict for recording/UI. Prefers the LLM-provided 1-based+ ``condition_index`` (coerced from ints, integral floats, or digit strings): this is order-safe+ and immune to hallucinated extra entries that would shift positional alignment. Falls back to+ positional parsing only when no usable index is present and the count matches exactly. A+ wrong-length or mis-indexed batch is discarded wholesale (never gap-filled) by raising+ ``MalformedBranchEvaluationError`` so the caller can retry or fail loudly.+ """+ n = len(branches)++ if isinstance(output_value, list):+ output_value = {"evaluations": output_value}+ if not isinstance(output_value, dict):+ raise MalformedBranchEvaluationError(f"unexpected output format: {type(output_value)}")+ try:+ raw_evaluations = _find_evaluations_array(output_value)+ except ValueError as exc:+ raise MalformedBranchEvaluationError(str(exc)) from exc++ by_index: dict[int, Any] = {}+ saw_condition_index = False+ for evaluation in raw_evaluations:+ if not isinstance(evaluation, dict):+ continue+ raw_index = _coerce_condition_index(evaluation.get("condition_index"))+ if raw_index is None:+ continue+ saw_condition_index = True+ if not (1 <= raw_index <= n):+ continue+ if raw_index in by_index:+ raise MalformedBranchEvaluationError(f"duplicate condition_index {raw_index}")+ by_index[raw_index] = evaluation++ if saw_condition_index:+ if set(by_index.keys()) != set(range(1, n + 1)):+ raise MalformedBranchEvaluationError(f"condition_index set {sorted(by_index.keys())} does not cover 1..{n}")+ ordered: list[Any] = [by_index[i] for i in range(1, n + 1)]+ else:+ # Legacy path (no condition_index): the LLM occasionally appends reasoning=None+ # placeholder entries. Strip them and accept only when the remainder is exactly N.+ well_formed = [e for e in raw_evaluations if not (isinstance(e, dict) and e.get("reasoning") is None)]+ ordered = well_formed if len(well_formed) == n else list(raw_evaluations)+ if len(ordered) != n:+ raise MalformedBranchEvaluationError(f"returned {len(ordered)} results for {n} branches")++ results_array: list[bool] = []+ llm_rendered_expressions: list[str] = []+ for idx, evaluation in enumerate(ordered):+ bool_result, rendered_expr = _parse_single_evaluation(+ evaluation=evaluation,+ idx=idx,+ fallback_rendered_expressions=rendered_expressions,+ )+ results_array.append(bool_result)+ llm_rendered_expressions.append(rendered_expr)+ return results_array, llm_rendered_expressions, output_value++# Pattern to find Jinja template blocks like {{ variable_name }}_JINJA_BLOCK_RE = re.compile(r"\{\{(.*?)\}\}")# Marker inserted into rendered expressions when a Jinja variable resolved to@@ -7522,63 +7663,52 @@ async def _evaluate_prompt_branch_conditions_batch(context_json=context_json,)- data_schema = {- "type": "object",- "properties": {- "evaluations": {- "type": "array",- "items": {- "type": "object",- "properties": {- "reasoning": {- "type": "string",- "description": "Explanation of the reasoning behind evaluating the condition.",- },- "result": {- "type": "boolean",- "description": "TRUE if the condition is satisfied, FALSE otherwise.",- },- },- "required": ["reasoning", "result"],- },- "description": "Array of evaluation results for each condition in the same order.",- "minItems": len(branches),- "maxItems": len(branches),- }- },- "required": ["evaluations"],- }+ data_schema = _build_branch_evaluation_schema(len(branches))desc_suffix = extraction_description_suffix or f"{len(branches)} conditions"- prompt_branch_eval_id = generate_random_string()- output_param = OutputParameter(- output_parameter_id=str(uuid.uuid4()),- key=f"prompt_branch_eval_{prompt_branch_eval_id}",- workflow_id=workflow_id,- created_at=datetime.now(),- modified_at=datetime.now(),- parameter_type=ParameterType.OUTPUT,- description=f"Conditional branch evaluation results ({desc_suffix})",- )- extraction_block = ExtractionBlock(- label=f"prompt_branch_eval_{prompt_branch_eval_id}",- data_extraction_goal=extraction_goal,- data_schema=data_schema,- output_parameter=output_param,- )- LOG.info(- "Conditional branch ExtractionBlock created (batched)",- block_label=log_label,- prompt_branch_eval_id=prompt_branch_eval_id,- num_conditions=len(branches),- extraction_goal_preview=extraction_goal[:500] if extraction_goal else None,- has_browser_session=browser_session_id is not None,- has_any_pure_natlang=has_any_pure_natlang,- has_context=context_json is not None,- )+ last_malformed: MalformedBranchEvaluationError | None = None+ for attempt in range(MAX_PROMPT_BRANCH_EVAL_ATTEMPTS):+ # Vary the goal on retries so the extraction cache key (which includes+ # data_extraction_goal) changes and we get a genuine re-roll instead of replaying+ # the malformed cached result that just failed validation.+ attempt_goal = extraction_goal+ if attempt > 0:+ attempt_goal = (+ f"{extraction_goal}\n\n"+ f"Re-evaluation attempt {attempt + 1}: return exactly {len(branches)} results, "+ f"one object per numbered condition, each tagged with its condition_index."+ )++ prompt_branch_eval_id = generate_random_string()+ output_param = OutputParameter(+ output_parameter_id=str(uuid.uuid4()),+ key=f"prompt_branch_eval_{prompt_branch_eval_id}",+ workflow_id=workflow_id,+ created_at=datetime.now(),+ modified_at=datetime.now(),+ parameter_type=ParameterType.OUTPUT,+ description=f"Conditional branch evaluation results ({desc_suffix})",+ )+ extraction_block = ExtractionBlock(+ label=f"prompt_branch_eval_{prompt_branch_eval_id}",+ data_extraction_goal=attempt_goal,+ data_schema=data_schema,+ output_parameter=output_param,+ )++ LOG.info(+ "Conditional branch ExtractionBlock created (batched)",+ block_label=log_label,+ prompt_branch_eval_id=prompt_branch_eval_id,+ num_conditions=len(branches),+ attempt=attempt,+ extraction_goal_preview=attempt_goal[:500] if attempt_goal else None,+ has_browser_session=browser_session_id is not None,+ has_any_pure_natlang=has_any_pure_natlang,+ has_context=context_json is not None,+ )- try:extraction_result = await extraction_block.execute(workflow_run_id=workflow_run_id,workflow_run_block_id=workflow_run_block_id,@@ -7587,22 +7717,45 @@ async def _evaluate_prompt_branch_conditions_batch()if not extraction_result.success:+ # Extraction-level failures already retry at the step level inside the task;+ # surface them immediately rather than re-rolling the whole batch.LOG.error("Conditional branch ExtractionBlock failed",block_label=log_label,failure_reason=extraction_result.failure_reason,)- raise ValueError(+ raise ConditionalBranchEvaluationError(f"Branch evaluation failed: "f"{extraction_result.failure_reason or 'Unknown error (no failure reason provided)'}")+ raw_output_value = extraction_result.output_parameter_value+ try:+ results_array, llm_rendered_expressions, normalized_output = _align_branch_evaluations(+ output_value=raw_output_value,+ branches=branches,+ rendered_expressions=rendered_expressions,+ )+ except MalformedBranchEvaluationError as malformed:+ last_malformed = malformed+ LOG.warning(+ "Conditional branch evaluation output malformed",+ block_label=log_label,+ attempt=attempt,+ will_retry=attempt + 1 < MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,+ error=str(malformed),+ raw_output=raw_output_value,+ )+ continue++ # Record the output parameter only for the attempt we actually accept, so a failed+ # attempt's payload never lands in the workflow context when a later attempt succeeds.if workflow_run_context:try:await extraction_block.record_output_parameter_value(workflow_run_context=workflow_run_context,workflow_run_id=workflow_run_id,- value=extraction_result.output_parameter_value,+ value=normalized_output,)except Exception:LOG.warning(@@ -7612,66 +7765,23 @@ async def _evaluate_prompt_branch_conditions_batch(exc_info=True,)- output_value = extraction_result.output_parameter_value-- results_array: list[bool] = []- llm_rendered_expressions: list[str] = []-- if isinstance(output_value, list):- output_value = {"evaluations": output_value}-- if not isinstance(output_value, dict):- raise ValueError(f"Unexpected output format: {type(output_value)}")-- raw_evaluations = _find_evaluations_array(output_value)-- # LLM sometimes splits compound criteria into per-clause sub-evaluations, emitting- # reasoning=None placeholder entries for each. Strip them and recover if the remainder- # matches len(branches); otherwise fall through to the existing hard-fail.- if len(raw_evaluations) > len(branches):- well_formed = [e for e in raw_evaluations if not (isinstance(e, dict) and e.get("reasoning") is None)]- if len(well_formed) == len(branches):- LOG.warning(- "LLM returned extra placeholder evaluations; using well-formed subset",- block_label=log_label,- total_returned=len(raw_evaluations),- well_formed_count=len(well_formed),- num_branches=len(branches),- )- raw_evaluations = well_formed-- for idx, evaluation in enumerate(raw_evaluations):- bool_result, rendered_expr = _parse_single_evaluation(- evaluation=evaluation,- idx=idx,- fallback_rendered_expressions=rendered_expressions,- )- results_array.append(bool_result)- llm_rendered_expressions.append(rendered_expr)-LOG.info("Conditional branch evaluation results",block_label=log_label,results=results_array,llm_rendered_expressions=llm_rendered_expressions,- raw_output=output_value,+ attempt=attempt,+ raw_output=normalized_output,)+ return (results_array, llm_rendered_expressions, extraction_goal, normalized_output)- if len(results_array) != len(branches):- raise ValueError(- f"Conditional branch evaluation returned {len(results_array)} results for {len(branches)} branches"- )-- return (results_array, llm_rendered_expressions, extraction_goal, output_value)-- except Exception as exc:- LOG.error(- "Conditional branch evaluation failed",- block_label=log_label,- error=str(exc),- exc_info=True,- )- raise ValueError(f"Conditional branch evaluation failed: {str(exc)}") from exc+ LOG.error(+ "Conditional branch evaluation failed after retries",+ block_label=log_label,+ attempts=MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,+ error=str(last_malformed),+ )+ raise ConditionalBranchEvaluationError(f"Conditional branch evaluation failed: {last_malformed}")class ConditionalBlock(Block):diff --git a/tests/unit/workflow/test_conditional_branch_evaluation.py b/tests/unit/workflow/test_conditional_branch_evaluation.pyindex 4d27a863b..d9abe2107 100644--- a/tests/unit/workflow/test_conditional_branch_evaluation.py+++ b/tests/unit/workflow/test_conditional_branch_evaluation.py@@ -6,12 +6,15 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestimport skyvern.forge.sdk.workflow.models.block as block_module+from skyvern.exceptions import ConditionalBranchEvaluationErrorfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.workflow.models.block import (BranchCondition,BranchEvaluationContext,ConditionalBlock,PromptBranchCriteria,+ _build_branch_evaluation_schema,+ _coerce_condition_index,_make_empty_params_explicit,)from skyvern.forge.sdk.workflow.models.parameter import OutputParameter@@ -439,7 +442,7 @@ async def test_extraction_failure_with_none_reason_produces_informative_error())mock_extraction_cls.return_value = mock_extraction- with pytest.raises(ValueError, match="Unknown error"):+ with pytest.raises(ConditionalBranchEvaluationError, match="Unknown error"):await block._evaluate_prompt_branches(branches=[branch],evaluation_context=evaluation_context,@@ -472,7 +475,7 @@ async def test_extraction_failure_with_reason_preserves_original_message() -> No)mock_extraction_cls.return_value = mock_extraction- with pytest.raises(ValueError, match="LLM rate limited"):+ with pytest.raises(ConditionalBranchEvaluationError, match="LLM rate limited"):await block._evaluate_prompt_branches(branches=[branch],evaluation_context=evaluation_context,@@ -558,7 +561,7 @@ async def test_extra_evals_not_recovered_when_well_formed_count_does_not_match()mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))mock_extraction_cls.return_value = mock_extraction- with pytest.raises(ValueError, match="3 results for 1 branches"):+ with pytest.raises(ConditionalBranchEvaluationError, match="3 results for 1 branches"):await block._evaluate_prompt_branches(branches=[branch],evaluation_context=evaluation_context,@@ -629,3 +632,322 @@ def test_prompt_template_includes_count_and_atomicity_for_compound_conditions()# Wording-coupled: if rephrased, confirm the replacement still conveys atomicity.assert "split" in rendered_one.lower()+++# ---------------------------------------------------------------------------+# Tests for condition_index alignment + malformed-batch retry (SKY-10682)+# ---------------------------------------------------------------------------+++def _two_branch_block() -> ConditionalBlock:+ return ConditionalBlock(+ label="cond",+ output_parameter=_output_parameter("out"),+ branch_conditions=[+ BranchCondition(criteria=PromptBranchCriteria(expression="condition A"), next_block_label="a"),+ BranchCondition(criteria=PromptBranchCriteria(expression="condition B"), next_block_label="b"),+ ],+ )+++def _no_context() -> BranchEvaluationContext:+ ctx = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)+ ctx.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]+ return ctx+++async def test_condition_index_alignment_is_order_independent() -> None:+ """Evaluations carrying condition_index must align by index, not position, so a+ reversed-order LLM response still maps each result to the correct branch."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ # Returned out of order: condition 2 first (True), then condition 1 (False).+ raw_evals = [+ {"condition_index": 2, "reasoning": "B", "result": True},+ {"condition_index": 1, "reasoning": "A", "result": False},+ ]+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))+ mock_extraction_cls.return_value = mock_extraction++ results, _, _, _ = await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )++ assert results == [False, True] # branch 0 (index 1) -> False, branch 1 (index 2) -> True+++async def test_hallucinated_unindexed_entry_does_not_misroute() -> None:+ """A hallucinated extra entry WITHOUT a condition_index (the shape that shifted+ positional alignment in SKY-10682) must be ignored; indexed entries align to the+ correct branches instead of misrouting."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ raw_evals = [+ {"reasoning": "hallucinated off-topic text", "result": True}, # junk, no condition_index+ {"condition_index": 1, "reasoning": "A", "result": False},+ {"condition_index": 2, "reasoning": "B", "result": True},+ ]+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))+ mock_extraction_cls.return_value = mock_extraction++ results, _, _, _ = await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )++ assert results == [False, True]+++async def test_under_return_retries_then_succeeds() -> None:+ """The SKY-10682 failure shape: the LLM returns fewer results than branches on the+ first attempt, then a clean response on retry. The batch must retry and succeed+ rather than failing the whole run."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ first = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])+ second = _extraction_result(+ block.output_parameter,+ [+ {"condition_index": 1, "reasoning": "A", "result": False},+ {"condition_index": 2, "reasoning": "B", "result": True},+ ],+ )+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(side_effect=[first, second])+ mock_extraction_cls.return_value = mock_extraction++ results, _, _, _ = await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )++ assert results == [False, True]+ assert mock_extraction.execute.await_count == 2+++async def test_under_return_fails_loudly_after_retries_exhausted() -> None:+ """If every attempt returns a malformed batch, the evaluation must fail loudly+ (raise) rather than silently routing to a default/wrong branch."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ bad = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=bad)+ mock_extraction_cls.return_value = mock_extraction++ with pytest.raises(ConditionalBranchEvaluationError):+ await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )+ assert mock_extraction.execute.await_count >= 2+++async def test_retry_varies_extraction_goal_for_true_reroll() -> None:+ """On retry the extraction goal must differ from the first attempt so the extraction+ cache key (which includes data_extraction_goal) changes and we get a genuine re-roll+ instead of replaying a cached malformed result (SKY-10682)."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ bad = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])+ good = _extraction_result(+ block.output_parameter,+ [+ {"condition_index": 1, "reasoning": "A", "result": False},+ {"condition_index": 2, "reasoning": "B", "result": True},+ ],+ )+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(side_effect=[bad, good])+ mock_extraction_cls.return_value = mock_extraction++ await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )++ goals = [call.kwargs["data_extraction_goal"] for call in mock_extraction_cls.call_args_list]+ assert len(goals) == 2+ assert goals[0] != goals[1]+++async def test_branch_eval_schema_is_strict_and_indexed() -> None:+ """The data_schema must require condition_index and forbid extra keys+ (additionalProperties: false) so the LLM cannot inject hallucinated fields like the+ off-topic `including` key seen in production (SKY-10682)."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ good = _extraction_result(+ block.output_parameter,+ [+ {"condition_index": 1, "reasoning": "A", "result": False},+ {"condition_index": 2, "reasoning": "B", "result": True},+ ],+ )+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=good)+ mock_extraction_cls.return_value = mock_extraction++ await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )++ schema = mock_extraction_cls.call_args.kwargs["data_schema"]+ item_schema = schema["properties"]["evaluations"]["items"]+ assert item_schema["additionalProperties"] is False+ assert "condition_index" in item_schema["properties"]+ assert "condition_index" in item_schema["required"]+++def test_prompt_template_requests_condition_index() -> None:+ rendered = prompt_engine.load_prompt(+ "conditional-prompt-branch-evaluation",+ conditions=["A", "B"],+ context_json=None,+ )+ assert "condition_index" in rendered+++def test_build_branch_evaluation_schema_is_strict_and_indexed() -> None:+ schema = _build_branch_evaluation_schema(3)+ evaluations = schema["properties"]["evaluations"]+ assert evaluations["minItems"] == 3+ assert evaluations["maxItems"] == 3+ item = evaluations["items"]+ assert item["additionalProperties"] is False+ assert item["required"] == ["condition_index", "reasoning", "result"]+ assert schema["additionalProperties"] is False+++async def test_string_condition_index_out_of_order_aligns_by_index() -> None:+ """The schema requests an integer condition_index but isn't provider-enforced, so the model+ may type it as a string. Out of order, positional alignment would misroute; digit strings must+ coerce to int and stay on the index-aligned path (SKY-10682)."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ raw_evals = [+ {"condition_index": "2", "reasoning": "B", "result": True},+ {"condition_index": "1", "reasoning": "A", "result": False},+ ]+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))+ mock_extraction_cls.return_value = mock_extraction++ results, _, _, _ = await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )++ assert results == [False, True]+++async def test_float_condition_index_out_of_order_aligns_by_index() -> None:+ """An integral-float condition_index (e.g. 2.0) must align by index too, not fall back to+ positional ordering, which would misroute on a reversed batch (SKY-10682)."""+ block = _two_branch_block()+ branches = [c for c in block.branch_conditions if not c.is_default]++ raw_evals = [+ {"condition_index": 2.0, "reasoning": "B", "result": True},+ {"condition_index": 1.0, "reasoning": "A", "result": False},+ ]+ with (+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,+ ):+ mock_extraction = MagicMock()+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))+ mock_extraction_cls.return_value = mock_extraction++ results, _, _, _ = await block._evaluate_prompt_branches(+ branches=branches,+ evaluation_context=_no_context(),+ workflow_run_id="wr",+ workflow_run_block_id="wrb",+ organization_id="org",+ )++ assert results == [False, True]+++def test_coerce_condition_index_handles_loose_types() -> None:+ assert _coerce_condition_index(2) == 2+ assert _coerce_condition_index(2.0) == 2+ assert _coerce_condition_index("2") == 2+ assert _coerce_condition_index(" 3 ") == 3+ # bool is an int subclass but is never a valid index+ assert _coerce_condition_index(True) is None+ assert _coerce_condition_index(False) is None+ # non-integral / unparseable values are rejected+ assert _coerce_condition_index(2.5) is None+ assert _coerce_condition_index("two") is None+ assert _coerce_condition_index("") is None+ assert _coerce_condition_index(None) is None