Skip to content

Automate browser based workflows with AI

fix(SKY-10682): harden conditional branch evaluation against malformed LLM batches (#6400)

Shuchang Zhengf9289402f2fc
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index c89854c44..f16e75555 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -161,6 +161,19 @@ class InvalidOpenAIResponseFormat(SkyvernException):
super().__init__(f"Invalid response format: {message}")
+class ConditionalBranchEvaluationError(SkyvernException):
+ """A conditional block could not resolve which branch to take."""
+
+
+class MalformedBranchEvaluationError(ConditionalBranchEvaluationError):
+ """The LLM's branch-evaluation output could not be safely aligned to the branches.
+
+ Raised for a wrong result count, a missing/duplicate condition_index, or an
+ unparseable shape. A distinct type lets the batch evaluator re-roll a fresh LLM call
+ instead of hard-failing immediately or silently routing to the wrong branch.
+ """
+
+
class FailedToSendWebhook(SkyvernException):
def __init__(
self,
diff --git a/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2 b/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2
index e6b9d4f66..61a0fced5 100644
--- a/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2
+++ b/skyvern/forge/prompts/skyvern/conditional-prompt-branch-evaluation.j2
@@ -17,6 +17,7 @@ Use this context to understand variable values:
{{ context_json }}
{% endif %}
-For each condition, provide IN THIS ORDER:
-1. "reasoning": Explain your reasoning for evaluating the condition. Compare the actual values.
-2. "result": The boolean result (true/false) based on the condition.
+Return exactly {{ conditions|length }} evaluation object{{ '' if conditions|length == 1 else 's' }} — one per numbered condition, covering every condition_index from 1 to {{ conditions|length }} exactly once. Do not merge, skip, or add conditions. For each condition, provide IN THIS ORDER:
+1. "condition_index": The 1-based number of the condition this object evaluates (Condition 1 -> 1, Condition 2 -> 2, ...).
+2. "reasoning": Explain your reasoning for evaluating the condition. Compare the actual values.
+3. "result": The boolean result (true/false) based on the condition.
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index b949d3377..071cac752 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -45,8 +45,10 @@ from skyvern.constants import (
)
from skyvern.exceptions import (
AzureConfigurationError,
+ ConditionalBranchEvaluationError,
ContextParameterValueNotFound,
DownloadFileMaxSizeExceeded,
+ MalformedBranchEvaluationError,
MissingBrowserState,
MissingBrowserStatePage,
MissingStarterUrl,
@@ -7293,6 +7295,145 @@ def _parse_single_evaluation(
return (bool_result, rendered_expression)
+# Number of times to evaluate the prompt-based conditional branches before giving up.
+# The dominant failure is a transient malformed/under-returned LLM batch; one re-roll
+# recovers most of them while still failing loudly when the model is persistently wrong.
+MAX_PROMPT_BRANCH_EVAL_ATTEMPTS = 2
+
+
+def _build_branch_evaluation_schema(num_branches: int) -> dict[str, Any]:
+ """Strict JSON schema for the batched branch-evaluation LLM call.
+
+ ``additionalProperties: false`` plus a required ``condition_index`` stop the model from
+ injecting hallucinated fields and force one self-identifying result per condition.
+ """
+ return {
+ "type": "object",
+ "additionalProperties": False,
+ "properties": {
+ "evaluations": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "additionalProperties": False,
+ "properties": {
+ "condition_index": {
+ "type": "integer",
+ "description": (
+ "The 1-based index of the condition this object evaluates "
+ "(Condition 1 -> 1, Condition 2 -> 2, ...)."
+ ),
+ },
+ "reasoning": {
+ "type": "string",
+ "description": "Explanation of the reasoning behind evaluating the condition.",
+ },
+ "result": {
+ "type": "boolean",
+ "description": "TRUE if the condition is satisfied, FALSE otherwise.",
+ },
+ },
+ "required": ["condition_index", "reasoning", "result"],
+ },
+ "description": "Exactly one evaluation per condition, covering condition_index 1..N.",
+ "minItems": num_branches,
+ "maxItems": num_branches,
+ }
+ },
+ "required": ["evaluations"],
+ }
+
+
+def _coerce_condition_index(raw: Any) -> int | None:
+ """Read a 1-based ``condition_index`` the LLM may have typed loosely.
+
+ Accepts ints, integral floats (``2.0``), and digit strings (``"2"``); returns ``None`` for
+ bools (an int subclass that must not read as 0/1) and non-integral/garbage values. The schema
+ is a prompt-level hint, not provider-enforced, so a model that under-returns can equally
+ mistype the index; coercing keeps a loosely-typed index on the order-safe alignment path
+ instead of misrouting via positional fallback.
+ """
+ if isinstance(raw, bool):
+ return None
+ if isinstance(raw, int):
+ return raw
+ if isinstance(raw, float):
+ return int(raw) if raw.is_integer() else None
+ if isinstance(raw, str):
+ try:
+ return int(raw.strip())
+ except ValueError:
+ return None
+ return None
+
+
+def _align_branch_evaluations(
+ *,
+ output_value: Any,
+ branches: list[BranchCondition],
+ rendered_expressions: list[str],
+) -> tuple[list[bool], list[str], dict[str, Any]]:
+ """Align the LLM evaluations to ``branches``.
+
+ Returns ``(results, rendered_expressions, normalized_output)``, where ``normalized_output``
+ is the evaluations wrapped in a dict for recording/UI. Prefers the LLM-provided 1-based
+ ``condition_index`` (coerced from ints, integral floats, or digit strings): this is order-safe
+ and immune to hallucinated extra entries that would shift positional alignment. Falls back to
+ positional parsing only when no usable index is present and the count matches exactly. A
+ wrong-length or mis-indexed batch is discarded wholesale (never gap-filled) by raising
+ ``MalformedBranchEvaluationError`` so the caller can retry or fail loudly.
+ """
+ n = len(branches)
+
+ if isinstance(output_value, list):
+ output_value = {"evaluations": output_value}
+ if not isinstance(output_value, dict):
+ raise MalformedBranchEvaluationError(f"unexpected output format: {type(output_value)}")
+ try:
+ raw_evaluations = _find_evaluations_array(output_value)
+ except ValueError as exc:
+ raise MalformedBranchEvaluationError(str(exc)) from exc
+
+ by_index: dict[int, Any] = {}
+ saw_condition_index = False
+ for evaluation in raw_evaluations:
+ if not isinstance(evaluation, dict):
+ continue
+ raw_index = _coerce_condition_index(evaluation.get("condition_index"))
+ if raw_index is None:
+ continue
+ saw_condition_index = True
+ if not (1 <= raw_index <= n):
+ continue
+ if raw_index in by_index:
+ raise MalformedBranchEvaluationError(f"duplicate condition_index {raw_index}")
+ by_index[raw_index] = evaluation
+
+ if saw_condition_index:
+ if set(by_index.keys()) != set(range(1, n + 1)):
+ raise MalformedBranchEvaluationError(f"condition_index set {sorted(by_index.keys())} does not cover 1..{n}")
+ ordered: list[Any] = [by_index[i] for i in range(1, n + 1)]
+ else:
+ # Legacy path (no condition_index): the LLM occasionally appends reasoning=None
+ # placeholder entries. Strip them and accept only when the remainder is exactly N.
+ well_formed = [e for e in raw_evaluations if not (isinstance(e, dict) and e.get("reasoning") is None)]
+ ordered = well_formed if len(well_formed) == n else list(raw_evaluations)
+ if len(ordered) != n:
+ raise MalformedBranchEvaluationError(f"returned {len(ordered)} results for {n} branches")
+
+ results_array: list[bool] = []
+ llm_rendered_expressions: list[str] = []
+ for idx, evaluation in enumerate(ordered):
+ bool_result, rendered_expr = _parse_single_evaluation(
+ evaluation=evaluation,
+ idx=idx,
+ fallback_rendered_expressions=rendered_expressions,
+ )
+ results_array.append(bool_result)
+ llm_rendered_expressions.append(rendered_expr)
+ return results_array, llm_rendered_expressions, output_value
+
+
# Pattern to find Jinja template blocks like {{ variable_name }}
_JINJA_BLOCK_RE = re.compile(r"\{\{(.*?)\}\}")
# Marker inserted into rendered expressions when a Jinja variable resolved to
@@ -7522,63 +7663,52 @@ async def _evaluate_prompt_branch_conditions_batch(
context_json=context_json,
)
- data_schema = {
- "type": "object",
- "properties": {
- "evaluations": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "reasoning": {
- "type": "string",
- "description": "Explanation of the reasoning behind evaluating the condition.",
- },
- "result": {
- "type": "boolean",
- "description": "TRUE if the condition is satisfied, FALSE otherwise.",
- },
- },
- "required": ["reasoning", "result"],
- },
- "description": "Array of evaluation results for each condition in the same order.",
- "minItems": len(branches),
- "maxItems": len(branches),
- }
- },
- "required": ["evaluations"],
- }
+ data_schema = _build_branch_evaluation_schema(len(branches))
desc_suffix = extraction_description_suffix or f"{len(branches)} conditions"
- prompt_branch_eval_id = generate_random_string()
- output_param = OutputParameter(
- output_parameter_id=str(uuid.uuid4()),
- key=f"prompt_branch_eval_{prompt_branch_eval_id}",
- workflow_id=workflow_id,
- created_at=datetime.now(),
- modified_at=datetime.now(),
- parameter_type=ParameterType.OUTPUT,
- description=f"Conditional branch evaluation results ({desc_suffix})",
- )
- extraction_block = ExtractionBlock(
- label=f"prompt_branch_eval_{prompt_branch_eval_id}",
- data_extraction_goal=extraction_goal,
- data_schema=data_schema,
- output_parameter=output_param,
- )
- LOG.info(
- "Conditional branch ExtractionBlock created (batched)",
- block_label=log_label,
- prompt_branch_eval_id=prompt_branch_eval_id,
- num_conditions=len(branches),
- extraction_goal_preview=extraction_goal[:500] if extraction_goal else None,
- has_browser_session=browser_session_id is not None,
- has_any_pure_natlang=has_any_pure_natlang,
- has_context=context_json is not None,
- )
+ last_malformed: MalformedBranchEvaluationError | None = None
+ for attempt in range(MAX_PROMPT_BRANCH_EVAL_ATTEMPTS):
+ # Vary the goal on retries so the extraction cache key (which includes
+ # data_extraction_goal) changes and we get a genuine re-roll instead of replaying
+ # the malformed cached result that just failed validation.
+ attempt_goal = extraction_goal
+ if attempt > 0:
+ attempt_goal = (
+ f"{extraction_goal}\n\n"
+ f"Re-evaluation attempt {attempt + 1}: return exactly {len(branches)} results, "
+ f"one object per numbered condition, each tagged with its condition_index."
+ )
+
+ prompt_branch_eval_id = generate_random_string()
+ output_param = OutputParameter(
+ output_parameter_id=str(uuid.uuid4()),
+ key=f"prompt_branch_eval_{prompt_branch_eval_id}",
+ workflow_id=workflow_id,
+ created_at=datetime.now(),
+ modified_at=datetime.now(),
+ parameter_type=ParameterType.OUTPUT,
+ description=f"Conditional branch evaluation results ({desc_suffix})",
+ )
+ extraction_block = ExtractionBlock(
+ label=f"prompt_branch_eval_{prompt_branch_eval_id}",
+ data_extraction_goal=attempt_goal,
+ data_schema=data_schema,
+ output_parameter=output_param,
+ )
+
+ LOG.info(
+ "Conditional branch ExtractionBlock created (batched)",
+ block_label=log_label,
+ prompt_branch_eval_id=prompt_branch_eval_id,
+ num_conditions=len(branches),
+ attempt=attempt,
+ extraction_goal_preview=attempt_goal[:500] if attempt_goal else None,
+ has_browser_session=browser_session_id is not None,
+ has_any_pure_natlang=has_any_pure_natlang,
+ has_context=context_json is not None,
+ )
- try:
extraction_result = await extraction_block.execute(
workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
@@ -7587,22 +7717,45 @@ async def _evaluate_prompt_branch_conditions_batch(
)
if not extraction_result.success:
+ # Extraction-level failures already retry at the step level inside the task;
+ # surface them immediately rather than re-rolling the whole batch.
LOG.error(
"Conditional branch ExtractionBlock failed",
block_label=log_label,
failure_reason=extraction_result.failure_reason,
)
- raise ValueError(
+ raise ConditionalBranchEvaluationError(
f"Branch evaluation failed: "
f"{extraction_result.failure_reason or 'Unknown error (no failure reason provided)'}"
)
+ raw_output_value = extraction_result.output_parameter_value
+ try:
+ results_array, llm_rendered_expressions, normalized_output = _align_branch_evaluations(
+ output_value=raw_output_value,
+ branches=branches,
+ rendered_expressions=rendered_expressions,
+ )
+ except MalformedBranchEvaluationError as malformed:
+ last_malformed = malformed
+ LOG.warning(
+ "Conditional branch evaluation output malformed",
+ block_label=log_label,
+ attempt=attempt,
+ will_retry=attempt + 1 < MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,
+ error=str(malformed),
+ raw_output=raw_output_value,
+ )
+ continue
+
+ # Record the output parameter only for the attempt we actually accept, so a failed
+ # attempt's payload never lands in the workflow context when a later attempt succeeds.
if workflow_run_context:
try:
await extraction_block.record_output_parameter_value(
workflow_run_context=workflow_run_context,
workflow_run_id=workflow_run_id,
- value=extraction_result.output_parameter_value,
+ value=normalized_output,
)
except Exception:
LOG.warning(
@@ -7612,66 +7765,23 @@ async def _evaluate_prompt_branch_conditions_batch(
exc_info=True,
)
- output_value = extraction_result.output_parameter_value
-
- results_array: list[bool] = []
- llm_rendered_expressions: list[str] = []
-
- if isinstance(output_value, list):
- output_value = {"evaluations": output_value}
-
- if not isinstance(output_value, dict):
- raise ValueError(f"Unexpected output format: {type(output_value)}")
-
- raw_evaluations = _find_evaluations_array(output_value)
-
- # LLM sometimes splits compound criteria into per-clause sub-evaluations, emitting
- # reasoning=None placeholder entries for each. Strip them and recover if the remainder
- # matches len(branches); otherwise fall through to the existing hard-fail.
- if len(raw_evaluations) > len(branches):
- well_formed = [e for e in raw_evaluations if not (isinstance(e, dict) and e.get("reasoning") is None)]
- if len(well_formed) == len(branches):
- LOG.warning(
- "LLM returned extra placeholder evaluations; using well-formed subset",
- block_label=log_label,
- total_returned=len(raw_evaluations),
- well_formed_count=len(well_formed),
- num_branches=len(branches),
- )
- raw_evaluations = well_formed
-
- for idx, evaluation in enumerate(raw_evaluations):
- bool_result, rendered_expr = _parse_single_evaluation(
- evaluation=evaluation,
- idx=idx,
- fallback_rendered_expressions=rendered_expressions,
- )
- results_array.append(bool_result)
- llm_rendered_expressions.append(rendered_expr)
-
LOG.info(
"Conditional branch evaluation results",
block_label=log_label,
results=results_array,
llm_rendered_expressions=llm_rendered_expressions,
- raw_output=output_value,
+ attempt=attempt,
+ raw_output=normalized_output,
)
+ return (results_array, llm_rendered_expressions, extraction_goal, normalized_output)
- if len(results_array) != len(branches):
- raise ValueError(
- f"Conditional branch evaluation returned {len(results_array)} results for {len(branches)} branches"
- )
-
- return (results_array, llm_rendered_expressions, extraction_goal, output_value)
-
- except Exception as exc:
- LOG.error(
- "Conditional branch evaluation failed",
- block_label=log_label,
- error=str(exc),
- exc_info=True,
- )
- raise ValueError(f"Conditional branch evaluation failed: {str(exc)}") from exc
+ LOG.error(
+ "Conditional branch evaluation failed after retries",
+ block_label=log_label,
+ attempts=MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,
+ error=str(last_malformed),
+ )
+ raise ConditionalBranchEvaluationError(f"Conditional branch evaluation failed: {last_malformed}")
class ConditionalBlock(Block):
diff --git a/tests/unit/workflow/test_conditional_branch_evaluation.py b/tests/unit/workflow/test_conditional_branch_evaluation.py
index 4d27a863b..d9abe2107 100644
--- a/tests/unit/workflow/test_conditional_branch_evaluation.py
+++ b/tests/unit/workflow/test_conditional_branch_evaluation.py
@@ -6,12 +6,15 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import skyvern.forge.sdk.workflow.models.block as block_module
+from skyvern.exceptions import ConditionalBranchEvaluationError
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.workflow.models.block import (
BranchCondition,
BranchEvaluationContext,
ConditionalBlock,
PromptBranchCriteria,
+ _build_branch_evaluation_schema,
+ _coerce_condition_index,
_make_empty_params_explicit,
)
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
@@ -439,7 +442,7 @@ async def test_extraction_failure_with_none_reason_produces_informative_error()
)
mock_extraction_cls.return_value = mock_extraction
- with pytest.raises(ValueError, match="Unknown error"):
+ with pytest.raises(ConditionalBranchEvaluationError, match="Unknown error"):
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
@@ -472,7 +475,7 @@ async def test_extraction_failure_with_reason_preserves_original_message() -> No
)
mock_extraction_cls.return_value = mock_extraction
- with pytest.raises(ValueError, match="LLM rate limited"):
+ with pytest.raises(ConditionalBranchEvaluationError, match="LLM rate limited"):
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
@@ -558,7 +561,7 @@ async def test_extra_evals_not_recovered_when_well_formed_count_does_not_match()
mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
mock_extraction_cls.return_value = mock_extraction
- with pytest.raises(ValueError, match="3 results for 1 branches"):
+ with pytest.raises(ConditionalBranchEvaluationError, match="3 results for 1 branches"):
await block._evaluate_prompt_branches(
branches=[branch],
evaluation_context=evaluation_context,
@@ -629,3 +632,322 @@ def test_prompt_template_includes_count_and_atomicity_for_compound_conditions()
# Wording-coupled: if rephrased, confirm the replacement still conveys atomicity.
assert "split" in rendered_one.lower()
+
+
+# ---------------------------------------------------------------------------
+# Tests for condition_index alignment + malformed-batch retry (SKY-10682)
+# ---------------------------------------------------------------------------
+
+
+def _two_branch_block() -> ConditionalBlock:
+ return ConditionalBlock(
+ label="cond",
+ output_parameter=_output_parameter("out"),
+ branch_conditions=[
+ BranchCondition(criteria=PromptBranchCriteria(expression="condition A"), next_block_label="a"),
+ BranchCondition(criteria=PromptBranchCriteria(expression="condition B"), next_block_label="b"),
+ ],
+ )
+
+
+def _no_context() -> BranchEvaluationContext:
+ ctx = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
+ ctx.build_llm_safe_context_snapshot = MagicMock(return_value={}) # type: ignore[method-assign]
+ return ctx
+
+
+async def test_condition_index_alignment_is_order_independent() -> None:
+ """Evaluations carrying condition_index must align by index, not position, so a
+ reversed-order LLM response still maps each result to the correct branch."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ # Returned out of order: condition 2 first (True), then condition 1 (False).
+ raw_evals = [
+ {"condition_index": 2, "reasoning": "B", "result": True},
+ {"condition_index": 1, "reasoning": "A", "result": False},
+ ]
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
+ mock_extraction_cls.return_value = mock_extraction
+
+ results, _, _, _ = await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+
+ assert results == [False, True] # branch 0 (index 1) -> False, branch 1 (index 2) -> True
+
+
+async def test_hallucinated_unindexed_entry_does_not_misroute() -> None:
+ """A hallucinated extra entry WITHOUT a condition_index (the shape that shifted
+ positional alignment in SKY-10682) must be ignored; indexed entries align to the
+ correct branches instead of misrouting."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ raw_evals = [
+ {"reasoning": "hallucinated off-topic text", "result": True}, # junk, no condition_index
+ {"condition_index": 1, "reasoning": "A", "result": False},
+ {"condition_index": 2, "reasoning": "B", "result": True},
+ ]
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
+ mock_extraction_cls.return_value = mock_extraction
+
+ results, _, _, _ = await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+
+ assert results == [False, True]
+
+
+async def test_under_return_retries_then_succeeds() -> None:
+ """The SKY-10682 failure shape: the LLM returns fewer results than branches on the
+ first attempt, then a clean response on retry. The batch must retry and succeed
+ rather than failing the whole run."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ first = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])
+ second = _extraction_result(
+ block.output_parameter,
+ [
+ {"condition_index": 1, "reasoning": "A", "result": False},
+ {"condition_index": 2, "reasoning": "B", "result": True},
+ ],
+ )
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(side_effect=[first, second])
+ mock_extraction_cls.return_value = mock_extraction
+
+ results, _, _, _ = await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+
+ assert results == [False, True]
+ assert mock_extraction.execute.await_count == 2
+
+
+async def test_under_return_fails_loudly_after_retries_exhausted() -> None:
+ """If every attempt returns a malformed batch, the evaluation must fail loudly
+ (raise) rather than silently routing to a default/wrong branch."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ bad = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(return_value=bad)
+ mock_extraction_cls.return_value = mock_extraction
+
+ with pytest.raises(ConditionalBranchEvaluationError):
+ await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+ assert mock_extraction.execute.await_count >= 2
+
+
+async def test_retry_varies_extraction_goal_for_true_reroll() -> None:
+ """On retry the extraction goal must differ from the first attempt so the extraction
+ cache key (which includes data_extraction_goal) changes and we get a genuine re-roll
+ instead of replaying a cached malformed result (SKY-10682)."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ bad = _extraction_result(block.output_parameter, [{"condition_index": 1, "reasoning": "A", "result": False}])
+ good = _extraction_result(
+ block.output_parameter,
+ [
+ {"condition_index": 1, "reasoning": "A", "result": False},
+ {"condition_index": 2, "reasoning": "B", "result": True},
+ ],
+ )
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(side_effect=[bad, good])
+ mock_extraction_cls.return_value = mock_extraction
+
+ await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+
+ goals = [call.kwargs["data_extraction_goal"] for call in mock_extraction_cls.call_args_list]
+ assert len(goals) == 2
+ assert goals[0] != goals[1]
+
+
+async def test_branch_eval_schema_is_strict_and_indexed() -> None:
+ """The data_schema must require condition_index and forbid extra keys
+ (additionalProperties: false) so the LLM cannot inject hallucinated fields like the
+ off-topic `including` key seen in production (SKY-10682)."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ good = _extraction_result(
+ block.output_parameter,
+ [
+ {"condition_index": 1, "reasoning": "A", "result": False},
+ {"condition_index": 2, "reasoning": "B", "result": True},
+ ],
+ )
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(return_value=good)
+ mock_extraction_cls.return_value = mock_extraction
+
+ await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+
+ schema = mock_extraction_cls.call_args.kwargs["data_schema"]
+ item_schema = schema["properties"]["evaluations"]["items"]
+ assert item_schema["additionalProperties"] is False
+ assert "condition_index" in item_schema["properties"]
+ assert "condition_index" in item_schema["required"]
+
+
+def test_prompt_template_requests_condition_index() -> None:
+ rendered = prompt_engine.load_prompt(
+ "conditional-prompt-branch-evaluation",
+ conditions=["A", "B"],
+ context_json=None,
+ )
+ assert "condition_index" in rendered
+
+
+def test_build_branch_evaluation_schema_is_strict_and_indexed() -> None:
+ schema = _build_branch_evaluation_schema(3)
+ evaluations = schema["properties"]["evaluations"]
+ assert evaluations["minItems"] == 3
+ assert evaluations["maxItems"] == 3
+ item = evaluations["items"]
+ assert item["additionalProperties"] is False
+ assert item["required"] == ["condition_index", "reasoning", "result"]
+ assert schema["additionalProperties"] is False
+
+
+async def test_string_condition_index_out_of_order_aligns_by_index() -> None:
+ """The schema requests an integer condition_index but isn't provider-enforced, so the model
+ may type it as a string. Out of order, positional alignment would misroute; digit strings must
+ coerce to int and stay on the index-aligned path (SKY-10682)."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ raw_evals = [
+ {"condition_index": "2", "reasoning": "B", "result": True},
+ {"condition_index": "1", "reasoning": "A", "result": False},
+ ]
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
+ mock_extraction_cls.return_value = mock_extraction
+
+ results, _, _, _ = await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+
+ assert results == [False, True]
+
+
+async def test_float_condition_index_out_of_order_aligns_by_index() -> None:
+ """An integral-float condition_index (e.g. 2.0) must align by index too, not fall back to
+ positional ordering, which would misroute on a reversed batch (SKY-10682)."""
+ block = _two_branch_block()
+ branches = [c for c in block.branch_conditions if not c.is_default]
+
+ raw_evals = [
+ {"condition_index": 2.0, "reasoning": "B", "result": True},
+ {"condition_index": 1.0, "reasoning": "A", "result": False},
+ ]
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt", return_value="goal"),
+ patch("skyvern.forge.sdk.workflow.models.block.ExtractionBlock") as mock_extraction_cls,
+ ):
+ mock_extraction = MagicMock()
+ mock_extraction.execute = AsyncMock(return_value=_extraction_result(block.output_parameter, raw_evals))
+ mock_extraction_cls.return_value = mock_extraction
+
+ results, _, _, _ = await block._evaluate_prompt_branches(
+ branches=branches,
+ evaluation_context=_no_context(),
+ workflow_run_id="wr",
+ workflow_run_block_id="wrb",
+ organization_id="org",
+ )
+
+ assert results == [False, True]
+
+
+def test_coerce_condition_index_handles_loose_types() -> None:
+ assert _coerce_condition_index(2) == 2
+ assert _coerce_condition_index(2.0) == 2
+ assert _coerce_condition_index("2") == 2
+ assert _coerce_condition_index(" 3 ") == 3
+ # bool is an int subclass but is never a valid index
+ assert _coerce_condition_index(True) is None
+ assert _coerce_condition_index(False) is None
+ # non-integral / unparseable values are rejected
+ assert _coerce_condition_index(2.5) is None
+ assert _coerce_condition_index("two") is None
+ assert _coerce_condition_index("") is None
+ assert _coerce_condition_index(None) is None