Skip to content

Automate browser based workflows with AI

perf(SKY-10319): check budget/max-steps before awaiting speculative task (#6313)

LawyZheng9224df71e339
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 659e19213..8b32f70dc 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -5012,19 +5012,7 @@ class ForgeAgent:
task_id=task.task_id,
)
- try:
- speculative_plan = await speculative_task
- except CancelledError:
- LOG.debug("Speculative extract-actions cancelled after verification finished", step_id=step.step_id)
- speculative_plan = None
- except Exception:
- LOG.warning(
- "Speculative extract-actions failed, next step will run sequentially",
- step_id=step.step_id,
- exc_info=True,
- )
- speculative_plan = None
-
+ # Budget check must precede speculative_task await so exhausted steps cancel early.
context = skyvern_context.current()
override_max_steps_per_run = context.max_steps_override if context else None
max_steps_per_run = (
@@ -5034,8 +5022,47 @@ class ForgeAgent:
or settings.MAX_STEPS_PER_RUN
)
- workflow_run_budget = await self._check_workflow_run_step_budget(organization, task)
- if workflow_run_budget is not None and workflow_run_budget[0] >= workflow_run_budget[1]:
+ try:
+ workflow_run_budget = await self._check_workflow_run_step_budget(organization, task)
+ except Exception:
+ LOG.warning("Budget preflight failed, cancelling speculative task", exc_info=True)
+ speculative_task.cancel()
+ try:
+ await speculative_task
+ except (CancelledError, Exception):
+ pass
+ raise
+ budget_exhausted = workflow_run_budget is not None and workflow_run_budget[0] >= workflow_run_budget[1]
+ steps_exhausted = step.order + 1 >= max_steps_per_run
+
+ if budget_exhausted or steps_exhausted:
+ speculative_task.cancel()
+ LOG.info(
+ "Cancelled speculative task — budget or max-steps exhausted",
+ step_id=step.step_id,
+ budget_exhausted=budget_exhausted,
+ steps_exhausted=steps_exhausted,
+ )
+ try:
+ await speculative_task
+ except (CancelledError, Exception):
+ pass
+ speculative_plan = None
+ else:
+ try:
+ speculative_plan = await speculative_task
+ except CancelledError:
+ LOG.debug("Speculative extract-actions cancelled after verification finished", step_id=step.step_id)
+ speculative_plan = None
+ except Exception:
+ LOG.warning(
+ "Speculative extract-actions failed, next step will run sequentially",
+ step_id=step.step_id,
+ exc_info=True,
+ )
+ speculative_plan = None
+
+ if budget_exhausted and workflow_run_budget is not None:
last_step = await self._terminate_for_workflow_run_step_budget(
organization=organization,
task=task,
@@ -5046,7 +5073,7 @@ class ForgeAgent:
)
return False, last_step, None
- if step.order + 1 >= max_steps_per_run:
+ if steps_exhausted:
LOG.info(
"Step completed but max steps reached, marking task as failed",
step_order=step.order,
diff --git a/tests/unit/test_budget_preflight.py b/tests/unit/test_budget_preflight.py
new file mode 100644
index 000000000..a680cefbb
--- /dev/null
+++ b/tests/unit/test_budget_preflight.py
@@ -0,0 +1,52 @@
+"""Tests for budget/max-steps preflight check before speculative scrape+LLM."""
+
+import ast
+import inspect
+import textwrap
+
+from skyvern.forge import agent as agent_module
+
+
+def _get_await_order(source: str, target_names: list[str]) -> list[str]:
+ """Return the order in which target coroutine names are awaited in source."""
+ tree = ast.parse(textwrap.dedent(source))
+ found = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Await):
+ continue
+ val = node.value
+ name = None
+ if isinstance(val, ast.Call):
+ func = val.func
+ if isinstance(func, ast.Attribute):
+ name = func.attr
+ elif isinstance(func, ast.Name):
+ name = func.id
+ elif isinstance(val, ast.Name):
+ name = val.id
+ if name and name in target_names and name not in found:
+ found.append(name)
+ return found
+
+
+def test_budget_check_before_speculative_await() -> None:
+ """_check_workflow_run_step_budget must be called before speculative_task is awaited."""
+ source = inspect.getsource(agent_module.ForgeAgent._handle_completed_step_with_parallel_verification)
+ order = _get_await_order(source, ["_check_workflow_run_step_budget", "speculative_task"])
+ assert "_check_workflow_run_step_budget" in order, "Budget check not found in function"
+ budget_idx = order.index("_check_workflow_run_step_budget")
+ # speculative_task may appear multiple times (cancel path + normal path);
+ # the first await of speculative_task should be AFTER the budget check
+ assert "speculative_task" in order, "speculative_task await not found"
+ spec_idx = order.index("speculative_task")
+ assert budget_idx < spec_idx, (
+ f"Budget check (position {budget_idx}) must come before speculative_task await (position {spec_idx})"
+ )
+
+
+def test_speculative_task_cancelled_on_budget_path() -> None:
+ """The code must call speculative_task.cancel() when budget is exhausted."""
+ source = inspect.getsource(agent_module.ForgeAgent._handle_completed_step_with_parallel_verification)
+ assert "speculative_task.cancel()" in source, "Missing speculative_task.cancel() call"
+ assert "budget_exhausted" in source, "Missing budget_exhausted variable"
+ assert "steps_exhausted" in source, "Missing steps_exhausted variable"