Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
perf(SKY-10319): check budget/max-steps before awaiting speculative task (#6313)
LawyZheng9224df71e339
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 659e19213..8b32f70dc 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -5012,19 +5012,7 @@ class ForgeAgent:task_id=task.task_id,)- try:- speculative_plan = await speculative_task- except CancelledError:- LOG.debug("Speculative extract-actions cancelled after verification finished", step_id=step.step_id)- speculative_plan = None- except Exception:- LOG.warning(- "Speculative extract-actions failed, next step will run sequentially",- step_id=step.step_id,- exc_info=True,- )- speculative_plan = None-+ # Budget check must precede speculative_task await so exhausted steps cancel early.context = skyvern_context.current()override_max_steps_per_run = context.max_steps_override if context else Nonemax_steps_per_run = (@@ -5034,8 +5022,47 @@ class ForgeAgent:or settings.MAX_STEPS_PER_RUN)- workflow_run_budget = await self._check_workflow_run_step_budget(organization, task)- if workflow_run_budget is not None and workflow_run_budget[0] >= workflow_run_budget[1]:+ try:+ workflow_run_budget = await self._check_workflow_run_step_budget(organization, task)+ except Exception:+ LOG.warning("Budget preflight failed, cancelling speculative task", exc_info=True)+ speculative_task.cancel()+ try:+ await speculative_task+ except (CancelledError, Exception):+ pass+ raise+ budget_exhausted = workflow_run_budget is not None and workflow_run_budget[0] >= workflow_run_budget[1]+ steps_exhausted = step.order + 1 >= max_steps_per_run++ if budget_exhausted or steps_exhausted:+ speculative_task.cancel()+ LOG.info(+ "Cancelled speculative task — budget or max-steps exhausted",+ step_id=step.step_id,+ budget_exhausted=budget_exhausted,+ steps_exhausted=steps_exhausted,+ )+ try:+ await speculative_task+ except (CancelledError, Exception):+ pass+ speculative_plan = None+ else:+ try:+ speculative_plan = await speculative_task+ except CancelledError:+ LOG.debug("Speculative extract-actions cancelled after verification finished", step_id=step.step_id)+ speculative_plan = None+ except Exception:+ LOG.warning(+ "Speculative extract-actions failed, next step will run sequentially",+ step_id=step.step_id,+ exc_info=True,+ )+ speculative_plan = None++ if budget_exhausted and workflow_run_budget is not None:last_step = await self._terminate_for_workflow_run_step_budget(organization=organization,task=task,@@ -5046,7 +5073,7 @@ class ForgeAgent:)return False, last_step, None- if step.order + 1 >= max_steps_per_run:+ if steps_exhausted:LOG.info("Step completed but max steps reached, marking task as failed",step_order=step.order,diff --git a/tests/unit/test_budget_preflight.py b/tests/unit/test_budget_preflight.pynew file mode 100644index 000000000..a680cefbb--- /dev/null+++ b/tests/unit/test_budget_preflight.py@@ -0,0 +1,52 @@+"""Tests for budget/max-steps preflight check before speculative scrape+LLM."""++import ast+import inspect+import textwrap++from skyvern.forge import agent as agent_module+++def _get_await_order(source: str, target_names: list[str]) -> list[str]:+ """Return the order in which target coroutine names are awaited in source."""+ tree = ast.parse(textwrap.dedent(source))+ found = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.Await):+ continue+ val = node.value+ name = None+ if isinstance(val, ast.Call):+ func = val.func+ if isinstance(func, ast.Attribute):+ name = func.attr+ elif isinstance(func, ast.Name):+ name = func.id+ elif isinstance(val, ast.Name):+ name = val.id+ if name and name in target_names and name not in found:+ found.append(name)+ return found+++def test_budget_check_before_speculative_await() -> None:+ """_check_workflow_run_step_budget must be called before speculative_task is awaited."""+ source = inspect.getsource(agent_module.ForgeAgent._handle_completed_step_with_parallel_verification)+ order = _get_await_order(source, ["_check_workflow_run_step_budget", "speculative_task"])+ assert "_check_workflow_run_step_budget" in order, "Budget check not found in function"+ budget_idx = order.index("_check_workflow_run_step_budget")+ # speculative_task may appear multiple times (cancel path + normal path);+ # the first await of speculative_task should be AFTER the budget check+ assert "speculative_task" in order, "speculative_task await not found"+ spec_idx = order.index("speculative_task")+ assert budget_idx < spec_idx, (+ f"Budget check (position {budget_idx}) must come before speculative_task await (position {spec_idx})"+ )+++def test_speculative_task_cancelled_on_budget_path() -> None:+ """The code must call speculative_task.cancel() when budget is exhausted."""+ source = inspect.getsource(agent_module.ForgeAgent._handle_completed_step_with_parallel_verification)+ assert "speculative_task.cancel()" in source, "Missing speculative_task.cancel() call"+ assert "budget_exhausted" in source, "Missing budget_exhausted variable"+ assert "steps_exhausted" in source, "Missing steps_exhausted variable"