Skip to content

Automate browser based workflows with AI

v3 reviewer 3/3: wire mint-audit + post-run dispatch + mid-run hooks (SKY-7676) (#6184)

pedrohsdb5143e85e10b6
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index 550874c3d..feb1b8aa6 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -1,9 +1,10 @@
from __future__ import annotations
+import asyncio
import json
import re
from datetime import datetime, timezone
-from typing import Any, cast
+from typing import Any, cast, get_args
import structlog
from jinja2.sandbox import SandboxedEnvironment
@@ -23,6 +24,9 @@ from skyvern.forge.sdk.schemas.totp_codes import OTPType
from skyvern.schemas.workflows import BlockStatus
from skyvern.services import script_service
from skyvern.services.otp_service import poll_otp_value
+from skyvern.services.script_reviewer_v3.cohort import is_v3_cohort
+from skyvern.services.script_reviewer_v3.midrun import v3_review_in_flight
+from skyvern.services.script_reviewer_v3.types import FailureContext, InterceptedActionType
from skyvern.utils.css_selector import compute_selector_options
from skyvern.utils.prompt_engine import load_prompt_with_elements, load_prompt_with_elements_tracked
from skyvern.utils.prompt_truncation import truncate_extraction_schema
@@ -49,6 +53,8 @@ jinja_sandbox_env = SandboxedEnvironment()
LOG = structlog.get_logger()
+_INTERCEPTED_ACTION_TYPES = frozenset(get_args(InterceptedActionType))
+
INPUT_GOAL = """- The intention to fill out an input: {intention}.
- The overall goal that the user wants to achieve: {prompt}."""
@@ -171,6 +177,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
failed_selector: str | None = None,
block_label: str | None = None,
recoverable_marker_id: int | None = None,
+ v3_parent_episode_id: str | None = None,
) -> str | None:
"""Click an element using AI to locate it based on intention.
@@ -180,7 +187,31 @@ class RealSkyvernPageAi(SkyvernPageAi):
the script reviewer can fix the selector. Only set when called from
the ai='fallback' path in skyvern_page.py.
block_label: The cached block label (from SkyvernPage.current_label).
+ v3_parent_episode_id: pre-existing episode id from v3 mid-run Class B
+ fall-through. When set, this call updates that episode instead of
+ creating a duplicate.
"""
+ # v3 mid-run hook: only on the original call (not on internal recursion).
+ # If v3 commits an in-flight fix (Class A), we short-circuit; if it
+ # gives up (Class B), the episode already exists and we propagate
+ # v3_parent_episode_id so the agent-fallback path updates it instead
+ # of creating a duplicate row.
+ if v3_parent_episode_id is None:
+ ep_id, class_a = await self._maybe_run_v3_midrun(
+ action_type="click",
+ failed_selector=failed_selector,
+ intention=intention,
+ value=None,
+ totp_identifier=None,
+ totp_url=None,
+ block_label=block_label,
+ )
+ if class_a:
+ # v3's live_try_click already advanced the page state.
+ return None
+ if ep_id is not None:
+ v3_parent_episode_id = ep_id
+
try:
# Build the element tree of the current page for the prompt
context = skyvern_context.ensure_context()
@@ -241,6 +272,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
action=action,
block_label=block_label,
recoverable_marker_id=recoverable_marker_id,
+ v3_parent_episode_id=v3_parent_episode_id,
)
return selector
@@ -278,9 +310,29 @@ class RealSkyvernPageAi(SkyvernPageAi):
failed_selector: str | None = None,
block_label: str | None = None,
recoverable_marker_id: int | None = None,
+ v3_parent_episode_id: str | None = None,
) -> str:
"""Input text into an element using AI to determine the value."""
+ # v3 mid-run hook — see ai_click for the contract.
+ if v3_parent_episode_id is None:
+ ep_id, class_a = await self._maybe_run_v3_midrun(
+ action_type="fill",
+ failed_selector=failed_selector,
+ intention=intention,
+ value=value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ block_label=block_label,
+ )
+ if class_a:
+ # v3 mid-run committed the fill on the live page. The
+ # caller relies on page state, not this return value; "" means
+ # no selector was resolved.
+ return ""
+ if ep_id is not None:
+ v3_parent_episode_id = ep_id
+
context = skyvern_context.ensure_context()
value = value or ""
transformed_value = value
@@ -402,6 +454,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
action=action,
block_label=block_label,
recoverable_marker_id=recoverable_marker_id,
+ v3_parent_episode_id=v3_parent_episode_id,
)
else:
locator = self.page.locator(selector)
@@ -723,6 +776,116 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
return result
+ async def _maybe_run_v3_midrun(
+ self,
+ *,
+ action_type: str,
+ failed_selector: str | None,
+ intention: str,
+ value: str | None,
+ totp_identifier: str | None,
+ totp_url: str | None,
+ block_label: str | None,
+ ) -> tuple[str | None, bool]:
+ """v3 mid-run gate. Called at the top of ai_click / ai_input_text.
+
+ Returns ``(parent_episode_id, did_class_a)``.
+
+ - ``parent_episode_id`` is set when a v3 episode was created and v3
+ ran. None means v3 didn't fire (not v3 cohort, missing context,
+ or pre-conditions not met).
+ - ``did_class_a`` is True when v3's agent committed an in-flight fix
+ via a successful live_try_*. The caller short-circuits the AI
+ fallback in that case — the workflow already advanced.
+
+ Never raises — any internal failure logs and returns ``(None, False)``
+ so the caller falls through to the existing AI fallback path.
+ """
+ if failed_selector is None or not failed_selector.strip():
+ return None, False
+
+ episode_id_for_cleanup: str | None = None
+ organization_id_for_cleanup: str | None = None
+ try:
+ context = skyvern_context.current()
+ if context is None or not context.workflow_permanent_id or not context.workflow_run_id:
+ return None, False
+ # organization_id is required for episode FK + cohort lookup.
+ # Guard explicitly rather than silently coercing to "".
+ if not context.organization_id:
+ return None, False
+ if (context.code_version or 0) < 2:
+ return None, False
+ # FailureContext narrows the action_type set; bail if the
+ # caller passed something outside the typed set.
+ if action_type not in _INTERCEPTED_ACTION_TYPES:
+ return None, False
+ narrowed_action_type = cast(InterceptedActionType, action_type)
+
+ use_v3 = await is_v3_cohort(
+ workflow_permanent_id=context.workflow_permanent_id,
+ organization_id=context.organization_id,
+ workflow_run_id=context.workflow_run_id,
+ )
+ if not use_v3:
+ return None, False
+
+ # Create the episode now so v3's agent can attach decisions to it.
+ episode = await app.DATABASE.scripts.create_fallback_episode(
+ organization_id=context.organization_id,
+ workflow_permanent_id=context.workflow_permanent_id,
+ workflow_run_id=context.workflow_run_id,
+ block_label=block_label or self.current_label or "unknown",
+ fallback_type="element",
+ script_revision_id=context.script_revision_id,
+ page_url=self.page.url,
+ reviewer_version="v3",
+ )
+ episode_id_for_cleanup = episode.episode_id
+ organization_id_for_cleanup = context.organization_id
+
+ fc = FailureContext(
+ failed_selector=failed_selector,
+ intention=intention,
+ action_type=narrowed_action_type,
+ value=value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ page=self.page,
+ context=context,
+ episode_id=episode.episode_id,
+ )
+ result = await v3_review_in_flight(fc)
+ return episode.episode_id, result.decision.is_midrun_class_a()
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ LOG.warning(
+ "v3 mid-run hook raised — falling through to agent fallback",
+ action_type=action_type,
+ block_label=block_label,
+ exc_info=True,
+ )
+ if episode_id_for_cleanup is not None and organization_id_for_cleanup is not None:
+ try:
+ await app.DATABASE.scripts.delete_fallback_episode(
+ episode_id=episode_id_for_cleanup,
+ organization_id=organization_id_for_cleanup,
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ # Double-fault: v3_review_in_flight raised AND cleanup also
+ # failed. Log at error so the orphan is observable in DD —
+ # post-run v3 will see reviewed=False, reviewer_version="v3"
+ # with no useful agent data.
+ LOG.error(
+ "v3_midrun_orphan_episode_cleanup_failed",
+ episode_id=episode_id_for_cleanup,
+ exc_info=True,
+ )
+ return None, False
+
async def _record_element_fallback_episode(
self,
context: skyvern_context.SkyvernContext,
@@ -732,6 +895,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
action: Any,
block_label: str | None = None,
recoverable_marker_id: int | None = None,
+ v3_parent_episode_id: str | None = None,
) -> None:
"""Record an element-level fallback episode when ai_click/ai_input_text fires
because a CSS selector failed or was missing. Gated on code_version >= 2.
@@ -742,6 +906,10 @@ class RealSkyvernPageAi(SkyvernPageAi):
`ai='proactive'` because no semantic selector existed at codegen.
AI succeeded; capture the element pick so the reviewer can later
upgrade the call to `selector=, ai='fallback'`.
+
+ ``v3_parent_episode_id`` is set on Class B fall-through from mid-run v3.
+ The episode row was already created when v3 fired; we update_ it with
+ the agent fallback's action data instead of creating a duplicate.
"""
if failed_selector is None and recoverable_marker_id is None:
return
@@ -809,23 +977,40 @@ class RealSkyvernPageAi(SkyvernPageAi):
f"Original selector: {failed_selector or '(none)'}. "
f"Intention: {intention}"
)
- await app.DATABASE.scripts.create_fallback_episode(
- organization_id=context.organization_id or "",
- workflow_permanent_id=context.workflow_permanent_id,
- workflow_run_id=context.workflow_run_id,
- block_label=block_label or self.current_label or "unknown",
- fallback_type="element",
- script_revision_id=context.script_revision_id,
- error_message=error_msg,
- page_url=self.page.url,
- agent_actions=action_data,
- )
- LOG.info(
- "Recorded element fallback episode for selector failure",
- block_label=block_label or self.current_label,
- action_type=action_type,
- failed_selector=failed_selector,
- )
+ if v3_parent_episode_id:
+ # v3 mid-run Class B fall-through: episode already exists; update
+ # it with the agent's action_data + error_message instead of
+ # creating a duplicate row.
+ await app.DATABASE.scripts.update_fallback_episode(
+ episode_id=v3_parent_episode_id,
+ organization_id=context.organization_id or "",
+ agent_actions=action_data,
+ error_message=error_msg,
+ )
+ LOG.info(
+ "Updated v3 mid-run episode with agent fallback action data",
+ episode_id=v3_parent_episode_id,
+ block_label=block_label or self.current_label,
+ action_type=action_type,
+ )
+ else:
+ await app.DATABASE.scripts.create_fallback_episode(
+ organization_id=context.organization_id or "",
+ workflow_permanent_id=context.workflow_permanent_id,
+ workflow_run_id=context.workflow_run_id,
+ block_label=block_label or self.current_label or "unknown",
+ fallback_type="element",
+ script_revision_id=context.script_revision_id,
+ error_message=error_msg,
+ page_url=self.page.url,
+ agent_actions=action_data,
+ )
+ LOG.info(
+ "Recorded element fallback episode for selector failure",
+ block_label=block_label or self.current_label,
+ action_type=action_type,
+ failed_selector=failed_selector,
+ )
except Exception:
LOG.warning("Failed to record element fallback episode for selector failure", exc_info=True)
diff --git a/skyvern/core/script_generations/skyvern_page_ai.py b/skyvern/core/script_generations/skyvern_page_ai.py
index c91ab9146..cb6bd85f6 100644
--- a/skyvern/core/script_generations/skyvern_page_ai.py
+++ b/skyvern/core/script_generations/skyvern_page_ai.py
@@ -23,8 +23,14 @@ class SkyvernPageAi(Protocol):
failed_selector: str | None = None,
block_label: str | None = None,
recoverable_marker_id: int | None = None,
+ v3_parent_episode_id: str | None = None,
) -> str | None:
- """Click an element using AI to locate it based on intention."""
+ """Click an element using AI to locate it based on intention.
+
+ ``v3_parent_episode_id`` is set on the recursive fall-through call when
+ the v3 mid-run agent gave up (Class B). It tells the implementation to
+ update the existing fallback episode rather than create a duplicate.
+ """
...
async def ai_input_text(
@@ -39,8 +45,12 @@ class SkyvernPageAi(Protocol):
failed_selector: str | None = None,
block_label: str | None = None,
recoverable_marker_id: int | None = None,
+ v3_parent_episode_id: str | None = None,
) -> str:
- """Input text into an element using AI to determine the value."""
+ """Input text into an element using AI to determine the value.
+
+ See ``ai_click`` for the ``v3_parent_episode_id`` contract.
+ """
...
async def ai_upload_file(
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index a4864ec9b..7cb4d30c5 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -128,6 +128,8 @@ from skyvern.schemas.workflows import (
WorkflowStatus,
)
from skyvern.services import script_service, workflow_script_service
+from skyvern.services.script_reviewer_v3.cohort import is_v3_cohort
+from skyvern.services.script_reviewer_v3.postrun import v3_review_post_run
from skyvern.services.webhook_delivery import PreparedWorkflowWebhook, deliver_webhook_with_retries
from skyvern.utils.css_selector import build_action_summaries_with_timing # shared with script_service
from skyvern.utils.secret_headers import merge_masked_headers
@@ -6482,21 +6484,55 @@ class WorkflowService:
)
return
- # Cap ALL script reviews (fallback + failure) per wpid per day to prevent
- # runaway revision churn when the same issue repeats every run.
- cap_exceeded = await self._check_script_review_cap(
+ # Resolve cohort FIRST so the cap check + post-dispatch increment
+ # can route to the cohort's own counter.
+ use_v3 = await is_v3_cohort(
workflow_permanent_id=workflow.workflow_permanent_id,
organization_id=workflow.organization_id,
+ workflow_run_id=workflow_run.workflow_run_id,
)
+
+ # Cap ALL script reviews (fallback + failure) per wpid per day to prevent
+ # runaway revision churn when the same issue repeats every run.
+ if use_v3:
+ cap_exceeded = await self._check_script_review_cap_v3(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=workflow.organization_id,
+ )
+ else:
+ cap_exceeded = await self._check_script_review_cap(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=workflow.organization_id,
+ )
if cap_exceeded:
LOG.info(
"Skipping script review — daily cap exceeded for wpid",
workflow_permanent_id=workflow.workflow_permanent_id,
workflow_run_id=workflow_run.workflow_run_id,
pre_finally_status=pre_finally_status,
+ cohort="v3" if use_v3 else "v2",
)
return
+ # Fast-path skip: v3 cohort + zero fallback episodes = nothing
+ # to review. Saves an LLM round-trip and a cap-counter increment
+ # on cold-mint runs, which produce no episodes by construction.
+ # v2 keeps its existing behavior (reviews whether or not episodes
+ # exist) — only the v3 path adds this gate.
+ if use_v3:
+ v3_episodes = await app.DATABASE.scripts.get_all_episodes_by_workflow_run_id(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=workflow.organization_id,
+ )
+ if not v3_episodes:
+ LOG.info(
+ "v3_postrun_skipped_no_episodes",
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ script_revision_id=script_revision_id,
+ )
+ return
+
# Non-blocking lock per script family
cache = CacheFactory.get_cache()
lock = None
@@ -6507,11 +6543,23 @@ class WorkflowService:
except AttributeError:
LOG.debug("Cache doesn't support locking for script reviewer")
+ async def _run_reviewer_dispatch() -> Any | None:
+ if use_v3:
+ return await v3_review_post_run(
+ organization_id=workflow.organization_id,
+ workflow_run=workflow_run,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ script_revision_id=script_revision_id,
+ )
+ await self._run_reviewer_locked(workflow, workflow_run, script_revision_id, script_id)
+ return None
+
review_ran = False
+ review_result: Any | None = None
if lock is not None:
try:
async with lock:
- await self._run_reviewer_locked(workflow, workflow_run, script_revision_id, script_id)
+ review_result = await _run_reviewer_dispatch()
review_ran = True
except LockError:
LOG.info(
@@ -6521,15 +6569,20 @@ class WorkflowService:
)
else:
# No Redis/cache available - proceed without lock (graceful degradation for OSS)
- await self._run_reviewer_locked(workflow, workflow_run, script_revision_id, script_id)
+ review_result = await _run_reviewer_dispatch()
review_ran = True
- # Increment the review counter ONLY after a review actually ran.
- # Skipped reviews (e.g., LockError) should not consume cap budget.
if review_ran:
- await self._increment_script_review_counter(
- workflow_permanent_id=workflow.workflow_permanent_id,
- )
+ if use_v3:
+ v3_persist_cap_consumed = bool(getattr(review_result, "v3_persist_cap_consumed", False))
+ if not v3_persist_cap_consumed:
+ await self._increment_script_review_counter_v3(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ )
+ else:
+ await self._increment_script_review_counter(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ )
except Exception:
LOG.warning(
"Failed to trigger script reviewer",
@@ -6548,11 +6601,10 @@ class WorkflowService:
"""Build the Redis key for the daily script-review counter (v3 cohort).
Separate from the v2 key so cohort comparisons stay clean and v3
- traffic doesn't contaminate v2's counter. Thin wrapper around the
- module-level ``v3_script_review_cap_key`` so non-WorkflowService
- callers (e.g. the mint-audit trigger in workflow_script_service.py)
- can reach the same key construction without holding an instance
- reference.
+ traffic doesn't contaminate v2's counter. Thin wrapper around the module-level
+ ``v3_script_review_cap_key`` so non-WorkflowService callers (e.g.
+ the mint-audit trigger in workflow_script_service.py) can reach
+ the same key construction without holding an instance reference.
"""
return v3_script_review_cap_key(workflow_permanent_id)
@@ -6647,9 +6699,7 @@ class WorkflowService:
async def _increment_script_review_counter_v3(self, workflow_permanent_id: str) -> None:
"""v3 cohort mirror of ``_increment_script_review_counter`` writing v3 key.
- Used post-dispatch in ``_trigger_script_reviewer`` after a successful
- v3 cohort review. Separating from v2 lets us compare cohort counters
- cleanly.
+ Called from ``_trigger_script_reviewer`` after a successful v3 review_ran.
"""
await self._increment_review_counter_for_key(self._v3_script_review_cap_key(workflow_permanent_id))
@@ -6694,6 +6744,7 @@ class WorkflowService:
if cache is None:
return None
cap = await self._get_script_review_cap(organization_id)
+ # v3-specific key — separate from v2's shared key so cohort comparisons stay clean.
cap_key = self._v3_script_review_cap_key(workflow_permanent_id)
lock_name = f"v3_cap:{workflow_permanent_id}"
lock = cache.get_lock(lock_name, blocking_timeout=2, timeout=5)
@@ -6887,6 +6938,7 @@ class WorkflowService:
episode_id=episode.episode_id,
organization_id=workflow.organization_id,
reviewer_output=None,
+ reviewer_version="v2",
)
return
@@ -6933,6 +6985,7 @@ class WorkflowService:
organization_id=workflow.organization_id,
reviewer_output=str(updated_blocks) if updated_blocks else None,
new_script_revision_id=new_script.script_revision_id if new_script else None,
+ reviewer_version="v2",
)
except Exception:
diff --git a/skyvern/library/skyvern_browser_page_ai.py b/skyvern/library/skyvern_browser_page_ai.py
index 1244a3b23..a9d39a3fe 100644
--- a/skyvern/library/skyvern_browser_page_ai.py
+++ b/skyvern/library/skyvern_browser_page_ai.py
@@ -49,6 +49,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):
failed_selector: str | None = None, # noqa: ARG002 — accepted for Protocol compat, no episode recording in library path
block_label: str | None = None, # noqa: ARG002
recoverable_marker_id: int | None = None, # noqa: ARG002 — Protocol compat
+ v3_parent_episode_id: str | None = None, # noqa: ARG002 — Protocol compat; v3 mid-run is server-side only
) -> str | None:
"""Click an element using AI via API call.
@@ -87,6 +88,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):
failed_selector: str | None = None, # noqa: ARG002 — Protocol compat, see ai_click docstring
block_label: str | None = None, # noqa: ARG002
recoverable_marker_id: int | None = None, # noqa: ARG002 — Protocol compat
+ v3_parent_episode_id: str | None = None, # noqa: ARG002 — Protocol compat; v3 mid-run is server-side only
) -> str:
"""Input text into an element using AI via API call."""
diff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.py
index 2fd82a7ac..4c2108d07 100644
--- a/skyvern/services/script_reviewer.py
+++ b/skyvern/services/script_reviewer.py
@@ -600,6 +600,7 @@ class ScriptReviewer:
episode_id=episode.episode_id,
organization_id=organization_id,
reviewer_output="TRIAGE: not_code_fixable — skipped",
+ reviewer_version="v2",
)
LOG.info(
"ScriptReviewer: skipping non-code-fixable episode",
diff --git a/skyvern/services/script_reviewer_v3/budget.py b/skyvern/services/script_reviewer_v3/budget.py
index e3a8d5c0e..6dea96583 100644
--- a/skyvern/services/script_reviewer_v3/budget.py
+++ b/skyvern/services/script_reviewer_v3/budget.py
@@ -137,8 +137,8 @@ class RunBudget:
pre-charged at the per-review ceiling). Returns ``None`` if either cap
would be exceeded — caller should skip v3 and fall through to agent.
- Reservation (not actual usage) is held until :meth:`InvocationHandle.finalize_cost`
- reconciles with the true cost spent during the review.
+ Reservation (not actual usage) is held until :meth:`reconcile_invocation`
+ replaces the reservation with the true cost spent during the review.
"""
async with self._lock:
if self._invocations_used >= self.max_invocations_per_run:
@@ -150,6 +150,21 @@ class RunBudget:
self._cost_reserved_usd = new_reservation
return InvocationHandle(budget=self, reserved_usd=self.per_review_cost_ceiling_usd)
+ async def reconcile_invocation(self, *, reserved_usd: float, actual_usd: float) -> None:
+ """Replace a pre-charged invocation reservation with actual spend."""
+ async with self._lock:
+ self._cost_reserved_usd = max(
+ 0.0,
+ self._cost_reserved_usd - reserved_usd + actual_usd,
+ )
+ if actual_usd > reserved_usd:
+ LOG.warning(
+ "v3 review exceeded cost reservation",
+ reserved_usd=reserved_usd,
+ actual_usd=actual_usd,
+ overshoot_usd=actual_usd - reserved_usd,
+ )
+
def to_metrics(self) -> dict[str, float | int]:
return {
"invocations_used": self._invocations_used,
@@ -175,18 +190,7 @@ class InvocationHandle:
reserved_usd: float
async def finalize_cost(self, actual_usd: float) -> None:
- async with self.budget._lock:
- # Adjust reserved -> actual:
- # - remove the reserved amount, add back the actual amount
- # - clamp at 0 to handle transient arithmetic edge cases
- self.budget._cost_reserved_usd = max(
- 0.0,
- self.budget._cost_reserved_usd - self.reserved_usd + actual_usd,
- )
- if actual_usd > self.reserved_usd:
- LOG.warning(
- "v3 review exceeded cost reservation",
- reserved_usd=self.reserved_usd,
- actual_usd=actual_usd,
- overshoot_usd=actual_usd - self.reserved_usd,
- )
+ await self.budget.reconcile_invocation(
+ reserved_usd=self.reserved_usd,
+ actual_usd=actual_usd,
+ )
diff --git a/skyvern/services/script_reviewer_v3/midrun.py b/skyvern/services/script_reviewer_v3/midrun.py
index ceeae0b17..8b01bdad9 100644
--- a/skyvern/services/script_reviewer_v3/midrun.py
+++ b/skyvern/services/script_reviewer_v3/midrun.py
@@ -150,79 +150,87 @@ async def v3_review_in_flight(
timeline=[],
)
- registry = build_registry()
- llm_caller = LLMCaller(llm_key=V3_REVIEWER_MODEL)
-
- user_prompt = build_midrun_user_prompt(
- episode_id=fc.episode_id,
- fc_summary=_summarize_failure_context(fc),
- )
+ # If setup fails after this point, the reserved invocation is finalized
+ # at $0.00 but the invocation slot remains consumed to prevent retry storms.
+ try:
+ registry = build_registry()
+ llm_caller = LLMCaller(llm_key=V3_REVIEWER_MODEL)
- LOG.info(
- "v3_midrun_started",
- episode_id=fc.episode_id,
- workflow_run_id=getattr(fc.context, "workflow_run_id", None),
- action_type=fc.action_type,
- failed_selector=fc.failed_selector,
- budget_max_cycles=budget.max_cycles,
- budget_max_cost_usd=budget.max_cost_usd,
- )
+ user_prompt = build_midrun_user_prompt(
+ episode_id=fc.episode_id,
+ fc_summary=_summarize_failure_context(fc),
+ )
- try:
- result = await asyncio.wait_for(
- run_agent_loop(
- llm_caller=llm_caller,
- system_prompt=MIDRUN_SYSTEM_PROMPT,
- user_prompt=user_prompt,
- registry=registry,
- agent_kind="midrun",
- context=fc,
- budget=budget,
- organization_id=org_id,
- prompt_name=prompt_name,
- invocation_handle=invocation_handle,
- ),
- timeout=wall_clock_seconds,
+ LOG.info(
+ "v3_midrun_started",
+ episode_id=fc.episode_id,
+ workflow_run_id=getattr(fc.context, "workflow_run_id", None),
+ action_type=fc.action_type,
+ failed_selector=fc.failed_selector,
+ budget_max_cycles=budget.max_cycles,
+ budget_max_cost_usd=budget.max_cost_usd,
)
- except asyncio.TimeoutError:
- LOG.warning(
- "v3_midrun_wall_clock_exceeded",
+
+ try:
+ result = await asyncio.wait_for(
+ run_agent_loop(
+ llm_caller=llm_caller,
+ system_prompt=MIDRUN_SYSTEM_PROMPT,
+ user_prompt=user_prompt,
+ registry=registry,
+ agent_kind="midrun",
+ context=fc,
+ budget=budget,
+ organization_id=org_id,
+ prompt_name=prompt_name,
+ ),
+ timeout=wall_clock_seconds,
+ )
+ except asyncio.TimeoutError:
+ LOG.warning(
+ "v3_midrun_wall_clock_exceeded",
+ episode_id=fc.episode_id,
+ wall_clock_seconds=wall_clock_seconds,
+ cycles_completed=budget.cycles_used,
+ )
+ decision = Decision.loop_error("wall_clock_exceeded")
+ await _persist_midrun_terminal(organization_id=org_id, fc=fc, decision=decision)
+ return V3MidRunResult(decision=decision, budget_used=budget, timeline=[])
+
+ final_decision = result.terminal_decision or Decision.loop_error("no_terminal")
+ LOG.info(
+ "v3_midrun_complete",
episode_id=fc.episode_id,
- wall_clock_seconds=wall_clock_seconds,
- cycles_completed=budget.cycles_used,
+ workflow_run_id=getattr(fc.context, "workflow_run_id", None),
+ decision_type=final_decision.type,
+ decision_reason=final_decision.reason,
+ investigation_summary=(final_decision.investigation_summary or "")[:300],
+ new_script_revision_id=final_decision.new_script_revision_id,
+ class_a=final_decision.is_midrun_class_a(),
+ class_b=final_decision.is_midrun_class_b(),
+ cycles_used=result.budget.cycles_used,
+ tokens_used=result.budget.tokens_used,
+ cost_usd_used=round(result.budget.cost_usd_used, 6),
+ elapsed_seconds=round(result.budget.elapsed_seconds, 2),
)
- decision = Decision.loop_error("wall_clock_exceeded")
- await _persist_midrun_terminal(organization_id=org_id, fc=fc, decision=decision)
- return V3MidRunResult(decision=decision, budget_used=budget, timeline=[])
-
- final_decision = result.terminal_decision or Decision.loop_error("no_terminal")
- LOG.info(
- "v3_midrun_complete",
- episode_id=fc.episode_id,
- workflow_run_id=getattr(fc.context, "workflow_run_id", None),
- decision_type=final_decision.type,
- decision_reason=final_decision.reason,
- investigation_summary=(final_decision.investigation_summary or "")[:300],
- new_script_revision_id=final_decision.new_script_revision_id,
- class_a=final_decision.is_midrun_class_a(),
- class_b=final_decision.is_midrun_class_b(),
- cycles_used=result.budget.cycles_used,
- tokens_used=result.budget.tokens_used,
- cost_usd_used=round(result.budget.cost_usd_used, 6),
- elapsed_seconds=round(result.budget.elapsed_seconds, 2),
- )
- await _persist_midrun_terminal(
- organization_id=org_id,
- fc=fc,
- decision=final_decision,
- )
+ await _persist_midrun_terminal(
+ organization_id=org_id,
+ fc=fc,
+ decision=final_decision,
+ )
- return V3MidRunResult(
- decision=final_decision,
- budget_used=result.budget,
- timeline=result.timeline,
- )
+ return V3MidRunResult(
+ decision=final_decision,
+ budget_used=result.budget,
+ timeline=result.timeline,
+ )
+ finally:
+ if invocation_handle is not None:
+ try:
+ await invocation_handle.finalize_cost(budget.cost_usd_used)
+ except Exception: # pragma: no cover
+ LOG.warning("v3_midrun_invocation_handle_finalize_failed", exc_info=True)
__all__ = ["v3_review_in_flight"]
diff --git a/skyvern/services/script_reviewer_v3/postrun.py b/skyvern/services/script_reviewer_v3/postrun.py
index 923911fdd..83c9866bb 100644
--- a/skyvern/services/script_reviewer_v3/postrun.py
+++ b/skyvern/services/script_reviewer_v3/postrun.py
@@ -330,6 +330,8 @@ async def v3_review_post_run(
budget_used=prc.budget,
timeline=[],
per_episode_persist_results=persist_results,
+ v3_persist_cap_consumed=prc.v3_persist_cap_consumed,
+ v3_persist_cap_acquisitions=prc.v3_persist_cap_acquisitions,
)
# Single deterministic pass to apply per-episode persists. Persists run
@@ -369,6 +371,8 @@ async def v3_review_post_run(
budget_used=result.budget,
timeline=result.timeline,
per_episode_persist_results=persist_results,
+ v3_persist_cap_consumed=prc.v3_persist_cap_consumed,
+ v3_persist_cap_acquisitions=prc.v3_persist_cap_acquisitions,
)
diff --git a/skyvern/services/script_reviewer_v3/skills/persist.py b/skyvern/services/script_reviewer_v3/skills/persist.py
index 8c35447e2..c73b405f0 100644
--- a/skyvern/services/script_reviewer_v3/skills/persist.py
+++ b/skyvern/services/script_reviewer_v3/skills/persist.py
@@ -31,6 +31,7 @@ from skyvern.forge import app
from skyvern.forge.sdk.cache.factory import CacheFactory
from skyvern.forge.sdk.core import skyvern_context
from skyvern.services.script_reviewer_v3.skills.base import Skill, SkillError, SkillResult
+from skyvern.services.script_reviewer_v3.types import PostRunContext
LOG = structlog.get_logger()
@@ -114,6 +115,15 @@ def _update_context_revision(
agent_context.script_id = new_script_id
+def _record_v3_persist_cap_acquisition(agent_context: Any) -> None:
+ """Mark contexts that already consumed the v3 persist cap."""
+ if not isinstance(agent_context, PostRunContext):
+ return
+ agent_context.v3_persist_cap_consumed = True
+ current_count = getattr(agent_context, "v3_persist_cap_acquisitions", 0) or 0
+ agent_context.v3_persist_cap_acquisitions = int(current_count) + 1
+
+
async def _enforce_v3_cap(
workflow_permanent_id: str,
organization_id: str | None = None,
@@ -166,6 +176,7 @@ async def _handler_persist_block_edit(args: dict[str, Any], context: Any) -> Ski
"hint": "Per-wpid daily v3 persist cap reached. No edit was applied.",
}
)
+ _record_v3_persist_cap_acquisition(context)
# Per-script Redis lock to serialize concurrent persists. v3_persist scope
# is distinct from v2's script_reviewer:{} lock so the two don't deadlock.
@@ -256,6 +267,7 @@ async def _handler_persist_script_rewrite(args: dict[str, Any], context: Any) ->
"reason": "daily_cap_exceeded",
}
)
+ _record_v3_persist_cap_acquisition(context)
cache = CacheFactory.get_cache()
base_script_id = persist_ctx["base_script"].script_id
diff --git a/skyvern/services/script_reviewer_v3/types.py b/skyvern/services/script_reviewer_v3/types.py
index a840e35c8..8e48b15e7 100644
--- a/skyvern/services/script_reviewer_v3/types.py
+++ b/skyvern/services/script_reviewer_v3/types.py
@@ -72,6 +72,8 @@ class PostRunContext:
# demotion review during the post-run pass.
mid_run_class_a_episode_ids: list[str]
budget: Budget
+ v3_persist_cap_consumed: bool = False
+ v3_persist_cap_acquisitions: int = 0
@dataclasses.dataclass
@@ -149,6 +151,10 @@ class V3PostRunResult:
# Structured persist outcomes from the post-loop deterministic pass.
# Populated even when persists fail so the caller sees what was attempted.
per_episode_persist_results: dict[str, PerEpisodePersistResult] = dataclasses.field(default_factory=dict)
+ # True when a post-run persist skill acquired the v3 daily cap before
+ # attempting a script write. This is distinct from per-episode DB persists.
+ v3_persist_cap_consumed: bool = False
+ v3_persist_cap_acquisitions: int = 0
__all__ = [
diff --git a/skyvern/services/workflow_script_service.py b/skyvern/services/workflow_script_service.py
index 37d25f5bc..dea9c1165 100644
--- a/skyvern/services/workflow_script_service.py
+++ b/skyvern/services/workflow_script_service.py
@@ -1,3 +1,4 @@
+import ast
import base64
import hashlib
import re
@@ -18,14 +19,27 @@ from skyvern.core.script_generations.generate_script import (
)
from skyvern.core.script_generations.transform_workflow_run import transform_workflow_run_to_code_gen_input
from skyvern.forge import app
+from skyvern.forge.sdk.cache.factory import CacheFactory
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.workflow.models.block import get_all_blocks
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, is_adaptive_caching
from skyvern.schemas.scripts import FileEncoding, Script, ScriptFileCreate, ScriptStatus
from skyvern.schemas.workflows import BlockType
from skyvern.services import script_service
+from skyvern.services.script_reviewer_v3.cohort import is_v3_cohort
+from skyvern.services.script_reviewer_v3.mint_audit import (
+ SuspiciousLiteralFinding,
+ build_allowed_strings_envelope,
+ find_suspicious_selector_literals,
+)
+from skyvern.services.script_reviewer_v3.mint_review import fire_and_forget_mint_review
from skyvern.utils.url_validators import prepend_scheme_and_validate_url
+# Deferred to call-site because ``skyvern.forge.sdk.workflow.service``
+# imports this module, creating a circular import at module-load time.
+# Hoisting upstream would require splitting the cap helpers out of
+# service.py into their own module (future cleanup).
+
LOG = structlog.get_logger()
jinja_sandbox_env = SandboxedEnvironment()
strict_jinja_sandbox_env = SandboxedEnvironment(undefined=StrictUndefined)
@@ -37,24 +51,123 @@ _CACHED_DECORATOR_RE = re.compile(
)
+def _cached_label_from_decorator(decorator: ast.expr) -> str | None:
+ if not isinstance(decorator, ast.Call):
+ return None
+ func = decorator.func
+ if not (
+ isinstance(func, ast.Attribute)
+ and func.attr == "cached"
+ and isinstance(func.value, ast.Name)
+ and func.value.id == "skyvern"
+ ):
+ return None
+ for keyword in decorator.keywords:
+ if keyword.arg == "cache_key" and isinstance(keyword.value, ast.Constant):
+ value = keyword.value.value
+ if isinstance(value, str):
+ return value
+ return None
+
+
+def _cached_block_spans_regex(content: str) -> list[tuple[str, int, int]]:
+ """Best-effort fallback for malformed generated scripts.
+
+ Valid scripts use the AST path above; this fallback may stop early if
+ malformed generated code contains column-zero multiline string content.
+ """
+ spans: list[tuple[str, int, int]] = []
+ for match in _CACHED_DECORATOR_RE.finditer(content):
+ label = match.group(1)
+ end = len(content)
+ saw_function_def = False
+ for line_match in re.finditer(r"(?m)^.*(?:\n|$)", content[match.end() :]):
+ line_start = match.end() + line_match.start()
+ line = line_match.group(0)
+ stripped = line.strip()
+ if not stripped:
+ continue
+ if line[:1].isspace():
+ continue
+ if not saw_function_def and stripped.startswith(("async def ", "def ")):
+ saw_function_def = True
+ continue
+ if saw_function_def:
+ end = line_start
+ break
+ spans.append((label, match.start(), end))
+ return spans
+
+
+def _cached_block_spans(content: str) -> list[tuple[str, int, int]]:
+ try:
+ tree = ast.parse(content)
+ except SyntaxError:
+ return _cached_block_spans_regex(content)
+
+ line_offsets: list[int] = []
+ pos = 0
+ for line in content.splitlines(keepends=True):
+ line_offsets.append(pos)
+ pos += len(line)
+
+ def _offset(lineno: int, col_offset: int) -> int:
+ return line_offsets[lineno - 1] + col_offset
+
+ spans: list[tuple[str, int, int]] = []
+ for node in ast.walk(tree):
+ if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ label = next(
+ (
+ label
+ for decorator in node.decorator_list
+ if (label := _cached_label_from_decorator(decorator)) is not None
+ ),
+ None,
+ )
+ if label is None or node.end_lineno is None or node.end_col_offset is None:
+ continue
+ start_lineno = (
+ min(decorator.lineno for decorator in node.decorator_list) if node.decorator_list else node.lineno
+ )
+ start = _offset(start_lineno, 0)
+ end = _offset(node.end_lineno, node.end_col_offset)
+ spans.append((label, start, end))
+ return sorted(spans, key=lambda span: span[1])
+
+
def extract_cached_blocks_from_source(content: str) -> dict[str, str]:
"""Parse all @skyvern.cached blocks from Python source.
Returns {block_label: code} for every block found. Each value includes
- the decorator line through to the start of the next decorator (or EOF).
+ the decorator line through that function body. Valid Python source uses
+ AST spans; malformed generated scripts fall back to best-effort regex
+ spans so reviewer repair paths can still see cached blocks.
"""
- matches = list(_CACHED_DECORATOR_RE.finditer(content))
result: dict[str, str] = {}
- for idx, match in enumerate(matches):
- label = match.group(1)
- start = match.start()
- end = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)
+ for label, start, end in _cached_block_spans(content):
block_code = content[start:end].rstrip()
if block_code:
result[label] = block_code
return result
+def remove_cached_blocks_from_source(content: str) -> str:
+ """Return source outside @skyvern.cached block bodies."""
+ spans = _cached_block_spans(content)
+ if not spans:
+ return content
+
+ pieces: list[str] = []
+ cursor = 0
+ for _, start, end in spans:
+ pieces.append(content[cursor:start])
+ cursor = end
+ pieces.append(content[cursor:])
+ return "".join(pieces)
+
+
def extract_single_cached_block(content: str, block_label: str) -> str | None:
"""Extract a single @skyvern.cached block by label from Python source."""
pattern = re.compile(
@@ -854,6 +967,19 @@ async def generate_workflow_script(
status=status,
)
+ # 5) Mint-time static audit — only on published mints, not pending
+ # intermediate revisions. The pending path fires per-block during a
+ # workflow run; mint-audit there would consume LLM/cap budget on
+ # draft revisions and rewrite intermediate scripts that aren't yet
+ # the final published version.
+ if not pending:
+ await _log_mint_audit_findings(
+ python_src=python_src,
+ workflow=workflow,
+ script_revision_id=script.script_revision_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
+
# ---------------------------------------------------------------------------
# Post-processing: fix static actions inside for-loop blocks
@@ -986,6 +1112,198 @@ def _find_click_calls(text: str) -> list[tuple[int, int, str]]:
return results
+async def _log_mint_audit_findings(
+ *,
+ python_src: str,
+ workflow: Workflow,
+ script_revision_id: str,
+ workflow_run_id: str,
+) -> None:
+ """Run the v3 mint-time static audit on a freshly-generated script.
+
+ Static-pass validator runs unconditionally and emits structured log
+ lines. If findings exist, the LLM trigger is gated on the v3 cohort
+ flag, Redis cache availability, and the daily review cap. Caller is
+ ``generate_workflow_script`` AFTER ``build_file_tree`` +
+ ``create_workflow_script``, so the agent loop always operates on a
+ persisted, mapped revision.
+
+ Never raises — audit failures degrade silently to "no findings" so a
+ broken validator can't block legitimate script persistence.
+ """
+ try:
+ # See module-level comment on the deferred import.
+ from skyvern.forge.sdk.workflow.service import peek_script_review_cap_v3
+
+ prompts: list[str] = []
+ param_dict: dict[str, Any] = {}
+ try:
+ wf_def = workflow.workflow_definition
+ for block in getattr(wf_def, "blocks", []) or []:
+ block_prompt = getattr(block, "prompt", None)
+ if isinstance(block_prompt, str) and block_prompt:
+ prompts.append(block_prompt)
+ nav_goal = getattr(block, "navigation_goal", None)
+ if isinstance(nav_goal, str) and nav_goal:
+ prompts.append(nav_goal)
+ for param in getattr(wf_def, "parameters", []) or []:
+ key = getattr(param, "key", None)
+ if key:
+ param_dict[str(key)] = getattr(param, "default_value", None)
+ except Exception:
+ LOG.debug(
+ "Failed to build mint-audit envelope from workflow definition",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ exc_info=True,
+ )
+
+ envelope = build_allowed_strings_envelope(
+ user_prompt="\n".join(prompts) if prompts else None,
+ workflow_parameters=param_dict or None,
+ upstream_block_outputs=None,
+ )
+
+ # Per-block scan — each finding gets file_path=blocks/{label}.skyvern
+ # so block_label propagates through to the agent's persist_block_edit.
+ all_findings: list[SuspiciousLiteralFinding] = []
+ block_sources = extract_cached_blocks_from_source(python_src)
+ for block_label, block_source in block_sources.items():
+ all_findings.extend(
+ find_suspicious_selector_literals(
+ source=block_source,
+ allowed_strings=envelope,
+ file_path=f"blocks/{block_label}.skyvern",
+ )
+ )
+ seen = {(f.file_path, f.block_label, f.literal, f.selector) for f in all_findings}
+ for f in find_suspicious_selector_literals(
+ source=remove_cached_blocks_from_source(python_src),
+ allowed_strings=envelope,
+ file_path="main.py",
+ ):
+ key = (f.file_path, f.block_label, f.literal, f.selector)
+ if key not in seen:
+ seen.add(key)
+ all_findings.append(f)
+
+ if not all_findings:
+ LOG.info(
+ "v3_mint_audit_clean",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ envelope_size=len(envelope),
+ )
+ return
+ for f in all_findings:
+ LOG.warning(
+ "v3_mint_audit_finding",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ finding_type=f.type,
+ literal=f.literal,
+ selector=f.selector,
+ block_label=f.block_label,
+ reason=f.reason,
+ )
+ LOG.warning(
+ "v3_mint_audit_summary",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ finding_count=len(all_findings),
+ envelope_size=len(envelope),
+ )
+
+ # Cohort gate — only v3-cohort wpids get the LLM follow-up.
+ try:
+ in_v3_cohort = await is_v3_cohort(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=workflow.organization_id,
+ workflow_run_id=workflow_run_id,
+ )
+ except Exception:
+ LOG.debug(
+ "is_v3_cohort raised during mint-audit; skipping trigger",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ exc_info=True,
+ )
+ return
+ if not in_v3_cohort:
+ LOG.info(
+ "v3_mint_audit_skipped_v2_cohort",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ )
+ return
+
+ # Cache-availability gate — fail-closed under Redis outage.
+ if CacheFactory.get_cache() is None:
+ LOG.warning(
+ "v3_mint_audit_cache_unavailable",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ )
+ return
+
+ # Cap gate — read-only check; persist owns the atomic increment.
+ cap_exceeded = await peek_script_review_cap_v3(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=workflow.organization_id,
+ fail_closed=True,
+ )
+ if cap_exceeded:
+ LOG.info(
+ "v3_mint_audit_cap_skipped",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ )
+ return
+
+ # All gates passed — fire the v3 agent loop as a background task.
+ try:
+ finding_dicts = [
+ {
+ "type": f.type,
+ "literal": f.literal,
+ "selector": f.selector,
+ "reason": f.reason,
+ "file_path": f.file_path,
+ "block_label": f.block_label,
+ }
+ for f in all_findings
+ ]
+ user_prompt_for_v3 = "\n".join(prompts) if prompts else ""
+ fire_and_forget_mint_review(
+ organization_id=workflow.organization_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ user_prompt=user_prompt_for_v3,
+ findings=finding_dicts,
+ )
+ except Exception:
+ LOG.warning(
+ "v3_mint_audit_trigger_failed",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ exc_info=True,
+ )
+ except Exception:
+ LOG.warning(
+ "v3_mint_audit_failed",
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ script_revision_id=script_revision_id,
+ exc_info=True,
+ )
+
+
def _fix_static_actions_in_for_loops(code: str) -> str:
"""Detect page.click() calls inside for-loops that don't reference the loop variable.