Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
v3 reviewer 3/3: wire mint-audit + post-run dispatch + mid-run hooks (SKY-7676) (#6184)
pedrohsdb5143e85e10b6
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex 550874c3d..feb1b8aa6 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -1,9 +1,10 @@from __future__ import annotations+import asyncioimport jsonimport refrom datetime import datetime, timezone-from typing import Any, cast+from typing import Any, cast, get_argsimport structlogfrom jinja2.sandbox import SandboxedEnvironment@@ -23,6 +24,9 @@ from skyvern.forge.sdk.schemas.totp_codes import OTPTypefrom skyvern.schemas.workflows import BlockStatusfrom skyvern.services import script_servicefrom skyvern.services.otp_service import poll_otp_value+from skyvern.services.script_reviewer_v3.cohort import is_v3_cohort+from skyvern.services.script_reviewer_v3.midrun import v3_review_in_flight+from skyvern.services.script_reviewer_v3.types import FailureContext, InterceptedActionTypefrom skyvern.utils.css_selector import compute_selector_optionsfrom skyvern.utils.prompt_engine import load_prompt_with_elements, load_prompt_with_elements_trackedfrom skyvern.utils.prompt_truncation import truncate_extraction_schema@@ -49,6 +53,8 @@ jinja_sandbox_env = SandboxedEnvironment()LOG = structlog.get_logger()+_INTERCEPTED_ACTION_TYPES = frozenset(get_args(InterceptedActionType))+INPUT_GOAL = """- The intention to fill out an input: {intention}.- The overall goal that the user wants to achieve: {prompt}."""@@ -171,6 +177,7 @@ class RealSkyvernPageAi(SkyvernPageAi):failed_selector: str | None = None,block_label: str | None = None,recoverable_marker_id: int | None = None,+ v3_parent_episode_id: str | None = None,) -> str | None:"""Click an element using AI to locate it based on intention.@@ -180,7 +187,31 @@ class RealSkyvernPageAi(SkyvernPageAi):the script reviewer can fix the selector. Only set when called fromthe ai='fallback' path in skyvern_page.py.block_label: The cached block label (from SkyvernPage.current_label).+ v3_parent_episode_id: pre-existing episode id from v3 mid-run Class B+ fall-through. When set, this call updates that episode instead of+ creating a duplicate."""+ # v3 mid-run hook: only on the original call (not on internal recursion).+ # If v3 commits an in-flight fix (Class A), we short-circuit; if it+ # gives up (Class B), the episode already exists and we propagate+ # v3_parent_episode_id so the agent-fallback path updates it instead+ # of creating a duplicate row.+ if v3_parent_episode_id is None:+ ep_id, class_a = await self._maybe_run_v3_midrun(+ action_type="click",+ failed_selector=failed_selector,+ intention=intention,+ value=None,+ totp_identifier=None,+ totp_url=None,+ block_label=block_label,+ )+ if class_a:+ # v3's live_try_click already advanced the page state.+ return None+ if ep_id is not None:+ v3_parent_episode_id = ep_id+try:# Build the element tree of the current page for the promptcontext = skyvern_context.ensure_context()@@ -241,6 +272,7 @@ class RealSkyvernPageAi(SkyvernPageAi):action=action,block_label=block_label,recoverable_marker_id=recoverable_marker_id,+ v3_parent_episode_id=v3_parent_episode_id,)return selector@@ -278,9 +310,29 @@ class RealSkyvernPageAi(SkyvernPageAi):failed_selector: str | None = None,block_label: str | None = None,recoverable_marker_id: int | None = None,+ v3_parent_episode_id: str | None = None,) -> str:"""Input text into an element using AI to determine the value."""+ # v3 mid-run hook — see ai_click for the contract.+ if v3_parent_episode_id is None:+ ep_id, class_a = await self._maybe_run_v3_midrun(+ action_type="fill",+ failed_selector=failed_selector,+ intention=intention,+ value=value,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ block_label=block_label,+ )+ if class_a:+ # v3 mid-run committed the fill on the live page. The+ # caller relies on page state, not this return value; "" means+ # no selector was resolved.+ return ""+ if ep_id is not None:+ v3_parent_episode_id = ep_id+context = skyvern_context.ensure_context()value = value or ""transformed_value = value@@ -402,6 +454,7 @@ class RealSkyvernPageAi(SkyvernPageAi):action=action,block_label=block_label,recoverable_marker_id=recoverable_marker_id,+ v3_parent_episode_id=v3_parent_episode_id,)else:locator = self.page.locator(selector)@@ -723,6 +776,116 @@ class RealSkyvernPageAi(SkyvernPageAi):)return result+ async def _maybe_run_v3_midrun(+ self,+ *,+ action_type: str,+ failed_selector: str | None,+ intention: str,+ value: str | None,+ totp_identifier: str | None,+ totp_url: str | None,+ block_label: str | None,+ ) -> tuple[str | None, bool]:+ """v3 mid-run gate. Called at the top of ai_click / ai_input_text.++ Returns ``(parent_episode_id, did_class_a)``.++ - ``parent_episode_id`` is set when a v3 episode was created and v3+ ran. None means v3 didn't fire (not v3 cohort, missing context,+ or pre-conditions not met).+ - ``did_class_a`` is True when v3's agent committed an in-flight fix+ via a successful live_try_*. The caller short-circuits the AI+ fallback in that case — the workflow already advanced.++ Never raises — any internal failure logs and returns ``(None, False)``+ so the caller falls through to the existing AI fallback path.+ """+ if failed_selector is None or not failed_selector.strip():+ return None, False++ episode_id_for_cleanup: str | None = None+ organization_id_for_cleanup: str | None = None+ try:+ context = skyvern_context.current()+ if context is None or not context.workflow_permanent_id or not context.workflow_run_id:+ return None, False+ # organization_id is required for episode FK + cohort lookup.+ # Guard explicitly rather than silently coercing to "".+ if not context.organization_id:+ return None, False+ if (context.code_version or 0) < 2:+ return None, False+ # FailureContext narrows the action_type set; bail if the+ # caller passed something outside the typed set.+ if action_type not in _INTERCEPTED_ACTION_TYPES:+ return None, False+ narrowed_action_type = cast(InterceptedActionType, action_type)++ use_v3 = await is_v3_cohort(+ workflow_permanent_id=context.workflow_permanent_id,+ organization_id=context.organization_id,+ workflow_run_id=context.workflow_run_id,+ )+ if not use_v3:+ return None, False++ # Create the episode now so v3's agent can attach decisions to it.+ episode = await app.DATABASE.scripts.create_fallback_episode(+ organization_id=context.organization_id,+ workflow_permanent_id=context.workflow_permanent_id,+ workflow_run_id=context.workflow_run_id,+ block_label=block_label or self.current_label or "unknown",+ fallback_type="element",+ script_revision_id=context.script_revision_id,+ page_url=self.page.url,+ reviewer_version="v3",+ )+ episode_id_for_cleanup = episode.episode_id+ organization_id_for_cleanup = context.organization_id++ fc = FailureContext(+ failed_selector=failed_selector,+ intention=intention,+ action_type=narrowed_action_type,+ value=value,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ page=self.page,+ context=context,+ episode_id=episode.episode_id,+ )+ result = await v3_review_in_flight(fc)+ return episode.episode_id, result.decision.is_midrun_class_a()+ except asyncio.CancelledError:+ raise+ except Exception:+ LOG.warning(+ "v3 mid-run hook raised — falling through to agent fallback",+ action_type=action_type,+ block_label=block_label,+ exc_info=True,+ )+ if episode_id_for_cleanup is not None and organization_id_for_cleanup is not None:+ try:+ await app.DATABASE.scripts.delete_fallback_episode(+ episode_id=episode_id_for_cleanup,+ organization_id=organization_id_for_cleanup,+ )+ except asyncio.CancelledError:+ raise+ except Exception:+ # Double-fault: v3_review_in_flight raised AND cleanup also+ # failed. Log at error so the orphan is observable in DD —+ # post-run v3 will see reviewed=False, reviewer_version="v3"+ # with no useful agent data.+ LOG.error(+ "v3_midrun_orphan_episode_cleanup_failed",+ episode_id=episode_id_for_cleanup,+ exc_info=True,+ )+ return None, False+async def _record_element_fallback_episode(self,context: skyvern_context.SkyvernContext,@@ -732,6 +895,7 @@ class RealSkyvernPageAi(SkyvernPageAi):action: Any,block_label: str | None = None,recoverable_marker_id: int | None = None,+ v3_parent_episode_id: str | None = None,) -> None:"""Record an element-level fallback episode when ai_click/ai_input_text firesbecause a CSS selector failed or was missing. Gated on code_version >= 2.@@ -742,6 +906,10 @@ class RealSkyvernPageAi(SkyvernPageAi):`ai='proactive'` because no semantic selector existed at codegen.AI succeeded; capture the element pick so the reviewer can laterupgrade the call to `selector=, ai='fallback'`.++ ``v3_parent_episode_id`` is set on Class B fall-through from mid-run v3.+ The episode row was already created when v3 fired; we update_ it with+ the agent fallback's action data instead of creating a duplicate."""if failed_selector is None and recoverable_marker_id is None:return@@ -809,23 +977,40 @@ class RealSkyvernPageAi(SkyvernPageAi):f"Original selector: {failed_selector or '(none)'}. "f"Intention: {intention}")- await app.DATABASE.scripts.create_fallback_episode(- organization_id=context.organization_id or "",- workflow_permanent_id=context.workflow_permanent_id,- workflow_run_id=context.workflow_run_id,- block_label=block_label or self.current_label or "unknown",- fallback_type="element",- script_revision_id=context.script_revision_id,- error_message=error_msg,- page_url=self.page.url,- agent_actions=action_data,- )- LOG.info(- "Recorded element fallback episode for selector failure",- block_label=block_label or self.current_label,- action_type=action_type,- failed_selector=failed_selector,- )+ if v3_parent_episode_id:+ # v3 mid-run Class B fall-through: episode already exists; update+ # it with the agent's action_data + error_message instead of+ # creating a duplicate row.+ await app.DATABASE.scripts.update_fallback_episode(+ episode_id=v3_parent_episode_id,+ organization_id=context.organization_id or "",+ agent_actions=action_data,+ error_message=error_msg,+ )+ LOG.info(+ "Updated v3 mid-run episode with agent fallback action data",+ episode_id=v3_parent_episode_id,+ block_label=block_label or self.current_label,+ action_type=action_type,+ )+ else:+ await app.DATABASE.scripts.create_fallback_episode(+ organization_id=context.organization_id or "",+ workflow_permanent_id=context.workflow_permanent_id,+ workflow_run_id=context.workflow_run_id,+ block_label=block_label or self.current_label or "unknown",+ fallback_type="element",+ script_revision_id=context.script_revision_id,+ error_message=error_msg,+ page_url=self.page.url,+ agent_actions=action_data,+ )+ LOG.info(+ "Recorded element fallback episode for selector failure",+ block_label=block_label or self.current_label,+ action_type=action_type,+ failed_selector=failed_selector,+ )except Exception:LOG.warning("Failed to record element fallback episode for selector failure", exc_info=True)diff --git a/skyvern/core/script_generations/skyvern_page_ai.py b/skyvern/core/script_generations/skyvern_page_ai.pyindex c91ab9146..cb6bd85f6 100644--- a/skyvern/core/script_generations/skyvern_page_ai.py+++ b/skyvern/core/script_generations/skyvern_page_ai.py@@ -23,8 +23,14 @@ class SkyvernPageAi(Protocol):failed_selector: str | None = None,block_label: str | None = None,recoverable_marker_id: int | None = None,+ v3_parent_episode_id: str | None = None,) -> str | None:- """Click an element using AI to locate it based on intention."""+ """Click an element using AI to locate it based on intention.++ ``v3_parent_episode_id`` is set on the recursive fall-through call when+ the v3 mid-run agent gave up (Class B). It tells the implementation to+ update the existing fallback episode rather than create a duplicate.+ """...async def ai_input_text(@@ -39,8 +45,12 @@ class SkyvernPageAi(Protocol):failed_selector: str | None = None,block_label: str | None = None,recoverable_marker_id: int | None = None,+ v3_parent_episode_id: str | None = None,) -> str:- """Input text into an element using AI to determine the value."""+ """Input text into an element using AI to determine the value.++ See ``ai_click`` for the ``v3_parent_episode_id`` contract.+ """...async def ai_upload_file(diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex a4864ec9b..7cb4d30c5 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -128,6 +128,8 @@ from skyvern.schemas.workflows import (WorkflowStatus,)from skyvern.services import script_service, workflow_script_service+from skyvern.services.script_reviewer_v3.cohort import is_v3_cohort+from skyvern.services.script_reviewer_v3.postrun import v3_review_post_runfrom skyvern.services.webhook_delivery import PreparedWorkflowWebhook, deliver_webhook_with_retriesfrom skyvern.utils.css_selector import build_action_summaries_with_timing # shared with script_servicefrom skyvern.utils.secret_headers import merge_masked_headers@@ -6482,21 +6484,55 @@ class WorkflowService:)return- # Cap ALL script reviews (fallback + failure) per wpid per day to prevent- # runaway revision churn when the same issue repeats every run.- cap_exceeded = await self._check_script_review_cap(+ # Resolve cohort FIRST so the cap check + post-dispatch increment+ # can route to the cohort's own counter.+ use_v3 = await is_v3_cohort(workflow_permanent_id=workflow.workflow_permanent_id,organization_id=workflow.organization_id,+ workflow_run_id=workflow_run.workflow_run_id,)++ # Cap ALL script reviews (fallback + failure) per wpid per day to prevent+ # runaway revision churn when the same issue repeats every run.+ if use_v3:+ cap_exceeded = await self._check_script_review_cap_v3(+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization_id=workflow.organization_id,+ )+ else:+ cap_exceeded = await self._check_script_review_cap(+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization_id=workflow.organization_id,+ )if cap_exceeded:LOG.info("Skipping script review — daily cap exceeded for wpid",workflow_permanent_id=workflow.workflow_permanent_id,workflow_run_id=workflow_run.workflow_run_id,pre_finally_status=pre_finally_status,+ cohort="v3" if use_v3 else "v2",)return+ # Fast-path skip: v3 cohort + zero fallback episodes = nothing+ # to review. Saves an LLM round-trip and a cap-counter increment+ # on cold-mint runs, which produce no episodes by construction.+ # v2 keeps its existing behavior (reviews whether or not episodes+ # exist) — only the v3 path adds this gate.+ if use_v3:+ v3_episodes = await app.DATABASE.scripts.get_all_episodes_by_workflow_run_id(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=workflow.organization_id,+ )+ if not v3_episodes:+ LOG.info(+ "v3_postrun_skipped_no_episodes",+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ script_revision_id=script_revision_id,+ )+ return+# Non-blocking lock per script familycache = CacheFactory.get_cache()lock = None@@ -6507,11 +6543,23 @@ class WorkflowService:except AttributeError:LOG.debug("Cache doesn't support locking for script reviewer")+ async def _run_reviewer_dispatch() -> Any | None:+ if use_v3:+ return await v3_review_post_run(+ organization_id=workflow.organization_id,+ workflow_run=workflow_run,+ workflow_permanent_id=workflow.workflow_permanent_id,+ script_revision_id=script_revision_id,+ )+ await self._run_reviewer_locked(workflow, workflow_run, script_revision_id, script_id)+ return None+review_ran = False+ review_result: Any | None = Noneif lock is not None:try:async with lock:- await self._run_reviewer_locked(workflow, workflow_run, script_revision_id, script_id)+ review_result = await _run_reviewer_dispatch()review_ran = Trueexcept LockError:LOG.info(@@ -6521,15 +6569,20 @@ class WorkflowService:)else:# No Redis/cache available - proceed without lock (graceful degradation for OSS)- await self._run_reviewer_locked(workflow, workflow_run, script_revision_id, script_id)+ review_result = await _run_reviewer_dispatch()review_ran = True- # Increment the review counter ONLY after a review actually ran.- # Skipped reviews (e.g., LockError) should not consume cap budget.if review_ran:- await self._increment_script_review_counter(- workflow_permanent_id=workflow.workflow_permanent_id,- )+ if use_v3:+ v3_persist_cap_consumed = bool(getattr(review_result, "v3_persist_cap_consumed", False))+ if not v3_persist_cap_consumed:+ await self._increment_script_review_counter_v3(+ workflow_permanent_id=workflow.workflow_permanent_id,+ )+ else:+ await self._increment_script_review_counter(+ workflow_permanent_id=workflow.workflow_permanent_id,+ )except Exception:LOG.warning("Failed to trigger script reviewer",@@ -6548,11 +6601,10 @@ class WorkflowService:"""Build the Redis key for the daily script-review counter (v3 cohort).Separate from the v2 key so cohort comparisons stay clean and v3- traffic doesn't contaminate v2's counter. Thin wrapper around the- module-level ``v3_script_review_cap_key`` so non-WorkflowService- callers (e.g. the mint-audit trigger in workflow_script_service.py)- can reach the same key construction without holding an instance- reference.+ traffic doesn't contaminate v2's counter. Thin wrapper around the module-level+ ``v3_script_review_cap_key`` so non-WorkflowService callers (e.g.+ the mint-audit trigger in workflow_script_service.py) can reach+ the same key construction without holding an instance reference."""return v3_script_review_cap_key(workflow_permanent_id)@@ -6647,9 +6699,7 @@ class WorkflowService:async def _increment_script_review_counter_v3(self, workflow_permanent_id: str) -> None:"""v3 cohort mirror of ``_increment_script_review_counter`` writing v3 key.- Used post-dispatch in ``_trigger_script_reviewer`` after a successful- v3 cohort review. Separating from v2 lets us compare cohort counters- cleanly.+ Called from ``_trigger_script_reviewer`` after a successful v3 review_ran."""await self._increment_review_counter_for_key(self._v3_script_review_cap_key(workflow_permanent_id))@@ -6694,6 +6744,7 @@ class WorkflowService:if cache is None:return Nonecap = await self._get_script_review_cap(organization_id)+ # v3-specific key — separate from v2's shared key so cohort comparisons stay clean.cap_key = self._v3_script_review_cap_key(workflow_permanent_id)lock_name = f"v3_cap:{workflow_permanent_id}"lock = cache.get_lock(lock_name, blocking_timeout=2, timeout=5)@@ -6887,6 +6938,7 @@ class WorkflowService:episode_id=episode.episode_id,organization_id=workflow.organization_id,reviewer_output=None,+ reviewer_version="v2",)return@@ -6933,6 +6985,7 @@ class WorkflowService:organization_id=workflow.organization_id,reviewer_output=str(updated_blocks) if updated_blocks else None,new_script_revision_id=new_script.script_revision_id if new_script else None,+ reviewer_version="v2",)except Exception:diff --git a/skyvern/library/skyvern_browser_page_ai.py b/skyvern/library/skyvern_browser_page_ai.pyindex 1244a3b23..a9d39a3fe 100644--- a/skyvern/library/skyvern_browser_page_ai.py+++ b/skyvern/library/skyvern_browser_page_ai.py@@ -49,6 +49,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):failed_selector: str | None = None, # noqa: ARG002 — accepted for Protocol compat, no episode recording in library pathblock_label: str | None = None, # noqa: ARG002recoverable_marker_id: int | None = None, # noqa: ARG002 — Protocol compat+ v3_parent_episode_id: str | None = None, # noqa: ARG002 — Protocol compat; v3 mid-run is server-side only) -> str | None:"""Click an element using AI via API call.@@ -87,6 +88,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):failed_selector: str | None = None, # noqa: ARG002 — Protocol compat, see ai_click docstringblock_label: str | None = None, # noqa: ARG002recoverable_marker_id: int | None = None, # noqa: ARG002 — Protocol compat+ v3_parent_episode_id: str | None = None, # noqa: ARG002 — Protocol compat; v3 mid-run is server-side only) -> str:"""Input text into an element using AI via API call."""diff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.pyindex 2fd82a7ac..4c2108d07 100644--- a/skyvern/services/script_reviewer.py+++ b/skyvern/services/script_reviewer.py@@ -600,6 +600,7 @@ class ScriptReviewer:episode_id=episode.episode_id,organization_id=organization_id,reviewer_output="TRIAGE: not_code_fixable — skipped",+ reviewer_version="v2",)LOG.info("ScriptReviewer: skipping non-code-fixable episode",diff --git a/skyvern/services/script_reviewer_v3/budget.py b/skyvern/services/script_reviewer_v3/budget.pyindex e3a8d5c0e..6dea96583 100644--- a/skyvern/services/script_reviewer_v3/budget.py+++ b/skyvern/services/script_reviewer_v3/budget.py@@ -137,8 +137,8 @@ class RunBudget:pre-charged at the per-review ceiling). Returns ``None`` if either capwould be exceeded — caller should skip v3 and fall through to agent.- Reservation (not actual usage) is held until :meth:`InvocationHandle.finalize_cost`- reconciles with the true cost spent during the review.+ Reservation (not actual usage) is held until :meth:`reconcile_invocation`+ replaces the reservation with the true cost spent during the review."""async with self._lock:if self._invocations_used >= self.max_invocations_per_run:@@ -150,6 +150,21 @@ class RunBudget:self._cost_reserved_usd = new_reservationreturn InvocationHandle(budget=self, reserved_usd=self.per_review_cost_ceiling_usd)+ async def reconcile_invocation(self, *, reserved_usd: float, actual_usd: float) -> None:+ """Replace a pre-charged invocation reservation with actual spend."""+ async with self._lock:+ self._cost_reserved_usd = max(+ 0.0,+ self._cost_reserved_usd - reserved_usd + actual_usd,+ )+ if actual_usd > reserved_usd:+ LOG.warning(+ "v3 review exceeded cost reservation",+ reserved_usd=reserved_usd,+ actual_usd=actual_usd,+ overshoot_usd=actual_usd - reserved_usd,+ )+def to_metrics(self) -> dict[str, float | int]:return {"invocations_used": self._invocations_used,@@ -175,18 +190,7 @@ class InvocationHandle:reserved_usd: floatasync def finalize_cost(self, actual_usd: float) -> None:- async with self.budget._lock:- # Adjust reserved -> actual:- # - remove the reserved amount, add back the actual amount- # - clamp at 0 to handle transient arithmetic edge cases- self.budget._cost_reserved_usd = max(- 0.0,- self.budget._cost_reserved_usd - self.reserved_usd + actual_usd,- )- if actual_usd > self.reserved_usd:- LOG.warning(- "v3 review exceeded cost reservation",- reserved_usd=self.reserved_usd,- actual_usd=actual_usd,- overshoot_usd=actual_usd - self.reserved_usd,- )+ await self.budget.reconcile_invocation(+ reserved_usd=self.reserved_usd,+ actual_usd=actual_usd,+ )diff --git a/skyvern/services/script_reviewer_v3/midrun.py b/skyvern/services/script_reviewer_v3/midrun.pyindex ceeae0b17..8b01bdad9 100644--- a/skyvern/services/script_reviewer_v3/midrun.py+++ b/skyvern/services/script_reviewer_v3/midrun.py@@ -150,79 +150,87 @@ async def v3_review_in_flight(timeline=[],)- registry = build_registry()- llm_caller = LLMCaller(llm_key=V3_REVIEWER_MODEL)-- user_prompt = build_midrun_user_prompt(- episode_id=fc.episode_id,- fc_summary=_summarize_failure_context(fc),- )+ # If setup fails after this point, the reserved invocation is finalized+ # at $0.00 but the invocation slot remains consumed to prevent retry storms.+ try:+ registry = build_registry()+ llm_caller = LLMCaller(llm_key=V3_REVIEWER_MODEL)- LOG.info(- "v3_midrun_started",- episode_id=fc.episode_id,- workflow_run_id=getattr(fc.context, "workflow_run_id", None),- action_type=fc.action_type,- failed_selector=fc.failed_selector,- budget_max_cycles=budget.max_cycles,- budget_max_cost_usd=budget.max_cost_usd,- )+ user_prompt = build_midrun_user_prompt(+ episode_id=fc.episode_id,+ fc_summary=_summarize_failure_context(fc),+ )- try:- result = await asyncio.wait_for(- run_agent_loop(- llm_caller=llm_caller,- system_prompt=MIDRUN_SYSTEM_PROMPT,- user_prompt=user_prompt,- registry=registry,- agent_kind="midrun",- context=fc,- budget=budget,- organization_id=org_id,- prompt_name=prompt_name,- invocation_handle=invocation_handle,- ),- timeout=wall_clock_seconds,+ LOG.info(+ "v3_midrun_started",+ episode_id=fc.episode_id,+ workflow_run_id=getattr(fc.context, "workflow_run_id", None),+ action_type=fc.action_type,+ failed_selector=fc.failed_selector,+ budget_max_cycles=budget.max_cycles,+ budget_max_cost_usd=budget.max_cost_usd,)- except asyncio.TimeoutError:- LOG.warning(- "v3_midrun_wall_clock_exceeded",++ try:+ result = await asyncio.wait_for(+ run_agent_loop(+ llm_caller=llm_caller,+ system_prompt=MIDRUN_SYSTEM_PROMPT,+ user_prompt=user_prompt,+ registry=registry,+ agent_kind="midrun",+ context=fc,+ budget=budget,+ organization_id=org_id,+ prompt_name=prompt_name,+ ),+ timeout=wall_clock_seconds,+ )+ except asyncio.TimeoutError:+ LOG.warning(+ "v3_midrun_wall_clock_exceeded",+ episode_id=fc.episode_id,+ wall_clock_seconds=wall_clock_seconds,+ cycles_completed=budget.cycles_used,+ )+ decision = Decision.loop_error("wall_clock_exceeded")+ await _persist_midrun_terminal(organization_id=org_id, fc=fc, decision=decision)+ return V3MidRunResult(decision=decision, budget_used=budget, timeline=[])++ final_decision = result.terminal_decision or Decision.loop_error("no_terminal")+ LOG.info(+ "v3_midrun_complete",episode_id=fc.episode_id,- wall_clock_seconds=wall_clock_seconds,- cycles_completed=budget.cycles_used,+ workflow_run_id=getattr(fc.context, "workflow_run_id", None),+ decision_type=final_decision.type,+ decision_reason=final_decision.reason,+ investigation_summary=(final_decision.investigation_summary or "")[:300],+ new_script_revision_id=final_decision.new_script_revision_id,+ class_a=final_decision.is_midrun_class_a(),+ class_b=final_decision.is_midrun_class_b(),+ cycles_used=result.budget.cycles_used,+ tokens_used=result.budget.tokens_used,+ cost_usd_used=round(result.budget.cost_usd_used, 6),+ elapsed_seconds=round(result.budget.elapsed_seconds, 2),)- decision = Decision.loop_error("wall_clock_exceeded")- await _persist_midrun_terminal(organization_id=org_id, fc=fc, decision=decision)- return V3MidRunResult(decision=decision, budget_used=budget, timeline=[])-- final_decision = result.terminal_decision or Decision.loop_error("no_terminal")- LOG.info(- "v3_midrun_complete",- episode_id=fc.episode_id,- workflow_run_id=getattr(fc.context, "workflow_run_id", None),- decision_type=final_decision.type,- decision_reason=final_decision.reason,- investigation_summary=(final_decision.investigation_summary or "")[:300],- new_script_revision_id=final_decision.new_script_revision_id,- class_a=final_decision.is_midrun_class_a(),- class_b=final_decision.is_midrun_class_b(),- cycles_used=result.budget.cycles_used,- tokens_used=result.budget.tokens_used,- cost_usd_used=round(result.budget.cost_usd_used, 6),- elapsed_seconds=round(result.budget.elapsed_seconds, 2),- )- await _persist_midrun_terminal(- organization_id=org_id,- fc=fc,- decision=final_decision,- )+ await _persist_midrun_terminal(+ organization_id=org_id,+ fc=fc,+ decision=final_decision,+ )- return V3MidRunResult(- decision=final_decision,- budget_used=result.budget,- timeline=result.timeline,- )+ return V3MidRunResult(+ decision=final_decision,+ budget_used=result.budget,+ timeline=result.timeline,+ )+ finally:+ if invocation_handle is not None:+ try:+ await invocation_handle.finalize_cost(budget.cost_usd_used)+ except Exception: # pragma: no cover+ LOG.warning("v3_midrun_invocation_handle_finalize_failed", exc_info=True)__all__ = ["v3_review_in_flight"]diff --git a/skyvern/services/script_reviewer_v3/postrun.py b/skyvern/services/script_reviewer_v3/postrun.pyindex 923911fdd..83c9866bb 100644--- a/skyvern/services/script_reviewer_v3/postrun.py+++ b/skyvern/services/script_reviewer_v3/postrun.py@@ -330,6 +330,8 @@ async def v3_review_post_run(budget_used=prc.budget,timeline=[],per_episode_persist_results=persist_results,+ v3_persist_cap_consumed=prc.v3_persist_cap_consumed,+ v3_persist_cap_acquisitions=prc.v3_persist_cap_acquisitions,)# Single deterministic pass to apply per-episode persists. Persists run@@ -369,6 +371,8 @@ async def v3_review_post_run(budget_used=result.budget,timeline=result.timeline,per_episode_persist_results=persist_results,+ v3_persist_cap_consumed=prc.v3_persist_cap_consumed,+ v3_persist_cap_acquisitions=prc.v3_persist_cap_acquisitions,)diff --git a/skyvern/services/script_reviewer_v3/skills/persist.py b/skyvern/services/script_reviewer_v3/skills/persist.pyindex 8c35447e2..c73b405f0 100644--- a/skyvern/services/script_reviewer_v3/skills/persist.py+++ b/skyvern/services/script_reviewer_v3/skills/persist.py@@ -31,6 +31,7 @@ from skyvern.forge import appfrom skyvern.forge.sdk.cache.factory import CacheFactoryfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.services.script_reviewer_v3.skills.base import Skill, SkillError, SkillResult+from skyvern.services.script_reviewer_v3.types import PostRunContextLOG = structlog.get_logger()@@ -114,6 +115,15 @@ def _update_context_revision(agent_context.script_id = new_script_id+def _record_v3_persist_cap_acquisition(agent_context: Any) -> None:+ """Mark contexts that already consumed the v3 persist cap."""+ if not isinstance(agent_context, PostRunContext):+ return+ agent_context.v3_persist_cap_consumed = True+ current_count = getattr(agent_context, "v3_persist_cap_acquisitions", 0) or 0+ agent_context.v3_persist_cap_acquisitions = int(current_count) + 1++async def _enforce_v3_cap(workflow_permanent_id: str,organization_id: str | None = None,@@ -166,6 +176,7 @@ async def _handler_persist_block_edit(args: dict[str, Any], context: Any) -> Ski"hint": "Per-wpid daily v3 persist cap reached. No edit was applied.",})+ _record_v3_persist_cap_acquisition(context)# Per-script Redis lock to serialize concurrent persists. v3_persist scope# is distinct from v2's script_reviewer:{} lock so the two don't deadlock.@@ -256,6 +267,7 @@ async def _handler_persist_script_rewrite(args: dict[str, Any], context: Any) ->"reason": "daily_cap_exceeded",})+ _record_v3_persist_cap_acquisition(context)cache = CacheFactory.get_cache()base_script_id = persist_ctx["base_script"].script_iddiff --git a/skyvern/services/script_reviewer_v3/types.py b/skyvern/services/script_reviewer_v3/types.pyindex a840e35c8..8e48b15e7 100644--- a/skyvern/services/script_reviewer_v3/types.py+++ b/skyvern/services/script_reviewer_v3/types.py@@ -72,6 +72,8 @@ class PostRunContext:# demotion review during the post-run pass.mid_run_class_a_episode_ids: list[str]budget: Budget+ v3_persist_cap_consumed: bool = False+ v3_persist_cap_acquisitions: int = 0@dataclasses.dataclass@@ -149,6 +151,10 @@ class V3PostRunResult:# Structured persist outcomes from the post-loop deterministic pass.# Populated even when persists fail so the caller sees what was attempted.per_episode_persist_results: dict[str, PerEpisodePersistResult] = dataclasses.field(default_factory=dict)+ # True when a post-run persist skill acquired the v3 daily cap before+ # attempting a script write. This is distinct from per-episode DB persists.+ v3_persist_cap_consumed: bool = False+ v3_persist_cap_acquisitions: int = 0__all__ = [diff --git a/skyvern/services/workflow_script_service.py b/skyvern/services/workflow_script_service.pyindex 37d25f5bc..dea9c1165 100644--- a/skyvern/services/workflow_script_service.py+++ b/skyvern/services/workflow_script_service.py@@ -1,3 +1,4 @@+import astimport base64import hashlibimport re@@ -18,14 +19,27 @@ from skyvern.core.script_generations.generate_script import ()from skyvern.core.script_generations.transform_workflow_run import transform_workflow_run_to_code_gen_inputfrom skyvern.forge import app+from skyvern.forge.sdk.cache.factory import CacheFactoryfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.workflow.models.block import get_all_blocksfrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, is_adaptive_cachingfrom skyvern.schemas.scripts import FileEncoding, Script, ScriptFileCreate, ScriptStatusfrom skyvern.schemas.workflows import BlockTypefrom skyvern.services import script_service+from skyvern.services.script_reviewer_v3.cohort import is_v3_cohort+from skyvern.services.script_reviewer_v3.mint_audit import (+ SuspiciousLiteralFinding,+ build_allowed_strings_envelope,+ find_suspicious_selector_literals,+)+from skyvern.services.script_reviewer_v3.mint_review import fire_and_forget_mint_reviewfrom skyvern.utils.url_validators import prepend_scheme_and_validate_url+# Deferred to call-site because ``skyvern.forge.sdk.workflow.service``+# imports this module, creating a circular import at module-load time.+# Hoisting upstream would require splitting the cap helpers out of+# service.py into their own module (future cleanup).+LOG = structlog.get_logger()jinja_sandbox_env = SandboxedEnvironment()strict_jinja_sandbox_env = SandboxedEnvironment(undefined=StrictUndefined)@@ -37,24 +51,123 @@ _CACHED_DECORATOR_RE = re.compile()+def _cached_label_from_decorator(decorator: ast.expr) -> str | None:+ if not isinstance(decorator, ast.Call):+ return None+ func = decorator.func+ if not (+ isinstance(func, ast.Attribute)+ and func.attr == "cached"+ and isinstance(func.value, ast.Name)+ and func.value.id == "skyvern"+ ):+ return None+ for keyword in decorator.keywords:+ if keyword.arg == "cache_key" and isinstance(keyword.value, ast.Constant):+ value = keyword.value.value+ if isinstance(value, str):+ return value+ return None+++def _cached_block_spans_regex(content: str) -> list[tuple[str, int, int]]:+ """Best-effort fallback for malformed generated scripts.++ Valid scripts use the AST path above; this fallback may stop early if+ malformed generated code contains column-zero multiline string content.+ """+ spans: list[tuple[str, int, int]] = []+ for match in _CACHED_DECORATOR_RE.finditer(content):+ label = match.group(1)+ end = len(content)+ saw_function_def = False+ for line_match in re.finditer(r"(?m)^.*(?:\n|$)", content[match.end() :]):+ line_start = match.end() + line_match.start()+ line = line_match.group(0)+ stripped = line.strip()+ if not stripped:+ continue+ if line[:1].isspace():+ continue+ if not saw_function_def and stripped.startswith(("async def ", "def ")):+ saw_function_def = True+ continue+ if saw_function_def:+ end = line_start+ break+ spans.append((label, match.start(), end))+ return spans+++def _cached_block_spans(content: str) -> list[tuple[str, int, int]]:+ try:+ tree = ast.parse(content)+ except SyntaxError:+ return _cached_block_spans_regex(content)++ line_offsets: list[int] = []+ pos = 0+ for line in content.splitlines(keepends=True):+ line_offsets.append(pos)+ pos += len(line)++ def _offset(lineno: int, col_offset: int) -> int:+ return line_offsets[lineno - 1] + col_offset++ spans: list[tuple[str, int, int]] = []+ for node in ast.walk(tree):+ if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):+ continue+ label = next(+ (+ label+ for decorator in node.decorator_list+ if (label := _cached_label_from_decorator(decorator)) is not None+ ),+ None,+ )+ if label is None or node.end_lineno is None or node.end_col_offset is None:+ continue+ start_lineno = (+ min(decorator.lineno for decorator in node.decorator_list) if node.decorator_list else node.lineno+ )+ start = _offset(start_lineno, 0)+ end = _offset(node.end_lineno, node.end_col_offset)+ spans.append((label, start, end))+ return sorted(spans, key=lambda span: span[1])++def extract_cached_blocks_from_source(content: str) -> dict[str, str]:"""Parse all @skyvern.cached blocks from Python source.Returns {block_label: code} for every block found. Each value includes- the decorator line through to the start of the next decorator (or EOF).+ the decorator line through that function body. Valid Python source uses+ AST spans; malformed generated scripts fall back to best-effort regex+ spans so reviewer repair paths can still see cached blocks."""- matches = list(_CACHED_DECORATOR_RE.finditer(content))result: dict[str, str] = {}- for idx, match in enumerate(matches):- label = match.group(1)- start = match.start()- end = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)+ for label, start, end in _cached_block_spans(content):block_code = content[start:end].rstrip()if block_code:result[label] = block_codereturn result+def remove_cached_blocks_from_source(content: str) -> str:+ """Return source outside @skyvern.cached block bodies."""+ spans = _cached_block_spans(content)+ if not spans:+ return content++ pieces: list[str] = []+ cursor = 0+ for _, start, end in spans:+ pieces.append(content[cursor:start])+ cursor = end+ pieces.append(content[cursor:])+ return "".join(pieces)++def extract_single_cached_block(content: str, block_label: str) -> str | None:"""Extract a single @skyvern.cached block by label from Python source."""pattern = re.compile(@@ -854,6 +967,19 @@ async def generate_workflow_script(status=status,)+ # 5) Mint-time static audit — only on published mints, not pending+ # intermediate revisions. The pending path fires per-block during a+ # workflow run; mint-audit there would consume LLM/cap budget on+ # draft revisions and rewrite intermediate scripts that aren't yet+ # the final published version.+ if not pending:+ await _log_mint_audit_findings(+ python_src=python_src,+ workflow=workflow,+ script_revision_id=script.script_revision_id,+ workflow_run_id=workflow_run.workflow_run_id,+ )+# ---------------------------------------------------------------------------# Post-processing: fix static actions inside for-loop blocks@@ -986,6 +1112,198 @@ def _find_click_calls(text: str) -> list[tuple[int, int, str]]:return results+async def _log_mint_audit_findings(+ *,+ python_src: str,+ workflow: Workflow,+ script_revision_id: str,+ workflow_run_id: str,+) -> None:+ """Run the v3 mint-time static audit on a freshly-generated script.++ Static-pass validator runs unconditionally and emits structured log+ lines. If findings exist, the LLM trigger is gated on the v3 cohort+ flag, Redis cache availability, and the daily review cap. Caller is+ ``generate_workflow_script`` AFTER ``build_file_tree`` ++ ``create_workflow_script``, so the agent loop always operates on a+ persisted, mapped revision.++ Never raises — audit failures degrade silently to "no findings" so a+ broken validator can't block legitimate script persistence.+ """+ try:+ # See module-level comment on the deferred import.+ from skyvern.forge.sdk.workflow.service import peek_script_review_cap_v3++ prompts: list[str] = []+ param_dict: dict[str, Any] = {}+ try:+ wf_def = workflow.workflow_definition+ for block in getattr(wf_def, "blocks", []) or []:+ block_prompt = getattr(block, "prompt", None)+ if isinstance(block_prompt, str) and block_prompt:+ prompts.append(block_prompt)+ nav_goal = getattr(block, "navigation_goal", None)+ if isinstance(nav_goal, str) and nav_goal:+ prompts.append(nav_goal)+ for param in getattr(wf_def, "parameters", []) or []:+ key = getattr(param, "key", None)+ if key:+ param_dict[str(key)] = getattr(param, "default_value", None)+ except Exception:+ LOG.debug(+ "Failed to build mint-audit envelope from workflow definition",+ workflow_permanent_id=workflow.workflow_permanent_id,+ exc_info=True,+ )++ envelope = build_allowed_strings_envelope(+ user_prompt="\n".join(prompts) if prompts else None,+ workflow_parameters=param_dict or None,+ upstream_block_outputs=None,+ )++ # Per-block scan — each finding gets file_path=blocks/{label}.skyvern+ # so block_label propagates through to the agent's persist_block_edit.+ all_findings: list[SuspiciousLiteralFinding] = []+ block_sources = extract_cached_blocks_from_source(python_src)+ for block_label, block_source in block_sources.items():+ all_findings.extend(+ find_suspicious_selector_literals(+ source=block_source,+ allowed_strings=envelope,+ file_path=f"blocks/{block_label}.skyvern",+ )+ )+ seen = {(f.file_path, f.block_label, f.literal, f.selector) for f in all_findings}+ for f in find_suspicious_selector_literals(+ source=remove_cached_blocks_from_source(python_src),+ allowed_strings=envelope,+ file_path="main.py",+ ):+ key = (f.file_path, f.block_label, f.literal, f.selector)+ if key not in seen:+ seen.add(key)+ all_findings.append(f)++ if not all_findings:+ LOG.info(+ "v3_mint_audit_clean",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ envelope_size=len(envelope),+ )+ return+ for f in all_findings:+ LOG.warning(+ "v3_mint_audit_finding",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ finding_type=f.type,+ literal=f.literal,+ selector=f.selector,+ block_label=f.block_label,+ reason=f.reason,+ )+ LOG.warning(+ "v3_mint_audit_summary",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ finding_count=len(all_findings),+ envelope_size=len(envelope),+ )++ # Cohort gate — only v3-cohort wpids get the LLM follow-up.+ try:+ in_v3_cohort = await is_v3_cohort(+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization_id=workflow.organization_id,+ workflow_run_id=workflow_run_id,+ )+ except Exception:+ LOG.debug(+ "is_v3_cohort raised during mint-audit; skipping trigger",+ workflow_permanent_id=workflow.workflow_permanent_id,+ exc_info=True,+ )+ return+ if not in_v3_cohort:+ LOG.info(+ "v3_mint_audit_skipped_v2_cohort",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ )+ return++ # Cache-availability gate — fail-closed under Redis outage.+ if CacheFactory.get_cache() is None:+ LOG.warning(+ "v3_mint_audit_cache_unavailable",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ )+ return++ # Cap gate — read-only check; persist owns the atomic increment.+ cap_exceeded = await peek_script_review_cap_v3(+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization_id=workflow.organization_id,+ fail_closed=True,+ )+ if cap_exceeded:+ LOG.info(+ "v3_mint_audit_cap_skipped",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ )+ return++ # All gates passed — fire the v3 agent loop as a background task.+ try:+ finding_dicts = [+ {+ "type": f.type,+ "literal": f.literal,+ "selector": f.selector,+ "reason": f.reason,+ "file_path": f.file_path,+ "block_label": f.block_label,+ }+ for f in all_findings+ ]+ user_prompt_for_v3 = "\n".join(prompts) if prompts else ""+ fire_and_forget_mint_review(+ organization_id=workflow.organization_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ user_prompt=user_prompt_for_v3,+ findings=finding_dicts,+ )+ except Exception:+ LOG.warning(+ "v3_mint_audit_trigger_failed",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ exc_info=True,+ )+ except Exception:+ LOG.warning(+ "v3_mint_audit_failed",+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ script_revision_id=script_revision_id,+ exc_info=True,+ )++def _fix_static_actions_in_for_loops(code: str) -> str:"""Detect page.click() calls inside for-loops that don't reference the loop variable.