Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9453 fix copilot reword-turn handling: test run, labels, goal wrap (#5791)
Andrew Neilson048dca71d112
diff --git a/alembic/versions/2026_05_04_2320-b19685399f97_credits_used_cached_credits_used.py b/alembic/versions/2026_05_04_2320-b19685399f97_credits_used_cached_credits_used.pynew file mode 100644index 000000000..e137ea867--- /dev/null+++ b/alembic/versions/2026_05_04_2320-b19685399f97_credits_used_cached_credits_used.py@@ -0,0 +1,33 @@+"""credits_used & cached_credits_used++Revision ID: b19685399f97+Revises: 696a7bba1ce6+Create Date: 2026-05-04 23:20:09.907428+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "b19685399f97"+down_revision: Union[str, None] = "696a7bba1ce6"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("workflow_runs", sa.Column("credits_used", sa.Integer(), server_default="0", nullable=True))+ op.add_column("workflow_runs", sa.Column("cached_credits_used", sa.Integer(), server_default="0", nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("workflow_runs", "cached_credits_used")+ op.drop_column("workflow_runs", "credits_used")+ # ### end Alembic commands ###diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2index 65d2c6864..828923893 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2@@ -105,6 +105,7 @@ When the user corrects, amends, or updates a previous request (e.g., "I meant X1. Update the workflow definition to reflect the correction — call update_workflow with the modified YAML.2. Do NOT just navigate the browser to the new URL. The user wants the WORKFLOW itself changed.3. After updating, test the corrected workflow with run_blocks_and_collect_debug.+4. Rename affected block labels and block titles whenever the correction changes the subject, URL, target action, or extracted data. Labels become output keys; stale labels mislead the agent's own future tool calls and Jinja templates that reference them. When you rename a label, update every related `next_block_label`, `block_labels` argument, and Jinja block reference.DIAGNOSING OUTPUT PROBLEMS:When the user reports a problem with workflow output (duplicates, missing data, wrong values, errors):diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex cefa0611c..b5e5d70c8 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -398,7 +398,9 @@ def _translate_to_agent_result(from skyvern.forge.sdk.copilot.tools import (_banned_block_reject_message,_detect_new_banned_blocks,+ _detect_stale_block_metadata,_record_banned_block_reject_span,+ _stale_block_metadata_message,)banned_items = _detect_new_banned_blocks(workflow_yaml, ctx.last_workflow_yaml)@@ -406,6 +408,11 @@ def _translate_to_agent_result(_record_banned_block_reject_span("replace_workflow_inline", banned_items)user_response = f"{user_response}\n\n(Note: {_banned_block_reject_message(banned_items)})"workflow_yaml = ""+ stale_metadata = _detect_stale_block_metadata(workflow_yaml, ctx.last_workflow_yaml or ctx.workflow_yaml)+ if stale_metadata:+ user_response = f"{user_response}\n\n(Note: {_stale_block_metadata_message(stale_metadata)})"+ ctx.last_test_ok = None+ workflow_yaml = ""if workflow_yaml:if ctx.user_message:workflow_yaml = wrap_block_goals(workflow_yaml, ctx.user_message)diff --git a/skyvern/forge/sdk/copilot/block_goal_wrapping.py b/skyvern/forge/sdk/copilot/block_goal_wrapping.pyindex d365ed9fc..a0d4bcb30 100644--- a/skyvern/forge/sdk/copilot/block_goal_wrapping.py+++ b/skyvern/forge/sdk/copilot/block_goal_wrapping.py@@ -13,6 +13,7 @@ Without this wrap:from __future__ import annotations+import refrom typing import Anyimport yaml@@ -39,6 +40,19 @@ _WRAPPABLE_FIELDS: tuple[str, ...] = (# hard-coded substring) keeps idempotency intact if the template's wording# changes._WRAPPED_PREFIX = MINI_GOAL_TEMPLATE.partition("{mini_goal}")[0]+_MINI_GOAL_HEADER = "Achieve the following mini goal and once it's achieved, complete:"+_MAIN_GOAL_HEADER = (+ "This mini goal is part of the big goal the user wants to achieve and use the big goal as context to achieve "+ "the mini goal:"+)+# Tolerates the spaced-fence variant (`` ` ` ` ``) that the LLM sometimes+# emits to avoid nested markdown fences inside an already-wrapped goal.+_FENCE_RE = r"`\s*`\s*`"+_WRAPPED_GOAL_RE = re.compile(+ rf"^\s*{re.escape(_MINI_GOAL_HEADER)}\s*{_FENCE_RE}\s*(?P<mini_goal>.*?)\s*{_FENCE_RE}\s*"+ rf"{re.escape(_MAIN_GOAL_HEADER)}\s*{_FENCE_RE}\s*(?P<main_goal>.*?)\s*{_FENCE_RE}\s*$",+ re.DOTALL,+)def wrap_block_goals(workflow_yaml: str, user_message: str) -> str:@@ -86,7 +100,15 @@ def _wrap_blocks_in_place(blocks: list[Any], user_message: str) -> bool:continuefor field_name in _WRAPPABLE_FIELDS:value = block.get(field_name)- if isinstance(value, str) and value and _WRAPPED_PREFIX not in value:+ if not isinstance(value, str) or not value:+ continue+ normalized_existing_goal = _normalize_existing_wrapped_goal(value)+ if normalized_existing_goal is not None:+ if normalized_existing_goal != value:+ block[field_name] = normalized_existing_goal+ mutated = True+ continue+ if _WRAPPED_PREFIX not in value:block[field_name] = MINI_GOAL_TEMPLATE.format(mini_goal=value,main_goal=user_message,@@ -96,3 +118,32 @@ def _wrap_blocks_in_place(blocks: list[Any], user_message: str) -> bool:if isinstance(loop_blocks, list):mutated = _wrap_blocks_in_place(loop_blocks, user_message) or mutatedreturn mutated+++def _normalize_existing_wrapped_goal(value: str) -> str | None:+ parsed = _extract_wrapped_goal(value)+ if parsed is None:+ return None+ mini_goal, main_goal = parsed+ return MINI_GOAL_TEMPLATE.format(mini_goal=mini_goal, main_goal=main_goal)+++# Bounded peel count: a reword turn stacks one layer; anything deeper is+# malformed and we'd rather stop than recurse on adversarial input.+_MAX_GOAL_UNWRAP_DEPTH = 8+++def _extract_wrapped_goal(value: str) -> tuple[str, str] | None:+ current = value+ last: tuple[str, str] | None = None+ for _ in range(_MAX_GOAL_UNWRAP_DEPTH):+ match = _WRAPPED_GOAL_RE.match(current)+ if not match:+ return last+ mini_goal = match.group("mini_goal").strip()+ main_goal = match.group("main_goal").strip()+ if not mini_goal or not main_goal:+ return last+ last = (mini_goal, main_goal)+ current = mini_goal+ return lastdiff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 0b40c4a60..f48d57d40 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -7,7 +7,7 @@ import base64import jsonimport reimport time-from collections import defaultdict+from collections import Counter, defaultdictfrom datetime import datetimefrom typing import Any, Literal@@ -15,6 +15,7 @@ import structlogimport yamlfrom agents import function_toolfrom agents.run_context import RunContextWrapper+from jinja2.sandbox import SandboxedEnvironmentfrom pydantic import ValidationErrorfrom skyvern.forge import app@@ -47,6 +48,7 @@ from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yamlfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPExceptionfrom skyvern.forge.sdk.workflow.models.parameter import (+ RESERVED_PARAMETER_KEYS,OutputParameter,WorkflowParameter,WorkflowParameterType,@@ -517,6 +519,15 @@ def _parameter_binding_invariant_error(async def _update_workflow(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:workflow_yaml = params["workflow_yaml"]+ # Prefer the most-recent in-turn emission so cross-path flows (inline+ # REPLACE_WORKFLOW followed by update_workflow) compare against what the+ # LLM actually saw, not the turn-start persisted state.+ last_yaml = getattr(ctx, "last_workflow_yaml", None)+ prior_yaml = last_yaml if isinstance(last_yaml, str) and last_yaml else ctx.workflow_yaml+ stale_metadata = _detect_stale_block_metadata(workflow_yaml, prior_yaml)+ if stale_metadata:+ return {"ok": False, "error": _stale_block_metadata_message(stale_metadata)}+# Post-emission reject of copilot-v2 writes that introduce a banned# block type. The schema pre_hook only fires when the LLM consults the# schema; this safety net fires regardless of emission path. Label-based@@ -619,7 +630,34 @@ async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[s# to an upstream state-establisher, or falls back to the full requested list._BLOCK_TYPES_STATE_ESTABLISHER = frozenset({"navigation", "login", "goto_url"})-_OUTPUT_REF_RE = re.compile(r"\{\{\s*([A-Za-z0-9_]+)_output\s*[\.}|]")+_JINJA_IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*"+_OUTPUT_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})_output\s*(?=[\.|}}])")+_BLOCK_FORM_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*\.")+_JINJA_ROOT_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*(?=[\.|}}])")++_JINJA_RUNTIME_GLOBAL_ROOTS = frozenset(SandboxedEnvironment().globals)+_JINJA_LITERAL_ROOTS = frozenset({"none", "true", "false"})+_JINJA_SPECIAL_CONTEXT_ROOTS = frozenset({"loop", "self", "varargs", "kwargs"})+_SKYVERN_TEMPLATE_CONTEXT_ROOTS = frozenset(RESERVED_PARAMETER_KEYS) | frozenset(+ {+ "parameters",+ "browser_session_id",+ "organization_id",+ # Conditional / branch evaluation roots — see BranchEvaluationContext.build_template_data.+ "params",+ "outputs",+ "environment",+ "env",+ "llm",+ }+)+_TEMPLATE_BUILTIN_ROOTS = (+ _JINJA_RUNTIME_GLOBAL_ROOTS | _JINJA_LITERAL_ROOTS | _JINJA_SPECIAL_CONTEXT_ROOTS | _SKYVERN_TEMPLATE_CONTEXT_ROOTS+)++# Keep this to grammatical glue only. Workflow/action words are intentionally+# not filtered; the two-token stale threshold is the conservative guardrail.+_BLOCK_METADATA_STOPWORDS = frozenset({"and", "for", "the", "with"})def _block_type_name(block: Any) -> str:@@ -643,6 +681,218 @@ def _blocks_by_label(workflow_definition: Any) -> dict[str, Any]:return by_label+# Minimum length to apply the trailing-``s`` plural strip; below this we+# leave the token alone so words like ``is``/``us``/``has`` aren't mangled.+_MIN_STEMMABLE_TOKEN_LEN = 5+++def _metadata_token(token: str) -> str:+ token = token.lower()+ if len(token) >= _MIN_STEMMABLE_TOKEN_LEN and token.endswith("s"):+ token = token[:-1]+ return token+++def _metadata_tokens(value: Any) -> set[str]:+ if not isinstance(value, str):+ return set()+ tokens: set[str] = set()+ for token in re.findall(r"[A-Za-z0-9]+", value):+ normalized = _metadata_token(token)+ if len(normalized) <= 2 or normalized in _BLOCK_METADATA_STOPWORDS:+ continue+ tokens.add(normalized)+ return tokens+++def _iter_yaml_blocks(blocks: Any) -> list[dict[str, Any]]:+ found: list[dict[str, Any]] = []+ if not isinstance(blocks, list):+ return found+ for block in blocks:+ if not isinstance(block, dict):+ continue+ found.append(block)+ loop_blocks = block.get("loop_blocks")+ if isinstance(loop_blocks, list):+ found.extend(_iter_yaml_blocks(loop_blocks))+ return found+++def _workflow_yaml_blocks_by_label(workflow_yaml: str | None) -> dict[str, dict[str, Any]]:+ if not workflow_yaml:+ return {}+ try:+ parsed = safe_load_no_dates(workflow_yaml)+ except yaml.YAMLError:+ return {}+ if not isinstance(parsed, dict):+ return {}+ workflow_definition = parsed.get("workflow_definition")+ if not isinstance(workflow_definition, dict):+ return {}+ by_label: dict[str, dict[str, Any]] = {}+ for block in _iter_yaml_blocks(workflow_definition.get("blocks")):+ label = block.get("label")+ if isinstance(label, str):+ by_label[label] = block+ return by_label+++def _semantic_tokens_from_yaml(value: Any, *, exclude_keys: frozenset[str]) -> set[str]:+ tokens: set[str] = set()+ if isinstance(value, str):+ return _metadata_tokens(value)+ if isinstance(value, list):+ for item in value:+ tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))+ return tokens+ if isinstance(value, dict):+ for key, item in value.items():+ if key in exclude_keys:+ continue+ tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))+ return tokens+++def _stale_metadata_reason(+ *,+ field_name: str,+ field_value: Any,+ prior_block: dict[str, Any],+ submitted_block: dict[str, Any],+ current_exclude_keys: frozenset[str],+) -> str | None:+ tokens = _metadata_tokens(field_value)+ if len(tokens) < 2:+ return None++ prior_tokens = _semantic_tokens_from_yaml(prior_block, exclude_keys=current_exclude_keys)+ current_tokens = _semantic_tokens_from_yaml(submitted_block, exclude_keys=current_exclude_keys)+ removed_tokens = prior_tokens - current_tokens+ if len(tokens & removed_tokens) < 2:+ return None++ return f"{field_name} {field_value!r} appears stale"+++def _prior_blocks_by_unique_title(prior_by_label: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:+ titled = [(b["title"], b) for b in prior_by_label.values() if isinstance(b.get("title"), str) and b["title"]]+ counts = Counter(title for title, _ in titled)+ return {title: block for title, block in titled if counts[title] == 1}+++_STALE_BASE_EXCLUDE = frozenset({"label", "next_block_label"})+_STALE_TITLE_EXCLUDE = frozenset({"label", "title", "next_block_label"})+++def _stale_for_renamed_label(+ label: str,+ submitted_block: dict[str, Any],+ prior_by_unique_title: dict[str, dict[str, Any]],+) -> dict[str, Any] | None:+ submitted_title = submitted_block.get("title")+ # Title-less blocks intentionally skip the renamed-label path: with no+ # title there is no cross-reference to a prior block, so we'd be+ # guessing whether the rename was warranted.+ if not (isinstance(submitted_title, str) and submitted_title):+ return None+ prior_block = prior_by_unique_title.get(submitted_title)+ if prior_block is None:+ return None+ title_reason = _stale_metadata_reason(+ field_name="title",+ field_value=submitted_title,+ prior_block=prior_block,+ submitted_block=submitted_block,+ current_exclude_keys=_STALE_TITLE_EXCLUDE,+ )+ if not title_reason:+ return None+ return {"label": label, "reasons": [title_reason]}+++def _stale_for_matched_label(+ label: str,+ submitted_block: dict[str, Any],+ prior_block: dict[str, Any],+) -> dict[str, Any] | None:+ reasons: list[str] = []+ submitted_title = submitted_block.get("title")+ prior_title = prior_block.get("title")+ title_reason = None+ if isinstance(submitted_title, str) and submitted_title and submitted_title == prior_title:+ title_reason = _stale_metadata_reason(+ field_name="title",+ field_value=submitted_title,+ prior_block=prior_block,+ submitted_block=submitted_block,+ current_exclude_keys=_STALE_TITLE_EXCLUDE,+ )+ if title_reason:+ reasons.append(title_reason)++ # When the title was already flagged stale, exclude its tokens from the+ # label-stale comparison so the same words don't double-count; otherwise+ # the title's tokens are stable content that should weigh in.+ label_exclude_keys = _STALE_TITLE_EXCLUDE if title_reason else _STALE_BASE_EXCLUDE+ label_reason = _stale_metadata_reason(+ field_name="label",+ field_value=label,+ prior_block=prior_block,+ submitted_block=submitted_block,+ current_exclude_keys=label_exclude_keys,+ )+ if label_reason:+ reasons.insert(0, label_reason)++ if not reasons:+ return None+ return {"label": label, "reasons": reasons}+++def _detect_stale_block_metadata(submitted_yaml: str | None, prior_yaml: str | None) -> list[dict[str, Any]]:+ """Find corrected blocks whose old label/title no longer matches their revised goal text."""+ prior_by_label = _workflow_yaml_blocks_by_label(prior_yaml)+ submitted_by_label = _workflow_yaml_blocks_by_label(submitted_yaml)+ if not prior_by_label or not submitted_by_label:+ return []++ prior_by_unique_title = _prior_blocks_by_unique_title(prior_by_label)++ stale_items: list[dict[str, Any]] = []+ for label, submitted_block in submitted_by_label.items():+ prior_block = prior_by_label.get(label)+ if prior_block is None:+ item = _stale_for_renamed_label(label, submitted_block, prior_by_unique_title)+ else:+ item = _stale_for_matched_label(label, submitted_block, prior_block)+ if item is not None:+ stale_items.append(item)+ return stale_items+++_STALE_BLOCK_METADATA_MESSAGE_LIMIT = 5+++def _stale_block_metadata_message(items: list[dict[str, Any]]) -> str:+ details = []+ for item in items[:_STALE_BLOCK_METADATA_MESSAGE_LIMIT]:+ label = item.get("label", "?")+ reasons = item.get("reasons") or []+ detail = "; ".join(str(reason) for reason in reasons)+ details.append(f"{label}: {detail}")+ if len(items) > _STALE_BLOCK_METADATA_MESSAGE_LIMIT:+ details.append(f"(and {len(items) - _STALE_BLOCK_METADATA_MESSAGE_LIMIT} more)")+ joined = "; ".join(details)+ return (+ "Workflow validation failed: corrected block metadata still appears stale. "+ "When changing a user's requested subject, URL, or action, rename affected block labels and titles "+ "to match the revised goal, and update next_block_label, block_labels, and Jinja references accordingly. "+ f"Stale metadata: {joined}"+ )++def _find_invalidated_labels(old_definition: Any,new_definition: Any,@@ -698,20 +948,98 @@ def _nearest_upstream_state_establisher(return None-def _referenced_output_labels(frontier_labels: list[str], new_definition: Any) -> set[str]:+def _serialized_frontier_block_configs(frontier_labels: list[str], new_definition: Any) -> list[str]:by_label = _blocks_by_label(new_definition)- needed: set[str] = set()+ serialized_configs: list[str] = []for label in frontier_labels:block = by_label.get(label)if block is None:continuetry:- serialized = json.dumps(_canonical_block_config(block), default=str, separators=(",", ":"))+ serialized_configs.append(json.dumps(_canonical_block_config(block), default=str, separators=(",", ":")))except (TypeError, ValueError):- serialized = repr(block)+ serialized_configs.append(repr(block))+ return serialized_configs+++def _workflow_parameter_keys(definition: Any) -> set[str]:+ parameters = getattr(definition, "parameters", None) if definition else None+ keys: set[str] = set()+ if not parameters:+ return keys+ for parameter in parameters:+ key = getattr(parameter, "key", None)+ if isinstance(key, str):+ keys.add(key)+ return keys+++_CREDENTIAL_REAL_VALUE_SUFFIXES = ("_real_username", "_real_password")+++def _classify_frontier_jinja_refs(+ frontier_labels: list[str],+ new_definition: Any,+ serialized_configs: list[str] | None = None,+) -> tuple[set[str], set[str], set[str]]:+ """Single pass over frontier blocks; returns ``(suffix_form_refs, block_form_refs, unknown_roots)``."""+ if serialized_configs is None:+ serialized_configs = _serialized_frontier_block_configs(frontier_labels, new_definition)+ known_labels = set(_blocks_by_label(new_definition))+ parameter_keys = _workflow_parameter_keys(new_definition)+ known_roots = known_labels | parameter_keys | _TEMPLATE_BUILTIN_ROOTS++ suffix_form_refs: set[str] = set()+ block_form_refs: set[str] = set()+ unknown_roots: set[str] = set()++ for serialized in serialized_configs:for match in _OUTPUT_REF_RE.findall(serialized):- needed.add(match)- return needed+ if match in known_labels:+ suffix_form_refs.add(match)+ for match in _BLOCK_FORM_REF_RE.findall(serialized):+ if match in known_labels:+ block_form_refs.add(match)+ for root in _JINJA_ROOT_RE.findall(serialized):+ if root in known_roots:+ continue+ if root.endswith("_output") and root[: -len("_output")] in known_labels:+ continue+ if any(+ root.endswith(suffix) and root[: -len(suffix)] in parameter_keys+ for suffix in _CREDENTIAL_REAL_VALUE_SUFFIXES+ ):+ continue+ unknown_roots.add(root)++ return suffix_form_refs, block_form_refs, unknown_roots+++def _referenced_output_labels(+ frontier_labels: list[str],+ new_definition: Any,+ serialized_configs: list[str] | None = None,+) -> set[str]:+ suffix_refs, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)+ return suffix_refs | block_form_refs+++def _block_form_output_labels(+ frontier_labels: list[str],+ new_definition: Any,+ serialized_configs: list[str] | None = None,+) -> set[str]:+ _, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)+ return block_form_refs+++def _unknown_jinja_roots(+ frontier_labels: list[str],+ new_definition: Any,+ serialized_configs: list[str] | None = None,+) -> set[str]:+ _, _, unknown_roots = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)+ return unknown_rootsdef _summarize_action_trace(action_trace: list[dict[str, Any]] | None) -> list[str]:@@ -846,20 +1174,24 @@ def _seed_for_frontier(prefix_labels = requested_labels[:idx]if not prefix_labels:return labels_to_execute, {}, frontier- # Only seed outputs that are actually referenced by the frontier suffix.- # Over-seeding would weaken the "seed what downstream needs" discipline- # and risks masking bugs where a block references an output that doesn't- # flow through the normal {{label_output}} path.- needed = _referenced_output_labels(labels_to_execute, new_definition)- prefix_needed = [label for label in prefix_labels if label in needed]+ serialized_configs = _serialized_frontier_block_configs(labels_to_execute, new_definition)+ suffix_refs, block_form_refs, unknown_roots = _classify_frontier_jinja_refs(+ labels_to_execute, new_definition, serialized_configs+ )+ if any(label in block_form_refs for label in prefix_labels):+ # Seeded block_outputs only register <label>_output; block-form refs+ # need a normal upstream execution to populate the <label> namespace.+ return requested_labels, {}, requested_labels[0]+ needed = suffix_refs | block_form_refsseed: dict[str, Any] = {}- for label in prefix_needed:- if label in verified_outputs:- seed[label] = verified_outputs[label]- else:- # A referenced output is missing from the verified cache — we- # can't safely seed just the suffix. Fall back to the full list.+ for label in prefix_labels:+ if label not in needed:+ continue+ if label not in verified_outputs:return requested_labels, {}, requested_labels[0]+ seed[label] = verified_outputs[label]+ if unknown_roots:+ return requested_labels, {}, requested_labels[0]return labels_to_execute, seed, frontierdiff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 0e6dbef10..1dc9f5b99 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -447,6 +447,9 @@ class WorkflowRunModel(Base):)copilot_session_id = Column(String, nullable=True)+ credits_used = Column(Integer, nullable=True, default=0, server_default="0")+ cached_credits_used = Column(Integer, nullable=True, default=0, server_default="0")+queued_at = Column(DateTime, nullable=True)started_at = Column(DateTime, nullable=True)finished_at = Column(DateTime, nullable=True)diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex 40211615f..59d6b50a5 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -297,6 +297,29 @@ class WorkflowRunsRepository(BaseRepository):else:raise WorkflowRunNotFound(workflow_run_id)+ @db_operation("increment_workflow_run_credits")+ async def increment_workflow_run_credits(+ self,+ workflow_run_id: str,+ credits: int,+ is_cached: bool = False,+ ) -> None:+ col = WorkflowRunModel.cached_credits_used if is_cached else WorkflowRunModel.credits_used+ async with self.Session() as session:+ result = await session.execute(+ update(WorkflowRunModel)+ .where(WorkflowRunModel.workflow_run_id == workflow_run_id)+ .values({col: func.coalesce(col, 0) + credits})+ )+ if result.rowcount == 0:+ LOG.warning(+ "increment_workflow_run_credits matched no rows",+ workflow_run_id=workflow_run_id,+ credits=credits,+ is_cached=is_cached,+ )+ await session.commit()+@db_operation("update_workflow_run_if_not_final")async def update_workflow_run_if_not_final(self,diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 7e259fbe5..4e785742e 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -507,6 +507,8 @@ def convert_to_workflow_run(failure_category=workflow_run_model.failure_category,ignore_inherited_workflow_system_prompt=workflow_run_model.ignore_inherited_workflow_system_prompt,copilot_session_id=workflow_run_model.copilot_session_id,+ credits_used=workflow_run_model.credits_used or 0,+ cached_credits_used=workflow_run_model.cached_credits_used or 0,)diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex 7d21f9aab..281d59e01 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -204,6 +204,8 @@ class WorkflowRun(BaseModel):workflow_schedule_id: str | None = Noneignore_inherited_workflow_system_prompt: bool = Falsecopilot_session_id: str | None = None+ credits_used: int = 0+ cached_credits_used: int = 0@field_validator("run_with", mode="before")@classmethoddiff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 28fd53a27..e52026f4a 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -258,6 +258,46 @@ class TestTranslateToAgentResultGating:assert agent_result.workflow_yaml is Noneassert agent_result.response_type == "REPLACE_WORKFLOW"+ def test_inline_replace_workflow_rejects_stale_block_metadata(self, monkeypatch) -> None:+ # Inline REPLACE_WORKFLOW bypasses _update_workflow, so it must also+ # reject a corrected workflow whose labels/titles still describe the+ # prior subject.+ process_mock = MagicMock(return_value=SimpleNamespace(name="new"))+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)++ prior_yaml = """+title: Count example.com topic alpha results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for topic alpha.+"""+ submitted_yaml = """+title: Count example.com sample beta results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for sample beta.+"""+ ctx = _ctx(workflow_yaml=prior_yaml, last_workflow_yaml=prior_yaml, last_workflow=object(), last_test_ok=True)+ result = _fake_run_result(+ {"type": "REPLACE_WORKFLOW", "user_response": "Here you go.", "workflow_yaml": submitted_yaml}+ )+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ process_mock.assert_not_called()+ assert "corrected block metadata still appears stale" in agent_result.user_response+ assert agent_result.updated_workflow is None+ assert agent_result.workflow_yaml is None+def test_inline_replace_with_invalid_yaml_keeps_prior_pass(self, monkeypatch) -> None:# _process_workflow_yaml raising on a malformed REPLACE must leave# ctx untouched — no spurious last_test_ok reset, no workflow swap —diff --git a/tests/unit/test_copilot_block_goal_wrapping.py b/tests/unit/test_copilot_block_goal_wrapping.pyindex 01d269518..0a96fe1c6 100644--- a/tests/unit/test_copilot_block_goal_wrapping.py+++ b/tests/unit/test_copilot_block_goal_wrapping.py@@ -143,6 +143,36 @@ def test_idempotent_on_already_wrapped_fields() -> None:assert block["complete_criterion"] == already_wrapped_criterion+def test_normalizes_spaced_fence_wrapped_goal_without_adding_reword_turn() -> None:+ revised_goal = "Go to https://example.com. Tell me how many results there are referencing topic beta"+ spaced_fence_wrapped_goal = textwrap.dedent(+ f"""+ Achieve the following mini goal and once it's achieved, complete:+ ` ` `In the main search box, enter the phrase 'topic beta' and submit the search.` ` `++ This mini goal is part of the big goal the user wants to achieve and use the big goal as context to achieve the mini goal:+ ` ` `{revised_goal}` ` `+ """+ ).strip()+ src = _yaml_with_blocks(+ {+ "block_type": "navigation",+ "label": "search_topic_beta",+ "navigation_goal": spaced_fence_wrapped_goal,+ }+ )++ out = wrap_block_goals(src, "I meant topic beta")++ goal = _blocks_from(out)[0]["navigation_goal"]+ assert goal == MINI_GOAL_TEMPLATE.format(+ mini_goal="In the main search box, enter the phrase 'topic beta' and submit the search.",+ main_goal=revised_goal,+ )+ assert "I meant topic beta" not in goal+ assert goal.count("Achieve the following mini goal") == 1++def test_noop_on_empty_user_message() -> None:src = _yaml_with_blocks({diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex b5e714c09..f1e67c072 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -5,7 +5,9 @@ from __future__ import annotationsfrom typing import Anyimport pytest+from jinja2.sandbox import SandboxedEnvironment+from skyvern.forge.sdk.copilot import toolsfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import (MAX_FAILED_TEST_NUDGES,@@ -26,6 +28,7 @@ from skyvern.forge.sdk.copilot.tools import (_plan_frontier,_referenced_output_labels,)+from skyvern.forge.sdk.workflow.models.parameter import RESERVED_PARAMETER_KEYSclass _FakeBlock:@@ -50,9 +53,15 @@ class _FakeBlock:}+class _FakeParameter:+ def __init__(self, key: str) -> None:+ self.key = key++class _FakeDefinition:- def __init__(self, blocks: list[_FakeBlock]) -> None:+ def __init__(self, blocks: list[_FakeBlock], parameters: list[_FakeParameter] | None = None) -> None:self.blocks = blocks+ self.parameters = parameters or []class _FakeWorkflow:@@ -213,6 +222,490 @@ def test_referenced_output_labels_finds_jinja_refs() -> None:assert "a" in refs+def test_referenced_output_labels_finds_block_form_jinja_refs() -> None:+ new = _FakeDefinition(+ [+ _FakeBlock("extract_article_info", "extraction"),+ _FakeBlock(+ "summarize_article",+ "text_prompt",+ {+ "prompt": (+ "Summarize {{ extract_article_info.output.extracted_information.abstract }} "+ "and {{ extract_article_info.title }}."+ )+ },+ ),+ ]+ )++ refs = _referenced_output_labels(["summarize_article"], new)++ assert refs == {"extract_article_info"}+++def test_plan_frontier_append_with_block_form_jinja_ref_falls_back_to_full_run() -> None:+ old = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),+ ]+ )+ new = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),+ _FakeBlock(+ "summarize_article",+ "text_prompt",+ {+ "prompt": (+ "Summarize the main findings from "+ "{{ extract_article_info.output.extracted_information.abstract }}."+ )+ },+ ),+ ]+ )+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open_page", "extract_article_info"]+ ctx.verified_block_outputs = {+ "open_page": "nav_ok",+ "extract_article_info": {"extracted_information": {"abstract": "Prior output"}},+ }++ labels, seed, frontier = _plan_frontier(+ ctx,+ ["open_page", "extract_article_info", "summarize_article"],+ old,+ new,+ )++ assert labels == ["open_page", "extract_article_info", "summarize_article"]+ assert seed == {}+ assert frontier == "open_page"+++def test_plan_frontier_append_seeds_output_parameter_jinja_ref() -> None:+ old = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),+ ]+ )+ new = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),+ _FakeBlock(+ "summarize_article",+ "text_prompt",+ {+ "prompt": (+ "Summarize the main findings from "+ "{{ extract_article_info_output.extracted_information.abstract }}."+ )+ },+ ),+ ]+ )+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open_page", "extract_article_info"]+ ctx.verified_block_outputs = {+ "open_page": "nav_ok",+ "extract_article_info": {"extracted_information": {"abstract": "Prior output"}},+ }++ labels, seed, frontier = _plan_frontier(+ ctx,+ ["open_page", "extract_article_info", "summarize_article"],+ old,+ new,+ )++ assert labels == ["summarize_article"]+ assert seed == {"extract_article_info": {"extracted_information": {"abstract": "Prior output"}}}+ assert frontier == "summarize_article"+++def test_stale_metadata_detects_corrected_subject_label_and_title() -> None:+ prior_yaml = """+title: Count example.com topic alpha results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: extract_results+ navigation_goal: Search example.com for topic alpha.+ - block_type: extraction+ label: extract_results+ title: Extract Results+ next_block_label: null+ data_extraction_goal: Extract the total number of topic alpha search results.+"""+ submitted_yaml = """+title: Count example.com sample beta results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: extract_results+ navigation_goal: Search example.com for sample beta.+ - block_type: extraction+ label: extract_results+ title: Extract Results+ next_block_label: null+ data_extraction_goal: Extract the total number of sample beta search results.+"""++ stale = tools._detect_stale_block_metadata(submitted_yaml, prior_yaml)++ assert stale == [+ {+ "label": "search_topic_alpha",+ "reasons": [+ "label 'search_topic_alpha' appears stale",+ "title 'Search Topic Alpha' appears stale",+ ],+ }+ ]+++def test_stale_metadata_accepts_renamed_corrected_subject() -> None:+ prior_yaml = """+title: Count example.com topic alpha results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: extract_results+ navigation_goal: Search example.com for topic alpha.+ - block_type: extraction+ label: extract_results+ title: Extract Results+ next_block_label: null+ data_extraction_goal: Extract the total number of topic alpha search results.+"""+ submitted_yaml = """+title: Count example.com sample beta results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_sample_beta+ title: Search Sample Beta+ next_block_label: extract_results+ navigation_goal: Search example.com for sample beta.+ - block_type: extraction+ label: extract_results+ title: Extract Results+ next_block_label: null+ data_extraction_goal: Extract the total number of sample beta search results.+"""++ assert tools._detect_stale_block_metadata(submitted_yaml, prior_yaml) == []+++def test_stale_metadata_accepts_reworded_action_with_same_subject() -> None:+ prior_yaml = """+title: Count example.com topic alpha results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for topic alpha.+"""+ submitted_yaml = """+title: Count example.com topic alpha results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Find example.com pages about topic alpha.+"""++ assert tools._detect_stale_block_metadata(submitted_yaml, prior_yaml) == []+++def test_plan_frontier_unknown_jinja_root_falls_back_to_full_requested_list() -> None:+ old = _FakeDefinition([_FakeBlock("open_page", "navigation")])+ new = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("summarize_article", "text_prompt", {"prompt": "Summarize {{ missing_block.abstract }}."}),+ ]+ )+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open_page"]+ ctx.verified_block_outputs = {"open_page": "nav_ok"}++ labels, seed, frontier = _plan_frontier(ctx, ["open_page", "summarize_article"], old, new)++ assert labels == ["open_page", "summarize_article"]+ assert seed == {}+ assert frontier == "open_page"+++def test_plan_frontier_falls_back_when_unknown_root_coexists_with_seedable_ref() -> None:+ # Even when the suffix references a verified upstream output (so seeding+ # would otherwise let us skip the prefix), an additional unknown Jinja+ # root must still trigger the conservative full-rerun fallback.+ old = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),+ ]+ )+ new = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),+ _FakeBlock(+ "summarize_article",+ "text_prompt",+ {+ "prompt": (+ "Summarize {{ extract_article_info_output.extracted_information.abstract }} "+ "with context {{ missing_block.note }}."+ )+ },+ ),+ ]+ )+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open_page", "extract_article_info"]+ ctx.verified_block_outputs = {+ "open_page": "nav_ok",+ "extract_article_info": {"extracted_information": {"abstract": "Prior output"}},+ }++ labels, seed, frontier = _plan_frontier(+ ctx,+ ["open_page", "extract_article_info", "summarize_article"],+ old,+ new,+ )++ assert labels == ["open_page", "extract_article_info", "summarize_article"]+ assert seed == {}+ assert frontier == "open_page"+++def test_unknown_jinja_roots_ignores_credential_real_value_synthetic_roots() -> None:+ new = _FakeDefinition(+ [+ _FakeBlock(+ "login",+ "login",+ {"prompt": "Sign in with {{ creds_real_username }} / {{ creds_real_password }}."},+ ),+ ],+ parameters=[_FakeParameter("creds")],+ )++ assert tools._unknown_jinja_roots(["login"], new) == set()+++def test_unknown_jinja_roots_ignores_conditional_branch_context_roots() -> None:+ new = _FakeDefinition(+ [+ _FakeBlock(+ "branch",+ "conditional",+ {+ "expression": (+ "{{ params.foo }} {{ outputs.bar }} {{ environment.region }} {{ env.flag }} {{ llm.model }}"+ )+ },+ ),+ ]+ )++ assert tools._unknown_jinja_roots(["branch"], new) == set()+++def test_stale_metadata_accepts_single_token_subject_change_as_known_limit() -> None:+ prior_yaml = """+title: Search results page+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_cats+ title: Search Cats+ next_block_label: null+ navigation_goal: Search the directory for cats.+"""+ submitted_yaml = """+title: Search results page+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_cats+ title: Search Cats+ next_block_label: null+ navigation_goal: Search the directory for dogs.+"""++ # The code gate is a conservative backstop: it requires at least two+ # removed metadata tokens before rejecting. Single-token subject swaps are+ # expected to be handled by the prompt instruction to rename changed+ # subject metadata.+ assert tools._detect_stale_block_metadata(submitted_yaml, prior_yaml) == []+++def test_stale_metadata_detects_stale_title_after_label_rename() -> None:+ prior_yaml = """+title: Count example.com topic alpha results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for topic alpha.+"""+ submitted_yaml = """+title: Count example.com sample beta results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_sample_beta+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for sample beta.+"""++ stale = tools._detect_stale_block_metadata(submitted_yaml, prior_yaml)++ assert stale == [+ {+ "label": "search_sample_beta",+ "reasons": ["title 'Search Topic Alpha' appears stale"],+ }+ ]+++def test_stale_metadata_detects_stale_block_inside_loop_blocks() -> None:+ prior_yaml = """+title: For-each search results+workflow_definition:+ blocks:+ - block_type: for_loop+ label: per_topic+ loop_blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for topic alpha.+"""+ submitted_yaml = """+title: For-each search results+workflow_definition:+ blocks:+ - block_type: for_loop+ label: per_topic+ loop_blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for sample beta.+"""++ stale = tools._detect_stale_block_metadata(submitted_yaml, prior_yaml)++ assert {item["label"] for item in stale} == {"search_topic_alpha"}+++def test_stale_metadata_message_indicates_truncation_when_over_limit() -> None:+ items = [{"label": f"label_{i}", "reasons": [f"reason {i}"]} for i in range(7)]+ message = tools._stale_block_metadata_message(items)+ assert "and 2 more" in message+++def test_stale_metadata_message_omits_truncation_indicator_under_limit() -> None:+ items = [{"label": f"label_{i}", "reasons": [f"reason {i}"]} for i in range(3)]+ message = tools._stale_block_metadata_message(items)+ assert "more" not in message+++def test_referenced_output_labels_ignores_non_block_jinja_roots() -> None:+ new = _FakeDefinition(+ [+ _FakeBlock(+ "summarize_article",+ "text_prompt",+ {"prompt": "Summarize {{ search_term.field }} for {{ loop.index }}."},+ ),+ ]+ )++ refs = _referenced_output_labels(["summarize_article"], new)++ assert refs == set()+++def test_plan_frontier_append_only_with_workflow_param_does_not_fall_back() -> None:+ old = _FakeDefinition(+ [_FakeBlock("open_page", "navigation")],+ parameters=[_FakeParameter("search_term")],+ )+ new = _FakeDefinition(+ [+ _FakeBlock("open_page", "navigation"),+ _FakeBlock("search", "navigation", {"prompt": "Search for {{ search_term }} on this site"}),+ ],+ parameters=[_FakeParameter("search_term")],+ )+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open_page"]+ ctx.verified_block_outputs = {"open_page": "nav_ok"}++ labels, seed, frontier = _plan_frontier(ctx, ["open_page", "search"], old, new)++ assert labels == ["search"]+ assert seed == {}+ assert frontier == "search"+++def test_template_builtin_roots_track_jinja_and_skyvern_contexts() -> None:+ assert tools._JINJA_RUNTIME_GLOBAL_ROOTS == frozenset(SandboxedEnvironment().globals)+ assert tools._JINJA_RUNTIME_GLOBAL_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS+ assert tools._JINJA_LITERAL_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS+ assert tools._JINJA_SPECIAL_CONTEXT_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS+ assert frozenset(RESERVED_PARAMETER_KEYS) <= tools._SKYVERN_TEMPLATE_CONTEXT_ROOTS+ assert {"parameters", "browser_session_id", "organization_id"} <= tools._SKYVERN_TEMPLATE_CONTEXT_ROOTS+ assert tools._SKYVERN_TEMPLATE_CONTEXT_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS+++def test_unknown_jinja_roots_ignores_jinja_and_skyvern_context_roots() -> None:+ new = _FakeDefinition(+ [+ _FakeBlock(+ "summarize",+ "text_prompt",+ {+ "prompt": (+ "{{ range }} {{ dict }} {{ namespace }} {{ cycler }} {{ joiner }} {{ lipsum }} "+ "{{ none }} {{ true }} {{ false }} {{ loop.index }} {{ self }} {{ varargs }} {{ kwargs }} "+ "{{ parameters.search_term }} {{ browser_session_id }} {{ organization_id }} "+ "{{ current_date }} {{ workflow_run_id }}"+ )+ },+ ),+ ]+ )++ assert tools._unknown_jinja_roots(["summarize"], new) == set()++# --------------------------------------------------------------------------- ## Compact packet shape ## --------------------------------------------------------------------------- #@@ -402,6 +895,36 @@ def test_failed_unchanged_rerun_preserves_verified_prefix_and_outputs() -> None:assert ctx.verified_block_outputs == {"a": "nav", "b": {"title": "hi"}}+def test_run_blocks_outcome_rolls_forward_after_failed_preview() -> None:+ ctx = _make_ctx()++ tools._record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_fail",+ "blocks": [{"label": "summarize", "status": "failed", "failure_reason": "Jinja ref undefined"}],+ },+ },+ )+ assert ctx.last_test_ok is False++ tools._record_run_blocks_result(+ ctx,+ {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_success",+ "blocks": [{"label": "summarize", "status": "completed", "extracted_data": {"summary": "ok"}}],+ },+ },+ )++ assert ctx.last_test_ok is True+ assert ctx.last_test_failure_reason is None++def test_yaml_diff_invalidation_drops_edited_label_and_downstream() -> None:"""When the YAML changes between runs, verified-state invalidation basedon the diff should drop edited labels and anything downstream so thediff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.pyindex fe75c1daa..ed1ef0029 100644--- a/tests/unit/test_copilot_session_injection.py+++ b/tests/unit/test_copilot_session_injection.py@@ -459,6 +459,86 @@ class TestUpdateWorkflowDirect:assert ctx.workflow_yaml == "original yaml"+ @pytest.mark.asyncio+ async def test_rejects_stale_block_metadata_before_persistence(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _update_workflow++ prior_yaml = """+title: Count example.com topic alpha results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for topic alpha.+"""+ submitted_yaml = """+title: Count example.com sample beta results+workflow_definition:+ blocks:+ - block_type: navigation+ label: search_topic_alpha+ title: Search Topic Alpha+ next_block_label: null+ navigation_goal: Search example.com for sample beta.+"""+ ctx = _make_ctx(workflow_yaml=prior_yaml)++ result = await _update_workflow({"workflow_yaml": submitted_yaml}, ctx)++ assert result["ok"] is False+ assert "corrected block metadata still appears stale" in result["error"]+ assert "search_topic_alpha" in result["error"]+ assert ctx.workflow_yaml == prior_yaml++ @pytest.mark.asyncio+ async def test_stale_check_prefers_in_turn_emission_over_turn_start_yaml(self) -> None:+ # Cross-path flow: an earlier inline REPLACE_WORKFLOW left+ # ctx.last_workflow_yaml at the renamed state. A subsequent+ # update_workflow call submits a regression that re-stales the label+ # — this must compare against ctx.last_workflow_yaml, not the older+ # ctx.workflow_yaml that still points at turn-start.+ from skyvern.forge.sdk.copilot.tools import _update_workflow++ turn_start_yaml = """+title: Alpha workflow+workflow_definition:+ blocks:+ - block_type: navigation+ label: review_alpha_one+ title: Review Alpha One+ next_block_label: null+ navigation_goal: Review the alpha one records on the directory.+"""+ in_turn_emission_yaml = """+title: Beta workflow+workflow_definition:+ blocks:+ - block_type: navigation+ label: review_beta_two+ title: Review Beta Two+ next_block_label: null+ navigation_goal: Review the beta two records on the directory.+"""+ regressed_yaml = """+title: Beta workflow+workflow_definition:+ blocks:+ - block_type: navigation+ label: review_beta_two+ title: Review Beta Two+ next_block_label: null+ navigation_goal: Review the gamma three records on the directory.+"""+ ctx = _make_ctx(workflow_yaml=turn_start_yaml)+ ctx.last_workflow_yaml = in_turn_emission_yaml++ result = await _update_workflow({"workflow_yaml": regressed_yaml}, ctx)++ assert result["ok"] is False+ assert "review_beta_two" in result["error"]+class TestWorkflowUpdatePersistence:def test_record_marks_persisted_even_when_workflow_has_zero_blocks(self) -> None:diff --git a/tests/unit/test_workflow_copilot_prompt_injection.py b/tests/unit/test_workflow_copilot_prompt_injection.pyindex f9a75524c..46dd8a0f0 100644--- a/tests/unit/test_workflow_copilot_prompt_injection.py+++ b/tests/unit/test_workflow_copilot_prompt_injection.py@@ -64,6 +64,15 @@ class TestSystemTemplateSecurity:assert "TRUSTED_KB_CONTENT" in rendered+class TestAgentTemplateCorrectionRules:+ def test_agent_template_requires_label_and_title_refresh_on_corrections(self) -> None:+ rendered = prompt_engine.load_prompt("workflow-copilot-agent", **_AGENT_TEMPLATE_DEFAULTS)++ assert "Rename affected block labels and block titles" in rendered+ assert "Labels become output keys" in rendered+ assert "Jinja block reference" in rendered++class TestUserTemplateCodeFencing:"""Verify untrusted variables are wrapped in code fences."""