Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10124: Make Copilot unexpected errors recoverable (#6143)
Andrew Neilson46a287bacf56
diff --git a/skyvern/cli/quickstart.py b/skyvern/cli/quickstart.pyindex aeefe2178..c59316151 100644--- a/skyvern/cli/quickstart.py+++ b/skyvern/cli/quickstart.py@@ -3,6 +3,8 @@import asyncioimport importlibimport importlib.metadata+import re+import shutilimport subprocessimport sysfrom enum import Enum@@ -556,6 +558,62 @@ def _configure_local_browser_streaming_defaults() -> None:set_key(str(frontend_env), "VITE_BROWSER_STREAMING_MODE", "cdp", quote_mode="never")+_LOCALHOST_DB_RE = re.compile(r"^\s*(?:export\s+)?DATABASE_STRING\s*=.*@localhost[/:\"]", re.IGNORECASE)+_COMPOSE_DATABASE_STRING = 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"'+++def _rewrite_localhost_db_string(content: str) -> str | None:+ lines = content.splitlines()+ changed = False+ for i, line in enumerate(lines):+ if _LOCALHOST_DB_RE.match(line):+ lines[i] = _COMPOSE_DATABASE_STRING+ changed = True+ if not changed:+ return None+ result = "\n".join(lines)+ if content.endswith("\n"):+ result += "\n"+ return result+++def _bootstrap_compose_env_files() -> None:+ env_path = Path(".env")+ env_example = Path(".env.example")++ if not env_path.exists():+ if env_example.exists():+ content = env_example.read_text()+ rewritten = _rewrite_localhost_db_string(content)+ env_path.write_text(rewritten if rewritten is not None else content, encoding="utf-8")+ console.print("[green]Created .env from .env.example[/green]")+ else:+ try:+ content = env_path.read_text()+ except OSError:+ content = ""+ rewritten = _rewrite_localhost_db_string(content)+ if rewritten is not None:+ console.print(+ Panel(+ "[bold yellow]DATABASE_STRING pointing to localhost detected in .env.[/bold yellow]\n\n"+ "Docker Compose uses its own Postgres service. The connection string will be "+ "rewritten to point to the compose [cyan]postgres[/cyan] host.",+ border_style="yellow",+ )+ )+ if Confirm.ask("Rewrite DATABASE_STRING for Docker Compose?", default=True):+ env_path.write_text(rewritten, encoding="utf-8")+ console.print("[green]Rewrote DATABASE_STRING for Docker Compose[/green]")++ frontend_env = Path("skyvern-frontend/.env")+ frontend_example = Path("skyvern-frontend/.env.example")+ if not frontend_env.exists() and frontend_example.exists():+ frontend_env.parent.mkdir(parents=True, exist_ok=True)+ shutil.copy(frontend_example, frontend_env)+ console.print("[green]Created skyvern-frontend/.env from .env.example[/green]")++def run_docker_compose_setup() -> None:"""Run the Docker Compose setup for Skyvern."""from skyvern.cli.llm_setup import setup_llm_providers # noqa: PLC0415@@ -563,6 +621,7 @@ def run_docker_compose_setup() -> None:console.print("\n[bold blue]Setting up Skyvern with Docker Compose...[/bold blue]")capture_setup_event("docker-compose-start")+ _bootstrap_compose_env_files()_handle_running_compose_stack()_handle_postgres_container_conflict()diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex f36a370ea..cec0d365f 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -46,12 +46,19 @@ from skyvern.forge.sdk.copilot.output_policy import (normalize_response_scaffolding,output_policy_verdict_from_trace_data,output_policy_verdict_to_trace_data,- url_origin,)from skyvern.forge.sdk.copilot.output_utils import (extract_final_text,parse_final_response,)+from skyvern.forge.sdk.copilot.recoverable_failure import (+ RecoverableFailure,+ build_recoverable_failure,+ clean_recorded_failure_text,+ format_recoverable_failure_reply,+ iter_exception_chain,+ merge_failure_into_context,+)from skyvern.forge.sdk.copilot.request_policy import (CREDENTIAL_DEFERRED_DRAFT_REASONS,RAW_SECRET_REFUSAL_SENTINEL,@@ -112,7 +119,7 @@ def _copilot_turn_span(chat_request: WorkflowCopilotChatRequest,chat_history: list[WorkflowCopilotChatHistoryMessage],turn_index: int | None,-) -> Iterator[None]:+) -> Iterator[Any]:tracer = otel_trace.get_tracer("skyvern")with tracer.start_as_current_span(_COPILOT_TURN_SPAN_NAME) as span:span.set_attribute("skyvern.span.role", "wrapper")@@ -125,7 +132,7 @@ def _copilot_turn_span(if chat_request.workflow_permanent_id:span.set_attribute("workflow_permanent_id", chat_request.workflow_permanent_id)apply_context_attrs(span)- yield+ yield spandef _resolve_request_policy_handler(fallback_handler: Any) -> Any:@@ -150,6 +157,10 @@ class RequestPolicyGuardrailInputs:browser_session_id: str | None = None+class CopilotRequestPolicyMissingError(Exception):+ """Raised when the request-policy guardrail fails before producing a policy."""++async def _resolve_live_browser_session_id(chat_request: WorkflowCopilotChatRequest,organization_id: str,@@ -675,7 +686,6 @@ _MAX_TURNS_REPLY_UNVALIDATED = (_MAX_TURNS_REPLY_TESTED = ("I've reached the maximum number of steps, but I have a tested draft for you. Accept it to save, or discard.")-_UNEXPECTED_ERROR_REPLY_DEFAULT = "An unexpected error occurred. Please try again."_UNEXPECTED_ERROR_REPLY_UNVALIDATED = ("I hit an unexpected issue before I could finish testing. I have a draft workflow you can keep — ""accept it to save (note: it hasn't been verified end-to-end), or discard."@@ -715,7 +725,6 @@ _UNVALIDATED_PROPOSAL_AFFORDANCE = ("I have a draft workflow proposal. Use Review to inspect it, Accept to save it, or Reject to discard it. ""It has not been tested or verified end-to-end.")-_RECORDED_FAILURE_URL_RE = re.compile(r"https?://[^\s)>,]+")def _workflow_block_count(ctx: CopilotContext) -> int | None:@@ -729,16 +738,8 @@ def _workflow_block_count(ctx: CopilotContext) -> int | None:def _clean_recorded_failure_text(value: Any, max_chars: int = 240) -> str:- from skyvern.forge.sdk.copilot.enforcement import redact_browser_session_references-- if not isinstance(value, str):- return ""- text = redact_raw_secrets_for_prompt(" ".join(value.split()))- text = redact_browser_session_references(text)- text = _RECORDED_FAILURE_URL_RE.sub(lambda match: url_origin(match.group(0)) or "[URL]", text)- if len(text) > max_chars:- text = text[: max_chars - 3].rstrip() + "..."- return text.rstrip(".")+ # Caller-owned sentence templates add punctuation around these fragments.+ return clean_recorded_failure_text(value, max_chars=max_chars).rstrip(".")def _recorded_failure_summary(ctx: CopilotContext) -> tuple[str, str]:@@ -870,6 +871,16 @@ def _build_wip_exit_result(return _build_exit_result(ctx, recorded_failure_reply or default_reply, global_llm_context, cancelled=cancelled)+def _merge_exit_context(+ global_llm_context: str | None,+ *,+ failure: RecoverableFailure | None = None,+) -> str | None:+ if failure is None:+ return global_llm_context+ return merge_failure_into_context(global_llm_context, failure)++def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:return _build_wip_exit_result(ctx,@@ -897,14 +908,48 @@ def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str |)-def _build_unexpected_error_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:- return _build_wip_exit_result(+def _build_unexpected_error_exit_result(+ ctx: CopilotContext,+ global_llm_context: str | None,+ error: BaseException | None = None,+ *,+ span: Any | None = None,+) -> AgentResult:+ failure = build_recoverable_failure(+ error,+ workflow_modified=ctx.workflow_persisted,+ )+ default_reply = format_recoverable_failure_reply(failure)+ enriched_context = _merge_exit_context(global_llm_context, failure=failure)+ result = _build_wip_exit_result(ctx,- global_llm_context,- default_reply=_UNEXPECTED_ERROR_REPLY_DEFAULT,+ enriched_context,+ default_reply=default_reply,unvalidated_reply=_UNEXPECTED_ERROR_REPLY_UNVALIDATED,tested_reply=_UNEXPECTED_ERROR_REPLY_TESTED,)+ LOG.warning(+ "Copilot unexpected error translated to recoverable reply",+ failure_kind=failure.failure_kind,+ internal_error_id=failure.internal_error_id,+ exception_type=failure.exception_type,+ error_type=type(error).__name__ if error else None,+ workflow_permanent_id=getattr(ctx, "workflow_permanent_id", None),+ workflow_copilot_chat_id=getattr(ctx, "workflow_copilot_chat_id", None),+ workflow_modified=failure.workflow_modified,+ has_proposal=result.updated_workflow is not None,+ proposal_disposition=result.proposal_disposition,+ last_test_ok=getattr(ctx, "last_test_ok", None),+ )+ current_span = span or otel_trace.get_current_span()+ current_span.set_attribute("copilot.error_recovered", True)+ current_span.set_attribute("copilot.error_failure_kind", failure.failure_kind)+ current_span.set_attribute("copilot.error_id", failure.internal_error_id)+ if failure.exception_type:+ current_span.set_attribute("copilot.error_exception_type", failure.exception_type)+ current_span.set_attribute("copilot.error_reply_proposal_disposition", result.proposal_disposition)+ current_span.set_attribute("copilot.error_workflow_modified", failure.workflow_modified)+ return resultdef _build_cancel_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:@@ -1237,17 +1282,8 @@ _RETRIABLE_LLM_ERROR_TEXT = (_LLM_ERROR_MODULE_MARKERS = ("openai", "litellm", "anthropic")-def _iter_exception_chain(exc: BaseException) -> list[BaseException]:- chain: list[BaseException] = []- current: BaseException | None = exc- while current is not None and current not in chain:- chain.append(current)- current = current.__cause__ or current.__context__- return chain--def _is_retriable_llm_error(exc: BaseException) -> bool:- for item in _iter_exception_chain(exc):+ for item in iter_exception_chain(exc):module = type(item).__module__.lower()name = type(item).__name__text = str(item).lower()@@ -1605,28 +1641,71 @@ async def run_copilot_agent(turn_index: int | None = None,prior_copilot_workflow_yaml: str | None = None,) -> AgentResult:- # Initialize tracing before opening the turn span so Logfire's OTel provider- # is installed; otherwise the very first turn lands the parent span on- # OTel's no-op ProxyTracer when running locally with COPILOT_TRACING_ENABLED.- ensure_tracing_initialized()- with _copilot_turn_span(- chat_request=chat_request,- chat_history=chat_history,- turn_index=turn_index,- ):- return await _run_copilot_turn_impl(- stream=stream,- organization_id=organization_id,+ try:+ # Initialize tracing before opening the turn span so Logfire's OTel provider+ # is installed; otherwise the very first turn lands the parent span on+ # OTel's no-op ProxyTracer when running locally with COPILOT_TRACING_ENABLED.+ ensure_tracing_initialized()+ with _copilot_turn_span(chat_request=chat_request,chat_history=chat_history,- global_llm_context=global_llm_context,- debug_run_info_text=debug_run_info_text,- llm_api_handler=llm_api_handler,+ turn_index=turn_index,+ ) as turn_span:+ try:+ return await _run_copilot_turn_impl(+ stream=stream,+ organization_id=organization_id,+ chat_request=chat_request,+ chat_history=chat_history,+ global_llm_context=global_llm_context,+ debug_run_info_text=debug_run_info_text,+ llm_api_handler=llm_api_handler,+ api_key=api_key,+ security_rules=security_rules,+ config=config,+ prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,+ )+ except Exception as exc:+ LOG.error(+ "Copilot turn unhandled error",+ error_type=type(exc).__name__,+ workflow_permanent_id=chat_request.workflow_permanent_id,+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ exc_info=True,+ )+ turn_span.record_exception(exc)+ ctx = CopilotContext(+ organization_id=organization_id,+ workflow_id=chat_request.workflow_id,+ workflow_permanent_id=chat_request.workflow_permanent_id,+ workflow_yaml=chat_request.workflow_yaml or "",+ browser_session_id=None,+ stream=stream,+ api_key=api_key,+ user_message=chat_request.message,+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ )+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=exc, span=turn_span)+ except Exception as exc:+ LOG.error(+ "Copilot turn unhandled error",+ error_type=type(exc).__name__,+ workflow_permanent_id=chat_request.workflow_permanent_id,+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ exc_info=True,+ )+ ctx = CopilotContext(+ organization_id=organization_id,+ workflow_id=chat_request.workflow_id,+ workflow_permanent_id=chat_request.workflow_permanent_id,+ workflow_yaml=chat_request.workflow_yaml or "",+ browser_session_id=None,+ stream=stream,api_key=api_key,- security_rules=security_rules,- config=config,- prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,+ user_message=chat_request.message,+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,)+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=exc)async def _run_copilot_turn_impl(@@ -1734,7 +1813,7 @@ async def _run_copilot_turn_impl(prior_workflow_yaml=chat_request.workflow_yaml,)if request_policy is None:- raise RuntimeError("Copilot request-policy input guardrail did not populate request policy")+ raise CopilotRequestPolicyMissingError()agent_user_message, safe_chat_history_text = _request_policy_agent_inputs(request_policy,@@ -1968,7 +2047,7 @@ async def _run_copilot_turn_impl(error_message=exc.error_message,organization_id=organization_id,)- return _build_unexpected_error_exit_result(ctx, global_llm_context)+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=exc)except CopilotNonRetriableNavError as exc:LOG.warning("Copilot run halted on non-retriable navigation error",@@ -1991,4 +2070,4 @@ async def _run_copilot_turn_impl()except Exception as e:LOG.error("Copilot agent error", error=str(e), exc_info=True)- return _build_unexpected_error_exit_result(ctx, global_llm_context)+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=e)diff --git a/skyvern/forge/sdk/copilot/recoverable_failure.py b/skyvern/forge/sdk/copilot/recoverable_failure.pynew file mode 100644index 000000000..8834afc5f--- /dev/null+++ b/skyvern/forge/sdk/copilot/recoverable_failure.py@@ -0,0 +1,149 @@+"""Recoverable Copilot failure rendering and telemetry helpers."""++from __future__ import annotations++import re+import uuid+from collections.abc import Iterable+from dataclasses import dataclass+from typing import Literal++import yaml+from pydantic import ValidationError++from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError+from skyvern.forge.sdk.copilot.context import StructuredContext+from skyvern.forge.sdk.copilot.output_policy import url_origin+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt+from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException++RecoverableFailureKind = Literal["validation", "tool_call", "external_dep", "timeout", "unknown"]++_REASON_SUMMARY_MAX_CHARS = 120+_RECOVERABLE_ERROR_ID_PREFIX = "cpe"+_RECORDED_FAILURE_URL_RE = re.compile(r"https?://[^\s)>,]+")+_RECORDED_FAILURE_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")+_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")+_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)+++@dataclass(frozen=True)+class RecoverableFailure:+ failure_kind: RecoverableFailureKind+ workflow_modified: bool+ reason_summary: str+ internal_error_id: str+ exception_type: str | None = None+++def iter_exception_chain(exc: BaseException) -> Iterable[BaseException]:+ seen: set[int] = set()+ current: BaseException | None = exc+ while current is not None and id(current) not in seen:+ seen.add(id(current))+ yield current+ current = current.__cause__ or current.__context__+++def _new_internal_error_id() -> str:+ return f"{_RECOVERABLE_ERROR_ID_PREFIX}_{uuid.uuid4().hex[:12]}"+++def _redact_browser_session_references(value: str) -> str:+ value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)+ return _BROWSER_SESSION_ID_RE.sub("the browser session", value)+++def _is_exception_named(exc: BaseException, name: str) -> bool:+ # Keep this helper import-light for the base/local wheel: importing+ # copilot.enforcement here would pull in the server-only agents package.+ return type(exc).__name__ == name+++def clean_recorded_failure_text(value: object, max_chars: int = _REASON_SUMMARY_MAX_CHARS) -> str:+ if not isinstance(value, str):+ return ""+ text = value.strip()+ if not text:+ return ""+ text = redact_raw_secrets_for_prompt(" ".join(text.split()))+ text = _redact_browser_session_references(text)+ text = _RECORDED_FAILURE_CREDENTIAL_ID_RE.sub("[CREDENTIAL_ID]", text)+ text = _RECORDED_FAILURE_URL_RE.sub(lambda match: url_origin(match.group(0)) or "[URL]", text)+ if len(text) > max_chars:+ text = text[: max_chars - 3].rstrip() + "..."+ return text+++def _reason_summary(value: str) -> str:+ return clean_recorded_failure_text(value, max_chars=_REASON_SUMMARY_MAX_CHARS)+++def _failure_kind_for_exception(error: BaseException | None) -> RecoverableFailureKind:+ if error is None:+ return "unknown"+ if any(_is_exception_named(item, "CopilotTotalTimeoutError") for item in iter_exception_chain(error)):+ return "timeout"+ if any(+ _is_exception_named(item, "CopilotUnrecoverableToolError")+ or _is_exception_named(item, "CopilotNonRetriableNavError")+ for item in iter_exception_chain(error)+ ):+ return "tool_call"+ if any(+ isinstance(item, (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException))+ for item in iter_exception_chain(error)+ ):+ return "validation"+ if any(isinstance(item, LLMProviderError) for item in iter_exception_chain(error)):+ return "external_dep"+ return "unknown"+++def _reason_for_failure(error: BaseException | None, failure_kind: RecoverableFailureKind) -> str:+ if failure_kind == "timeout":+ return "Copilot timed out before it could finish this turn"+ if failure_kind == "validation":+ return "The workflow update could not be validated"+ if failure_kind == "external_dep":+ return "A Copilot dependency stopped responding"+ if failure_kind == "tool_call":+ chain = list(iter_exception_chain(error)) if error is not None else []+ if any(_is_exception_named(item, "CopilotNonRetriableNavError") for item in chain):+ return "A browser navigation step could not reach the target URL"+ return "A browser or workflow helper stopped responding"+ return "Copilot hit an internal error before it could finish this turn"+++def build_recoverable_failure(+ error: BaseException | None,+ *,+ workflow_modified: bool,+ internal_error_id: str | None = None,+) -> RecoverableFailure:+ failure_kind = _failure_kind_for_exception(error)+ reason_summary = _reason_summary(_reason_for_failure(error, failure_kind))+ exception_type = type(error).__name__ if error is not None else None+ return RecoverableFailure(+ failure_kind=failure_kind,+ workflow_modified=workflow_modified,+ reason_summary=reason_summary,+ internal_error_id=internal_error_id or _new_internal_error_id(),+ exception_type=exception_type,+ )+++def format_recoverable_failure_reply(failure: RecoverableFailure) -> str:+ workflow_state = "preserved" if failure.workflow_modified else "not modified"+ return (+ f"{failure.reason_summary}. The workflow was {workflow_state}. "+ f"If this persists, reference {failure.internal_error_id} for support."+ )+++def merge_failure_into_context(global_llm_context: str | None, failure: RecoverableFailure) -> str:+ structured = StructuredContext.from_json_str(global_llm_context)+ note = f"copilot turn failed: {failure.failure_kind} {failure.internal_error_id}"+ if note not in structured.decisions_made:+ structured.decisions_made.append(note)+ return structured.to_json_str()diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 684c4bb24..c670a5d57 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -52,6 +52,8 @@ from skyvern.forge.sdk.db.enums import OrganizationAuthTokenTypefrom skyvern.forge.sdk.executor.factory import AsyncExecutorFactoryfrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.routes.code_samples import (+ BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON,+ BULK_CANCEL_RUNS_CODE_SAMPLE_TS,CANCEL_RUN_CODE_SAMPLE_PYTHON,CANCEL_RUN_CODE_SAMPLE_TS,CREATE_WORKFLOW_CODE_SAMPLE_CURL,@@ -121,6 +123,8 @@ from skyvern.schemas.runs import (CUA_ENGINES,BlockRunRequest,BlockRunResponse,+ BulkCancelRunsRequest,+ BulkCancelRunsResponse,RunEngine,RunResponse,RunStatus,@@ -570,6 +574,36 @@ async def cancel_run(await run_service.cancel_run(run_id, organization_id=current_org.organization_id, api_key=x_api_key)+@base_router.post(+ "/runs/cancel",+ tags=["Agent", "Workflow Runs"],+ openapi_extra={+ "x-fern-sdk-method-name": "bulk_cancel_runs",+ "x-fern-examples": [+ {+ "code-samples": [+ {"sdk": "python", "code": BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON},+ {"sdk": "typescript", "code": BULK_CANCEL_RUNS_CODE_SAMPLE_TS},+ ]+ }+ ],+ },+ description="Cancel multiple runs (tasks or workflows) in a single request",+ summary="Bulk cancel runs",+)+@base_router.post("/runs/cancel/", include_in_schema=False)+async def bulk_cancel_runs(+ data: BulkCancelRunsRequest = Body(...),+ current_org: Organization = Depends(org_auth_service.get_current_org),+ x_api_key: Annotated[str | None, Header()] = None,+) -> BulkCancelRunsResponse:+ analytics.capture("skyvern-oss-agent-bulk-cancel-runs")++ return await run_service.bulk_cancel_runs(+ data.run_ids, organization_id=current_org.organization_id, api_key=x_api_key+ )++@legacy_base_router.post("/workflows",openapi_extra={diff --git a/skyvern/forge/sdk/routes/code_samples.py b/skyvern/forge/sdk/routes/code_samples.pyindex 71ee7dc69..43df0a27a 100644--- a/skyvern/forge/sdk/routes/code_samples.py+++ b/skyvern/forge/sdk/routes/code_samples.py@@ -50,6 +50,16 @@ CANCEL_RUN_CODE_SAMPLE_TS = """import { SkyvernClient } from "@skyvern/client";const skyvern = new SkyvernClient({ apiKey: "YOUR_API_KEY" });await skyvern.cancelRun("tsk_v2_123");"""+BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON = """from skyvern import Skyvern++skyvern = Skyvern(api_key="YOUR_API_KEY")+await skyvern.bulk_cancel_runs(run_ids=["tsk_v2_123", "wr_456"])+"""+BULK_CANCEL_RUNS_CODE_SAMPLE_TS = """import { SkyvernClient } from "@skyvern/client";++const skyvern = new SkyvernClient({ apiKey: "YOUR_API_KEY" });+await skyvern.bulkCancelRuns({ run_ids: ["tsk_v2_123", "wr_456"] });+"""RETRY_RUN_WEBHOOK_CODE_SAMPLE_PYTHON = """from skyvern import Skyvernskyvern = Skyvern(api_key="YOUR_API_KEY")diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 25bbeba33..19b829661 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -11,6 +11,7 @@ from typing import Any, Iteratorimport structlogimport yamlfrom fastapi import Depends, HTTPException, Request, status+from opentelemetry import trace as otel_tracefrom pydantic import ValidationErrorfrom sse_starlette import EventSourceResponse@@ -27,6 +28,12 @@ from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_blockfrom skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.context import AgentResult, ProposalDispositionfrom skyvern.forge.sdk.copilot.output_utils import truncate_output+from skyvern.forge.sdk.copilot.recoverable_failure import (+ RecoverableFailure,+ build_recoverable_failure,+ format_recoverable_failure_reply,+ merge_failure_into_context,+)from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_typefrom skyvern.forge.sdk.routes.event_source_stream import EventSourceStream, FastAPIEventSourceStream@@ -292,6 +299,41 @@ def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: oreturn not _effective_auto_accept(auto_accept, agent_result)+def _record_recoverable_failure_span_attrs(+ failure: RecoverableFailure,+ *,+ proposal_disposition: ProposalDisposition,+) -> None:+ current_span = otel_trace.get_current_span()+ current_span.set_attribute("copilot.error_recovered", True)+ current_span.set_attribute("copilot.error_failure_kind", failure.failure_kind)+ current_span.set_attribute("copilot.error_id", failure.internal_error_id)+ if failure.exception_type:+ current_span.set_attribute("copilot.error_exception_type", failure.exception_type)+ current_span.set_attribute("copilot.error_workflow_modified", failure.workflow_modified)+ current_span.set_attribute("copilot.error_reply_proposal_disposition", proposal_disposition)+++def _build_recoverable_route_agent_result(+ error: BaseException,+ *,+ workflow_modified: bool,+ clear_proposed_workflow: bool,+ global_llm_context: str | None,+) -> tuple[AgentResult, RecoverableFailure]:+ failure = build_recoverable_failure(error, workflow_modified=workflow_modified)+ agent_result = AgentResult(+ user_response=format_recoverable_failure_reply(failure),+ updated_workflow=None,+ global_llm_context=merge_failure_into_context(global_llm_context, failure),+ workflow_was_persisted=False,+ proposal_disposition="no_proposal",+ clear_proposed_workflow=clear_proposed_workflow,+ )+ _record_recoverable_failure_span_attrs(failure, proposal_disposition="no_proposal")+ return agent_result, failure++async def _clear_proposed_workflow(chat: Any) -> None:await app.DATABASE.workflow_params.update_workflow_copilot_chat(organization_id=chat.organization_id,@@ -1315,6 +1357,7 @@ async def _new_copilot_chat_post(original_workflow: Workflow | None = Nonechat = Noneagent_result: AgentResult | None = None+ global_llm_context: str | None = Noneterminal_frame_emitted = Falsecancel_watcher: asyncio.Task[None] | None = None# Single-element list used as a closure flag (mutable bool by reference).@@ -1353,7 +1396,6 @@ async def _new_copilot_chat_post(chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(workflow_copilot_chat_id=chat.workflow_copilot_chat_id,)- global_llm_context = Nonefor message in reversed(chat_messages):if message.global_llm_context is not None:global_llm_context = message.global_llm_context@@ -1527,21 +1569,51 @@ async def _new_copilot_chat_post())except LLMProviderError as exc:- if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):+ restored = chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result)+ if restored:await _restore_workflow_definition(original_workflow, organization.organization_id)- LOG.error(- "LLM provider error (copilot v2)",- organization_id=organization.organization_id,- error=str(exc),- exc_info=True,- )- terminal_frame_emitted = True- await stream.send(- WorkflowCopilotStreamErrorUpdate(- type=WorkflowCopilotStreamMessageType.ERROR,- error="Failed to process your request. Please try again.",+ if chat is not None:+ workflow_modified = bool(getattr(agent_result, "workflow_was_persisted", False)) and not restored+ recovered_result, failure = _build_recoverable_route_agent_result(+ exc,+ workflow_modified=workflow_modified,+ clear_proposed_workflow=restored or workflow_modified,+ global_llm_context=global_llm_context,+ )+ LOG.error(+ "LLM provider error translated to recoverable workflow copilot v2 reply",+ organization_id=organization.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ failure_kind=failure.failure_kind,+ internal_error_id=failure.internal_error_id,+ exception_type=failure.exception_type,+ exc_info=True,+ )+ await asyncio.shield(+ _finalise_normal_turn(+ stream=stream,+ chat=chat,+ organization_id=organization.organization_id,+ original_workflow=original_workflow,+ chat_request=chat_request,+ agent_result=recovered_result,+ )+ )+ terminal_frame_emitted = True+ else:+ LOG.error(+ "LLM provider error (copilot v2)",+ organization_id=organization.organization_id,+ error=str(exc),+ exc_info=True,+ )+ terminal_frame_emitted = True+ await stream.send(+ WorkflowCopilotStreamErrorUpdate(+ type=WorkflowCopilotStreamMessageType.ERROR,+ error="Failed to process your request. Please try again.",+ ))- )except asyncio.CancelledError:if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):await asyncio.shield(_restore_workflow_definition(original_workflow, organization.organization_id))@@ -1578,21 +1650,51 @@ async def _new_copilot_chat_post()raiseexcept Exception as exc:- if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):+ restored = chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result)+ if restored:await _restore_workflow_definition(original_workflow, organization.organization_id)- LOG.error(- "Unexpected error in workflow copilot v2",- organization_id=organization.organization_id,- error=str(exc),- exc_info=True,- )- terminal_frame_emitted = True- await stream.send(- WorkflowCopilotStreamErrorUpdate(- type=WorkflowCopilotStreamMessageType.ERROR,- error="An error occurred. Please try again.",+ if chat is not None:+ workflow_modified = bool(getattr(agent_result, "workflow_was_persisted", False)) and not restored+ recovered_result, failure = _build_recoverable_route_agent_result(+ exc,+ workflow_modified=workflow_modified,+ clear_proposed_workflow=restored or workflow_modified,+ global_llm_context=global_llm_context,+ )+ LOG.error(+ "Unexpected workflow copilot v2 error translated to recoverable reply",+ organization_id=organization.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ failure_kind=failure.failure_kind,+ internal_error_id=failure.internal_error_id,+ exception_type=failure.exception_type,+ exc_info=True,+ )+ await asyncio.shield(+ _finalise_normal_turn(+ stream=stream,+ chat=chat,+ organization_id=organization.organization_id,+ original_workflow=original_workflow,+ chat_request=chat_request,+ agent_result=recovered_result,+ )+ )+ terminal_frame_emitted = True+ else:+ LOG.error(+ "Unexpected error in workflow copilot v2",+ organization_id=organization.organization_id,+ error=str(exc),+ exc_info=True,+ )+ terminal_frame_emitted = True+ await stream.send(+ WorkflowCopilotStreamErrorUpdate(+ type=WorkflowCopilotStreamMessageType.ERROR,+ error="An error occurred. Please try again.",+ ))- )finally:if cancel_watcher is not None and not cancel_watcher.done():cancel_watcher.cancel()diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.pyindex f446aa498..80837c352 100644--- a/skyvern/schemas/runs.py+++ b/skyvern/schemas/runs.py@@ -475,3 +475,12 @@ class TaskRunListItem(BaseModel):endpoint's Run.script_run field. Do not rely on dict contents here."""return bool(v)+++class BulkCancelRunsRequest(BaseModel):+ run_ids: list[str] = Field(max_length=100, description="List of run IDs to cancel")+++class BulkCancelRunsResponse(BaseModel):+ cancelled: list[str] = Field(description="Run IDs that were successfully cancelled")+ failed: list[str] = Field(description="Run IDs that could not be cancelled")diff --git a/skyvern/services/run_service.py b/skyvern/services/run_service.pyindex c732ca4b3..8ead2420e 100644--- a/skyvern/services/run_service.py+++ b/skyvern/services/run_service.py@@ -1,3 +1,6 @@+import asyncio++import structlogfrom fastapi import HTTPException, statusfrom skyvern.config import settings@@ -5,10 +8,19 @@ from skyvern.exceptions import OrganizationNotFound, TaskNotFound, WorkflowRunNofrom skyvern.forge import appfrom skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus-from skyvern.schemas.runs import RunEngine, RunResponse, RunType, TaskRunRequest, TaskRunResponse+from skyvern.schemas.runs import (+ BulkCancelRunsResponse,+ RunEngine,+ RunResponse,+ RunType,+ TaskRunRequest,+ TaskRunResponse,+)from skyvern.schemas.webhooks import RunWebhookReplayResponsefrom skyvern.services import task_v1_service, task_v2_service, webhook_service, workflow_service+LOG = structlog.get_logger()+async def get_run_response(run_id: str, organization_id: str | None = None) -> RunResponse | None:run = await app.DATABASE.tasks.get_run(run_id, organization_id=organization_id)@@ -150,6 +162,24 @@ async def cancel_run(run_id: str, organization_id: str | None = None, api_key: s)+async def bulk_cancel_runs(+ run_ids: list[str], organization_id: str | None = None, api_key: str | None = None+) -> BulkCancelRunsResponse:+ cancelled: list[str] = []+ failed: list[str] = []++ async def _cancel_one(run_id: str) -> None:+ try:+ await cancel_run(run_id, organization_id=organization_id, api_key=api_key)+ cancelled.append(run_id)+ except Exception:+ LOG.warning("bulk_cancel_runs: failed to cancel run", run_id=run_id, exc_info=True)+ failed.append(run_id)++ await asyncio.gather(*[_cancel_one(run_id) for run_id in dict.fromkeys(run_ids)])+ return BulkCancelRunsResponse(cancelled=cancelled, failed=failed)++async def retry_run_webhook(run_id: str,organization_id: str | None = None,diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 451f02f9f..398ad93b5 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -8,9 +8,17 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMockimport pytest+import yaml+from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.enforcement import (+ CopilotNonRetriableNavError,+ CopilotTotalTimeoutError,+ CopilotUnrecoverableToolError,+)+from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failurefrom skyvern.forge.sdk.copilot.request_policy import (_REDACTED_REFUSED_SECRET_TURN,TRANSCRIPT_ANCHOR_CHAR_CAP,@@ -918,6 +926,107 @@ workflow_definition:)assert "same IP/workflow shape" in agent_result.user_response+ def test_unexpected_error_exit_names_failure_and_preserves_context(self) -> None:+ ctx = _ctx()++ agent_result = agent_module._build_unexpected_error_exit_result(+ ctx,+ global_llm_context=None,+ error=agent_module.CopilotRequestPolicyMissingError(),+ )++ assert "An unexpected error occurred. Please try again." not in agent_result.user_response+ assert "Copilot hit an internal error before it could finish this turn" in agent_result.user_response+ assert "The workflow was not modified" in agent_result.user_response+ assert "reference cpe_" in agent_result.user_response+ assert "copilot turn failed: unknown cpe_" in (agent_result.global_llm_context or "")+ assert agent_result.updated_workflow is None++ def test_unexpected_error_exit_redacts_sensitive_identifiers(self) -> None:+ ctx = _ctx(workflow_persisted=True)++ agent_result = agent_module._build_unexpected_error_exit_result(+ ctx,+ global_llm_context=None,+ error=RuntimeError("credential cred_12345 was rejected while opening https://example.com/private/path"),+ )++ assert "cred_12345" not in agent_result.user_response+ assert "https://example.com/private/path" not in agent_result.user_response+ assert "credential" not in agent_result.user_response.lower()+ assert "https://example.com" not in agent_result.user_response+ assert "Copilot hit an internal error before it could finish this turn" in agent_result.user_response+ assert "The workflow was preserved" in agent_result.user_response+ assert "reference cpe_" in agent_result.user_response++ def test_unexpected_error_exit_does_not_persist_tool_output_preview(self) -> None:+ ctx = _ctx()+ ctx.tool_activity.append(+ {+ "tool": "get_run_results",+ "summary": "OK",+ "output_preview": "block_1: [email protected] password=hunter2",+ }+ )++ agent_result = agent_module._build_unexpected_error_exit_result(+ ctx,+ global_llm_context=None,+ error=RuntimeError("boom"),+ )++ assert "[email protected]" not in (agent_result.global_llm_context or "")+ assert "hunter2" not in (agent_result.global_llm_context or "")+ assert "copilot turn failed: unknown cpe_" in (agent_result.global_llm_context or "")++ def test_recoverable_failure_maps_expected_exception_families(self) -> None:+ assert (+ build_recoverable_failure(+ CopilotTotalTimeoutError(),+ workflow_modified=False,+ internal_error_id="cpe_timeout",+ ).failure_kind+ == "timeout"+ )+ assert (+ build_recoverable_failure(+ CopilotUnrecoverableToolError("click", "browser failed"),+ workflow_modified=False,+ internal_error_id="cpe_tool",+ ).failure_kind+ == "tool_call"+ )+ assert (+ build_recoverable_failure(+ yaml.YAMLError("bad yaml"),+ workflow_modified=False,+ internal_error_id="cpe_validation",+ ).failure_kind+ == "validation"+ )+ assert (+ build_recoverable_failure(+ LLMProviderError("OPENAI_GPT5_5"),+ workflow_modified=False,+ internal_error_id="cpe_external",+ ).failure_kind+ == "external_dep"+ )++ def test_recoverable_failure_uses_chained_navigation_reason(self) -> None:+ nav_error = CopilotNonRetriableNavError("https://example.com", "net::ERR_NAME_NOT_RESOLVED")+ wrapper = RuntimeError("wrapped")+ wrapper.__cause__ = nav_error++ failure = build_recoverable_failure(+ wrapper,+ workflow_modified=False,+ internal_error_id="cpe_nav",+ )++ assert failure.failure_kind == "tool_call"+ assert failure.reason_summary == "A browser navigation step could not reach the target URL"+def test_reply_still_rewrites_after_failed_test(self) -> None:ctx = _ctx(last_update_block_count=2,diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex 27c9a59e1..d51eacc7a 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -14,7 +14,6 @@ from skyvern.forge.sdk.copilot.agent import (_TIMEOUT_REPLY_DEFAULT,_TIMEOUT_REPLY_TESTED,_TIMEOUT_REPLY_UNVALIDATED,- _UNEXPECTED_ERROR_REPLY_DEFAULT,_UNEXPECTED_ERROR_REPLY_TESTED,_UNEXPECTED_ERROR_REPLY_UNVALIDATED,_build_cancel_exit_result,@@ -51,6 +50,7 @@ def _ctx(ctx.total_tokens_used = Nonectx.last_good_workflow = last_good_workflowctx.last_good_workflow_yaml = last_good_workflow_yaml+ ctx.tool_activity = []ctx.latest_diagnosis_repair_contract = Nonectx.test_after_update_done = last_test_ok is not Nonectx.last_update_block_count = None@@ -244,7 +244,10 @@ class TestBuildUnexpectedErrorExitResult:assert result.updated_workflow is Noneassert result.workflow_yaml is Noneassert result.unvalidated is False- assert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT+ assert "An unexpected error occurred. Please try again." not in result.user_response+ assert "Copilot hit an internal error before it could finish this turn" in result.user_response+ assert "The workflow was not modified" in result.user_response+ assert "reference cpe_" in result.user_responsedef test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:wf = MagicMock(name="wf")@@ -277,7 +280,9 @@ class TestBuildUnexpectedErrorExitResult:assert result.updated_workflow is Noneassert result.workflow_yaml is Noneassert result.unvalidated is False- assert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT+ assert "Copilot hit an internal error before it could finish this turn" in result.user_response+ assert "The workflow was preserved" in result.user_response+ assert "reference cpe_" in result.user_responsedef test_failed_test_uses_recorded_blocker_reply(self) -> None:wf = MagicMock(name="wf")@@ -346,7 +351,9 @@ class TestBuildUnexpectedErrorExitResult:assert result.updated_workflow is Noneassert result.workflow_yaml is Noneassert result.unvalidated is False- assert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT+ assert "Copilot hit an internal error before it could finish this turn" in result.user_response+ assert "The workflow was preserved" in result.user_response+ assert "reference cpe_" in result.user_responseclass TestBuildCancelExitResult:diff --git a/tests/unit/test_copilot_turn_span.py b/tests/unit/test_copilot_turn_span.pyindex 717248a9d..b6ee526d8 100644--- a/tests/unit/test_copilot_turn_span.py+++ b/tests/unit/test_copilot_turn_span.py@@ -57,6 +57,10 @@ async def _stub_build_request_policy_proceed(*_args: Any, **_kwargs: Any) -> Reqreturn RequestPolicy(user_response_policy="ask_clarification", clarification_question="?")+async def _raise_unhandled_turn_error(*_args: Any, **_kwargs: Any) -> None:+ raise RuntimeError("boom")++@pytest.fixturedef patched_build_request_policy(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(@@ -181,6 +185,35 @@ async def test_copilot_turn_span_preview_is_single_line_and_truncated(assert preview.endswith("…")+async def test_outer_turn_recovery_records_error_attrs_on_turn_span(+ span_exporter: Any,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(copilot_agent, "_run_copilot_turn_impl", _raise_unhandled_turn_error)+ chat_request = _make_chat_request()++ result = await copilot_agent.run_copilot_agent(+ stream=object(),+ organization_id="o_test",+ chat_request=chat_request,+ chat_history=[],+ global_llm_context=None,+ debug_run_info_text="",+ llm_api_handler=None,+ )++ assert result.updated_workflow is None+ assert "reference cpe_" in result.user_response+ turn_span = _find_span(span_exporter.get_finished_spans(), "copilot.turn")+ attrs = dict(turn_span.attributes or {})+ assert attrs.get("copilot.error_recovered") is True+ assert attrs.get("copilot.error_failure_kind") == "unknown"+ assert attrs.get("copilot.error_exception_type") == "RuntimeError"+ assert attrs.get("copilot.error_reply_proposal_disposition") == result.proposal_disposition+ assert attrs.get("copilot.error_workflow_modified") is False++def test_build_user_message_preview_redacts_known_secret_patterns() -> None:preview = copilot_agent._build_user_message_preview("password: hunter2 please")assert "hunter2" not in previewdiff --git a/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.pyindex ddf3ca2be..45d893fac 100644--- a/tests/unit/test_quickstart.py+++ b/tests/unit/test_quickstart.py@@ -1,6 +1,7 @@from __future__ import annotationsimport subprocess+from pathlib import Pathimport pytestimport typer@@ -73,3 +74,115 @@ def test_quickstart_reraises_intentional_typer_exit(monkeypatch: pytest.MonkeyPa)assert capture_errors == []+++def test_bootstrap_creates_env_and_rewrites_localhost_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ example = tmp_path / ".env.example"+ example.write_text('KEY=value\nDATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\n')+ monkeypatch.chdir(tmp_path)++ quickstart._bootstrap_compose_env_files()++ result = (tmp_path / ".env").read_text()+ assert "localhost" not in result+ assert "KEY=value" in result+ assert 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"' in result+++def test_bootstrap_creates_env_without_db_string_unchanged(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ example = tmp_path / ".env.example"+ example.write_text("KEY=value\n")+ monkeypatch.chdir(tmp_path)++ quickstart._bootstrap_compose_env_files()++ assert (tmp_path / ".env").read_text() == "KEY=value\n"+++def test_bootstrap_does_not_overwrite_existing_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ (tmp_path / ".env.example").write_text("KEY=example\n")+ existing = tmp_path / ".env"+ existing.write_text("KEY=existing\n")+ monkeypatch.chdir(tmp_path)++ quickstart._bootstrap_compose_env_files()++ assert existing.read_text() == "KEY=existing\n"+++def test_bootstrap_creates_frontend_env_from_example(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ (tmp_path / ".env.example").write_text("")+ frontend_dir = tmp_path / "skyvern-frontend"+ frontend_dir.mkdir()+ (frontend_dir / ".env.example").write_text("VITE_KEY=val\n")+ monkeypatch.chdir(tmp_path)++ quickstart._bootstrap_compose_env_files()++ assert (frontend_dir / ".env").read_text() == "VITE_KEY=val\n"+++def test_bootstrap_does_not_overwrite_existing_frontend_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ (tmp_path / ".env.example").write_text("")+ frontend_dir = tmp_path / "skyvern-frontend"+ frontend_dir.mkdir()+ (frontend_dir / ".env.example").write_text("VITE_KEY=example\n")+ existing = frontend_dir / ".env"+ existing.write_text("VITE_KEY=existing\n")+ monkeypatch.chdir(tmp_path)++ quickstart._bootstrap_compose_env_files()++ assert existing.read_text() == "VITE_KEY=existing\n"+++def test_bootstrap_rewrites_localhost_db_string_when_confirmed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ env_content = 'ENABLE_OPENAI=false\nDATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\nENV=local\n'+ (tmp_path / ".env").write_text(env_content)+ monkeypatch.chdir(tmp_path)+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *_args, **_kwargs: True)++ quickstart._bootstrap_compose_env_files()++ result = (tmp_path / ".env").read_text()+ assert "localhost" not in result+ assert 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"' in result+ assert "ENABLE_OPENAI=false" in result+ assert "ENV=local" in result+++def test_bootstrap_rewrites_export_prefixed_db_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ env_content = 'export DATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\n'+ (tmp_path / ".env").write_text(env_content)+ monkeypatch.chdir(tmp_path)+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *_args, **_kwargs: True)++ quickstart._bootstrap_compose_env_files()++ result = (tmp_path / ".env").read_text()+ assert "localhost" not in result+ assert 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"' in result+++def test_bootstrap_keeps_localhost_db_string_when_declined(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ env_content = 'DATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\n'+ (tmp_path / ".env").write_text(env_content)+ monkeypatch.chdir(tmp_path)+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *_args, **_kwargs: False)++ quickstart._bootstrap_compose_env_files()++ assert (tmp_path / ".env").read_text() == env_content+++def test_bootstrap_skips_rewrite_for_non_localhost_db_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ env_content = 'DATABASE_STRING="postgresql+psycopg://skyvern@postgres/skyvern"\n'+ (tmp_path / ".env").write_text(env_content)+ confirm_calls: list[object] = []+ monkeypatch.chdir(tmp_path)+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *args, **kwargs: confirm_calls.append(args) or True)++ quickstart._bootstrap_compose_env_files()++ assert confirm_calls == [], "should not prompt when DATABASE_STRING does not point to localhost"+ assert (tmp_path / ".env").read_text() == env_contentdiff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex c5019d491..94dc14d06 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -23,6 +23,9 @@ import pytestfrom skyvern.config import settingsfrom skyvern.forge import app+from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError+from skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_routefrom skyvern.forge.sdk.routes.workflow_copilot import COPILOT_V2_FLAG_KEY, workflow_copilot_chat_postfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatRequest,@@ -177,6 +180,7 @@ def _setup_new_copilot_mocks(get_workflow_run_blocks=AsyncMock(return_value=[]),)app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")+ app.AGENT_FUNCTION.get_copilot_config = MagicMock(return_value=None)return restore_mock@@ -260,6 +264,244 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_autrestore_mock.assert_not_awaited()+async def test_flag_on_pre_agent_failure_persists_recoverable_reply(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )++ async def fake_llm_handler(*args: object, **kwargs: object) -> None:+ del args, kwargs+ return None++ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.get_llm_handler_for_prompt_type",+ fake_llm_handler,+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot._restore_workflow_definition",+ AsyncMock(),+ )+ monkeypatch.setattr(agent_module, "ensure_tracing_initialized", lambda: None)+ monkeypatch.setattr(+ agent_module,+ "_run_copilot_turn_impl",+ AsyncMock(side_effect=agent_module.CopilotRequestPolicyMissingError()),+ )++ workflow_params = SimpleNamespace(+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),+ get_workflow_copilot_chat_messages=AsyncMock(return_value=[]),+ update_workflow_copilot_chat=AsyncMock(),+ create_workflow_copilot_chat_message=AsyncMock(+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 14, tzinfo=timezone.utc))+ ),+ )+ app.DATABASE.workflow_params = workflow_params+ app.DATABASE.workflows = SimpleNamespace(+ get_workflow_by_permanent_id=AsyncMock(return_value=original_workflow),+ )+ app.DATABASE.observer = SimpleNamespace(+ get_workflow_run_blocks=AsyncMock(return_value=[]),+ )+ app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")+ app.AGENT_FUNCTION.get_copilot_config = MagicMock(return_value=None)+ app.AGENT_FUNCTION.resolve_org_api_key = AsyncMock(return_value="sk-test-key")++ request = MagicMock()+ request.headers = {}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)+ assert response is captured["sentinel"]++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ await handler(stream)++ contents = [+ call.kwargs.get("content") for call in workflow_params.create_workflow_copilot_chat_message.await_args_list+ ]+ assert "Please update it" in contents+ assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]+ assert len(assistant_contents) == 1+ assert "An unexpected error occurred. Please try again." not in assistant_contents[0]+ assert "Copilot hit an internal error before it could finish this turn" in assistant_contents[0]+ assert "The workflow was not modified" in assistant_contents[0]+ assert "reference cpe_" in assistant_contents[0]++ frames = [call.args[0] for call in stream.send.await_args_list if call.args]+ assert any(isinstance(frame, WorkflowCopilotStreamResponseUpdate) for frame in frames)+ assert not any(isinstance(frame, WorkflowCopilotStreamErrorUpdate) for frame in frames)+++ ("error", "expected_summary"),+ [+ (RuntimeError("route boom"), "Copilot hit an internal error before it could finish this turn"),+ (LLMProviderError("OPENAI_GPT5_5"), "A Copilot dependency stopped responding"),+ ],+)+async def test_flag_on_route_error_after_chat_persists_recoverable_reply(+ monkeypatch: pytest.MonkeyPatch,+ error: BaseException,+ expected_summary: str,+) -> None:+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ agent_result = SimpleNamespace(+ user_response="unused",+ updated_workflow=None,+ global_llm_context=None,+ workflow_yaml=None,+ workflow_was_persisted=False,+ clear_proposed_workflow=False,+ unvalidated=False,+ )+ _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",+ AsyncMock(side_effect=error),+ )++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test-key"}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)+ assert response is captured["sentinel"]++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ await handler(stream)++ contents = [+ call.kwargs.get("content")+ for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list+ ]+ assert "Please update it" in contents+ assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]+ assert len(assistant_contents) == 1+ assert expected_summary in assistant_contents[0]+ assert "The workflow was not modified" in assistant_contents[0]+ assert "reference cpe_" in assistant_contents[0]++ frames = [call.args[0] for call in stream.send.await_args_list if call.args]+ assert any(isinstance(frame, WorkflowCopilotStreamResponseUpdate) for frame in frames)+ assert not any(isinstance(frame, WorkflowCopilotStreamErrorUpdate) for frame in frames)+++async def test_route_error_after_restore_reports_workflow_not_modified(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow={"workflow_id": "stale"},+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ agent_result = SimpleNamespace(+ user_response="unused",+ updated_workflow=None,+ global_llm_context=None,+ workflow_yaml=None,+ workflow_was_persisted=True,+ clear_proposed_workflow=False,+ unvalidated=False,+ )+ restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ finalise_results: list[object] = []+ original_finalise = workflow_copilot_route._finalise_normal_turn++ async def flaky_finalise(*args: object, **kwargs: object) -> object:+ finalise_results.append(kwargs["agent_result"])+ if len(finalise_results) == 1:+ raise RuntimeError("post-agent route boom")+ return await original_finalise(*args, **kwargs)++ monkeypatch.setattr(workflow_copilot_route, "_finalise_normal_turn", flaky_finalise)++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test-key"}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)+ assert response is captured["sentinel"]++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ await handler(stream)++ restore_mock.assert_awaited_once()+ assert len(finalise_results) == 2+ recovered_result = finalise_results[1]+ assert "The workflow was not modified" in recovered_result.user_response+ assert "The workflow was preserved" not in recovered_result.user_response+ assert recovered_result.clear_proposed_workflow is True+ contents = [+ call.kwargs.get("content")+ for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list+ ]+ assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]+ assert len(assistant_contents) == 1+ assert "The workflow was not modified" in assistant_contents[0]+ assert "The workflow was preserved" not in assistant_contents[0]+ update_calls = app.DATABASE.workflow_params.update_workflow_copilot_chat.await_args_list+ clear_calls = [c for c in update_calls if c.kwargs.get("proposed_workflow") is None]+ assert len(clear_calls) == 1++@pytest.mark.asyncio@pytest.mark.parametrize((