Skip to content

Automate browser based workflows with AI

SKY-10124: Make Copilot unexpected errors recoverable (#6143)

Andrew Neilson46a287bacf56
diff --git a/skyvern/cli/quickstart.py b/skyvern/cli/quickstart.py
index aeefe2178..c59316151 100644
--- a/skyvern/cli/quickstart.py
+++ b/skyvern/cli/quickstart.py
@@ -3,6 +3,8 @@
import asyncio
import importlib
import importlib.metadata
+import re
+import shutil
import subprocess
import sys
from enum import Enum
@@ -556,6 +558,62 @@ def _configure_local_browser_streaming_defaults() -> None:
set_key(str(frontend_env), "VITE_BROWSER_STREAMING_MODE", "cdp", quote_mode="never")
+_LOCALHOST_DB_RE = re.compile(r"^\s*(?:export\s+)?DATABASE_STRING\s*=.*@localhost[/:\"]", re.IGNORECASE)
+_COMPOSE_DATABASE_STRING = 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"'
+
+
+def _rewrite_localhost_db_string(content: str) -> str | None:
+ lines = content.splitlines()
+ changed = False
+ for i, line in enumerate(lines):
+ if _LOCALHOST_DB_RE.match(line):
+ lines[i] = _COMPOSE_DATABASE_STRING
+ changed = True
+ if not changed:
+ return None
+ result = "\n".join(lines)
+ if content.endswith("\n"):
+ result += "\n"
+ return result
+
+
+def _bootstrap_compose_env_files() -> None:
+ env_path = Path(".env")
+ env_example = Path(".env.example")
+
+ if not env_path.exists():
+ if env_example.exists():
+ content = env_example.read_text()
+ rewritten = _rewrite_localhost_db_string(content)
+ env_path.write_text(rewritten if rewritten is not None else content, encoding="utf-8")
+ console.print("[green]Created .env from .env.example[/green]")
+ else:
+ try:
+ content = env_path.read_text()
+ except OSError:
+ content = ""
+ rewritten = _rewrite_localhost_db_string(content)
+ if rewritten is not None:
+ console.print(
+ Panel(
+ "[bold yellow]DATABASE_STRING pointing to localhost detected in .env.[/bold yellow]\n\n"
+ "Docker Compose uses its own Postgres service. The connection string will be "
+ "rewritten to point to the compose [cyan]postgres[/cyan] host.",
+ border_style="yellow",
+ )
+ )
+ if Confirm.ask("Rewrite DATABASE_STRING for Docker Compose?", default=True):
+ env_path.write_text(rewritten, encoding="utf-8")
+ console.print("[green]Rewrote DATABASE_STRING for Docker Compose[/green]")
+
+ frontend_env = Path("skyvern-frontend/.env")
+ frontend_example = Path("skyvern-frontend/.env.example")
+ if not frontend_env.exists() and frontend_example.exists():
+ frontend_env.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy(frontend_example, frontend_env)
+ console.print("[green]Created skyvern-frontend/.env from .env.example[/green]")
+
+
def run_docker_compose_setup() -> None:
"""Run the Docker Compose setup for Skyvern."""
from skyvern.cli.llm_setup import setup_llm_providers # noqa: PLC0415
@@ -563,6 +621,7 @@ def run_docker_compose_setup() -> None:
console.print("\n[bold blue]Setting up Skyvern with Docker Compose...[/bold blue]")
capture_setup_event("docker-compose-start")
+ _bootstrap_compose_env_files()
_handle_running_compose_stack()
_handle_postgres_container_conflict()
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index f36a370ea..cec0d365f 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -46,12 +46,19 @@ from skyvern.forge.sdk.copilot.output_policy import (
normalize_response_scaffolding,
output_policy_verdict_from_trace_data,
output_policy_verdict_to_trace_data,
- url_origin,
)
from skyvern.forge.sdk.copilot.output_utils import (
extract_final_text,
parse_final_response,
)
+from skyvern.forge.sdk.copilot.recoverable_failure import (
+ RecoverableFailure,
+ build_recoverable_failure,
+ clean_recorded_failure_text,
+ format_recoverable_failure_reply,
+ iter_exception_chain,
+ merge_failure_into_context,
+)
from skyvern.forge.sdk.copilot.request_policy import (
CREDENTIAL_DEFERRED_DRAFT_REASONS,
RAW_SECRET_REFUSAL_SENTINEL,
@@ -112,7 +119,7 @@ def _copilot_turn_span(
chat_request: WorkflowCopilotChatRequest,
chat_history: list[WorkflowCopilotChatHistoryMessage],
turn_index: int | None,
-) -> Iterator[None]:
+) -> Iterator[Any]:
tracer = otel_trace.get_tracer("skyvern")
with tracer.start_as_current_span(_COPILOT_TURN_SPAN_NAME) as span:
span.set_attribute("skyvern.span.role", "wrapper")
@@ -125,7 +132,7 @@ def _copilot_turn_span(
if chat_request.workflow_permanent_id:
span.set_attribute("workflow_permanent_id", chat_request.workflow_permanent_id)
apply_context_attrs(span)
- yield
+ yield span
def _resolve_request_policy_handler(fallback_handler: Any) -> Any:
@@ -150,6 +157,10 @@ class RequestPolicyGuardrailInputs:
browser_session_id: str | None = None
+class CopilotRequestPolicyMissingError(Exception):
+ """Raised when the request-policy guardrail fails before producing a policy."""
+
+
async def _resolve_live_browser_session_id(
chat_request: WorkflowCopilotChatRequest,
organization_id: str,
@@ -675,7 +686,6 @@ _MAX_TURNS_REPLY_UNVALIDATED = (
_MAX_TURNS_REPLY_TESTED = (
"I've reached the maximum number of steps, but I have a tested draft for you. Accept it to save, or discard."
)
-_UNEXPECTED_ERROR_REPLY_DEFAULT = "An unexpected error occurred. Please try again."
_UNEXPECTED_ERROR_REPLY_UNVALIDATED = (
"I hit an unexpected issue before I could finish testing. I have a draft workflow you can keep — "
"accept it to save (note: it hasn't been verified end-to-end), or discard."
@@ -715,7 +725,6 @@ _UNVALIDATED_PROPOSAL_AFFORDANCE = (
"I have a draft workflow proposal. Use Review to inspect it, Accept to save it, or Reject to discard it. "
"It has not been tested or verified end-to-end."
)
-_RECORDED_FAILURE_URL_RE = re.compile(r"https?://[^\s)>,]+")
def _workflow_block_count(ctx: CopilotContext) -> int | None:
@@ -729,16 +738,8 @@ def _workflow_block_count(ctx: CopilotContext) -> int | None:
def _clean_recorded_failure_text(value: Any, max_chars: int = 240) -> str:
- from skyvern.forge.sdk.copilot.enforcement import redact_browser_session_references
-
- if not isinstance(value, str):
- return ""
- text = redact_raw_secrets_for_prompt(" ".join(value.split()))
- text = redact_browser_session_references(text)
- text = _RECORDED_FAILURE_URL_RE.sub(lambda match: url_origin(match.group(0)) or "[URL]", text)
- if len(text) > max_chars:
- text = text[: max_chars - 3].rstrip() + "..."
- return text.rstrip(".")
+ # Caller-owned sentence templates add punctuation around these fragments.
+ return clean_recorded_failure_text(value, max_chars=max_chars).rstrip(".")
def _recorded_failure_summary(ctx: CopilotContext) -> tuple[str, str]:
@@ -870,6 +871,16 @@ def _build_wip_exit_result(
return _build_exit_result(ctx, recorded_failure_reply or default_reply, global_llm_context, cancelled=cancelled)
+def _merge_exit_context(
+ global_llm_context: str | None,
+ *,
+ failure: RecoverableFailure | None = None,
+) -> str | None:
+ if failure is None:
+ return global_llm_context
+ return merge_failure_into_context(global_llm_context, failure)
+
+
def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
return _build_wip_exit_result(
ctx,
@@ -897,14 +908,48 @@ def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str |
)
-def _build_unexpected_error_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
- return _build_wip_exit_result(
+def _build_unexpected_error_exit_result(
+ ctx: CopilotContext,
+ global_llm_context: str | None,
+ error: BaseException | None = None,
+ *,
+ span: Any | None = None,
+) -> AgentResult:
+ failure = build_recoverable_failure(
+ error,
+ workflow_modified=ctx.workflow_persisted,
+ )
+ default_reply = format_recoverable_failure_reply(failure)
+ enriched_context = _merge_exit_context(global_llm_context, failure=failure)
+ result = _build_wip_exit_result(
ctx,
- global_llm_context,
- default_reply=_UNEXPECTED_ERROR_REPLY_DEFAULT,
+ enriched_context,
+ default_reply=default_reply,
unvalidated_reply=_UNEXPECTED_ERROR_REPLY_UNVALIDATED,
tested_reply=_UNEXPECTED_ERROR_REPLY_TESTED,
)
+ LOG.warning(
+ "Copilot unexpected error translated to recoverable reply",
+ failure_kind=failure.failure_kind,
+ internal_error_id=failure.internal_error_id,
+ exception_type=failure.exception_type,
+ error_type=type(error).__name__ if error else None,
+ workflow_permanent_id=getattr(ctx, "workflow_permanent_id", None),
+ workflow_copilot_chat_id=getattr(ctx, "workflow_copilot_chat_id", None),
+ workflow_modified=failure.workflow_modified,
+ has_proposal=result.updated_workflow is not None,
+ proposal_disposition=result.proposal_disposition,
+ last_test_ok=getattr(ctx, "last_test_ok", None),
+ )
+ current_span = span or otel_trace.get_current_span()
+ current_span.set_attribute("copilot.error_recovered", True)
+ current_span.set_attribute("copilot.error_failure_kind", failure.failure_kind)
+ current_span.set_attribute("copilot.error_id", failure.internal_error_id)
+ if failure.exception_type:
+ current_span.set_attribute("copilot.error_exception_type", failure.exception_type)
+ current_span.set_attribute("copilot.error_reply_proposal_disposition", result.proposal_disposition)
+ current_span.set_attribute("copilot.error_workflow_modified", failure.workflow_modified)
+ return result
def _build_cancel_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
@@ -1237,17 +1282,8 @@ _RETRIABLE_LLM_ERROR_TEXT = (
_LLM_ERROR_MODULE_MARKERS = ("openai", "litellm", "anthropic")
-def _iter_exception_chain(exc: BaseException) -> list[BaseException]:
- chain: list[BaseException] = []
- current: BaseException | None = exc
- while current is not None and current not in chain:
- chain.append(current)
- current = current.__cause__ or current.__context__
- return chain
-
-
def _is_retriable_llm_error(exc: BaseException) -> bool:
- for item in _iter_exception_chain(exc):
+ for item in iter_exception_chain(exc):
module = type(item).__module__.lower()
name = type(item).__name__
text = str(item).lower()
@@ -1605,28 +1641,71 @@ async def run_copilot_agent(
turn_index: int | None = None,
prior_copilot_workflow_yaml: str | None = None,
) -> AgentResult:
- # Initialize tracing before opening the turn span so Logfire's OTel provider
- # is installed; otherwise the very first turn lands the parent span on
- # OTel's no-op ProxyTracer when running locally with COPILOT_TRACING_ENABLED.
- ensure_tracing_initialized()
- with _copilot_turn_span(
- chat_request=chat_request,
- chat_history=chat_history,
- turn_index=turn_index,
- ):
- return await _run_copilot_turn_impl(
- stream=stream,
- organization_id=organization_id,
+ try:
+ # Initialize tracing before opening the turn span so Logfire's OTel provider
+ # is installed; otherwise the very first turn lands the parent span on
+ # OTel's no-op ProxyTracer when running locally with COPILOT_TRACING_ENABLED.
+ ensure_tracing_initialized()
+ with _copilot_turn_span(
chat_request=chat_request,
chat_history=chat_history,
- global_llm_context=global_llm_context,
- debug_run_info_text=debug_run_info_text,
- llm_api_handler=llm_api_handler,
+ turn_index=turn_index,
+ ) as turn_span:
+ try:
+ return await _run_copilot_turn_impl(
+ stream=stream,
+ organization_id=organization_id,
+ chat_request=chat_request,
+ chat_history=chat_history,
+ global_llm_context=global_llm_context,
+ debug_run_info_text=debug_run_info_text,
+ llm_api_handler=llm_api_handler,
+ api_key=api_key,
+ security_rules=security_rules,
+ config=config,
+ prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
+ )
+ except Exception as exc:
+ LOG.error(
+ "Copilot turn unhandled error",
+ error_type=type(exc).__name__,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ exc_info=True,
+ )
+ turn_span.record_exception(exc)
+ ctx = CopilotContext(
+ organization_id=organization_id,
+ workflow_id=chat_request.workflow_id,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_yaml=chat_request.workflow_yaml or "",
+ browser_session_id=None,
+ stream=stream,
+ api_key=api_key,
+ user_message=chat_request.message,
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ )
+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=exc, span=turn_span)
+ except Exception as exc:
+ LOG.error(
+ "Copilot turn unhandled error",
+ error_type=type(exc).__name__,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ exc_info=True,
+ )
+ ctx = CopilotContext(
+ organization_id=organization_id,
+ workflow_id=chat_request.workflow_id,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_yaml=chat_request.workflow_yaml or "",
+ browser_session_id=None,
+ stream=stream,
api_key=api_key,
- security_rules=security_rules,
- config=config,
- prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
+ user_message=chat_request.message,
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
)
+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=exc)
async def _run_copilot_turn_impl(
@@ -1734,7 +1813,7 @@ async def _run_copilot_turn_impl(
prior_workflow_yaml=chat_request.workflow_yaml,
)
if request_policy is None:
- raise RuntimeError("Copilot request-policy input guardrail did not populate request policy")
+ raise CopilotRequestPolicyMissingError()
agent_user_message, safe_chat_history_text = _request_policy_agent_inputs(
request_policy,
@@ -1968,7 +2047,7 @@ async def _run_copilot_turn_impl(
error_message=exc.error_message,
organization_id=organization_id,
)
- return _build_unexpected_error_exit_result(ctx, global_llm_context)
+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=exc)
except CopilotNonRetriableNavError as exc:
LOG.warning(
"Copilot run halted on non-retriable navigation error",
@@ -1991,4 +2070,4 @@ async def _run_copilot_turn_impl(
)
except Exception as e:
LOG.error("Copilot agent error", error=str(e), exc_info=True)
- return _build_unexpected_error_exit_result(ctx, global_llm_context)
+ return _build_unexpected_error_exit_result(ctx, global_llm_context, error=e)
diff --git a/skyvern/forge/sdk/copilot/recoverable_failure.py b/skyvern/forge/sdk/copilot/recoverable_failure.py
new file mode 100644
index 000000000..8834afc5f
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/recoverable_failure.py
@@ -0,0 +1,149 @@
+"""Recoverable Copilot failure rendering and telemetry helpers."""
+
+from __future__ import annotations
+
+import re
+import uuid
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Literal
+
+import yaml
+from pydantic import ValidationError
+
+from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
+from skyvern.forge.sdk.copilot.context import StructuredContext
+from skyvern.forge.sdk.copilot.output_policy import url_origin
+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
+from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
+
+RecoverableFailureKind = Literal["validation", "tool_call", "external_dep", "timeout", "unknown"]
+
+_REASON_SUMMARY_MAX_CHARS = 120
+_RECOVERABLE_ERROR_ID_PREFIX = "cpe"
+_RECORDED_FAILURE_URL_RE = re.compile(r"https?://[^\s)>,]+")
+_RECORDED_FAILURE_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
+_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")
+_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)
+
+
+@dataclass(frozen=True)
+class RecoverableFailure:
+ failure_kind: RecoverableFailureKind
+ workflow_modified: bool
+ reason_summary: str
+ internal_error_id: str
+ exception_type: str | None = None
+
+
+def iter_exception_chain(exc: BaseException) -> Iterable[BaseException]:
+ seen: set[int] = set()
+ current: BaseException | None = exc
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ yield current
+ current = current.__cause__ or current.__context__
+
+
+def _new_internal_error_id() -> str:
+ return f"{_RECOVERABLE_ERROR_ID_PREFIX}_{uuid.uuid4().hex[:12]}"
+
+
+def _redact_browser_session_references(value: str) -> str:
+ value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)
+ return _BROWSER_SESSION_ID_RE.sub("the browser session", value)
+
+
+def _is_exception_named(exc: BaseException, name: str) -> bool:
+ # Keep this helper import-light for the base/local wheel: importing
+ # copilot.enforcement here would pull in the server-only agents package.
+ return type(exc).__name__ == name
+
+
+def clean_recorded_failure_text(value: object, max_chars: int = _REASON_SUMMARY_MAX_CHARS) -> str:
+ if not isinstance(value, str):
+ return ""
+ text = value.strip()
+ if not text:
+ return ""
+ text = redact_raw_secrets_for_prompt(" ".join(text.split()))
+ text = _redact_browser_session_references(text)
+ text = _RECORDED_FAILURE_CREDENTIAL_ID_RE.sub("[CREDENTIAL_ID]", text)
+ text = _RECORDED_FAILURE_URL_RE.sub(lambda match: url_origin(match.group(0)) or "[URL]", text)
+ if len(text) > max_chars:
+ text = text[: max_chars - 3].rstrip() + "..."
+ return text
+
+
+def _reason_summary(value: str) -> str:
+ return clean_recorded_failure_text(value, max_chars=_REASON_SUMMARY_MAX_CHARS)
+
+
+def _failure_kind_for_exception(error: BaseException | None) -> RecoverableFailureKind:
+ if error is None:
+ return "unknown"
+ if any(_is_exception_named(item, "CopilotTotalTimeoutError") for item in iter_exception_chain(error)):
+ return "timeout"
+ if any(
+ _is_exception_named(item, "CopilotUnrecoverableToolError")
+ or _is_exception_named(item, "CopilotNonRetriableNavError")
+ for item in iter_exception_chain(error)
+ ):
+ return "tool_call"
+ if any(
+ isinstance(item, (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException))
+ for item in iter_exception_chain(error)
+ ):
+ return "validation"
+ if any(isinstance(item, LLMProviderError) for item in iter_exception_chain(error)):
+ return "external_dep"
+ return "unknown"
+
+
+def _reason_for_failure(error: BaseException | None, failure_kind: RecoverableFailureKind) -> str:
+ if failure_kind == "timeout":
+ return "Copilot timed out before it could finish this turn"
+ if failure_kind == "validation":
+ return "The workflow update could not be validated"
+ if failure_kind == "external_dep":
+ return "A Copilot dependency stopped responding"
+ if failure_kind == "tool_call":
+ chain = list(iter_exception_chain(error)) if error is not None else []
+ if any(_is_exception_named(item, "CopilotNonRetriableNavError") for item in chain):
+ return "A browser navigation step could not reach the target URL"
+ return "A browser or workflow helper stopped responding"
+ return "Copilot hit an internal error before it could finish this turn"
+
+
+def build_recoverable_failure(
+ error: BaseException | None,
+ *,
+ workflow_modified: bool,
+ internal_error_id: str | None = None,
+) -> RecoverableFailure:
+ failure_kind = _failure_kind_for_exception(error)
+ reason_summary = _reason_summary(_reason_for_failure(error, failure_kind))
+ exception_type = type(error).__name__ if error is not None else None
+ return RecoverableFailure(
+ failure_kind=failure_kind,
+ workflow_modified=workflow_modified,
+ reason_summary=reason_summary,
+ internal_error_id=internal_error_id or _new_internal_error_id(),
+ exception_type=exception_type,
+ )
+
+
+def format_recoverable_failure_reply(failure: RecoverableFailure) -> str:
+ workflow_state = "preserved" if failure.workflow_modified else "not modified"
+ return (
+ f"{failure.reason_summary}. The workflow was {workflow_state}. "
+ f"If this persists, reference {failure.internal_error_id} for support."
+ )
+
+
+def merge_failure_into_context(global_llm_context: str | None, failure: RecoverableFailure) -> str:
+ structured = StructuredContext.from_json_str(global_llm_context)
+ note = f"copilot turn failed: {failure.failure_kind} {failure.internal_error_id}"
+ if note not in structured.decisions_made:
+ structured.decisions_made.append(note)
+ return structured.to_json_str()
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 684c4bb24..c670a5d57 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -52,6 +52,8 @@ from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.routes.code_samples import (
+ BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON,
+ BULK_CANCEL_RUNS_CODE_SAMPLE_TS,
CANCEL_RUN_CODE_SAMPLE_PYTHON,
CANCEL_RUN_CODE_SAMPLE_TS,
CREATE_WORKFLOW_CODE_SAMPLE_CURL,
@@ -121,6 +123,8 @@ from skyvern.schemas.runs import (
CUA_ENGINES,
BlockRunRequest,
BlockRunResponse,
+ BulkCancelRunsRequest,
+ BulkCancelRunsResponse,
RunEngine,
RunResponse,
RunStatus,
@@ -570,6 +574,36 @@ async def cancel_run(
await run_service.cancel_run(run_id, organization_id=current_org.organization_id, api_key=x_api_key)
+@base_router.post(
+ "/runs/cancel",
+ tags=["Agent", "Workflow Runs"],
+ openapi_extra={
+ "x-fern-sdk-method-name": "bulk_cancel_runs",
+ "x-fern-examples": [
+ {
+ "code-samples": [
+ {"sdk": "python", "code": BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON},
+ {"sdk": "typescript", "code": BULK_CANCEL_RUNS_CODE_SAMPLE_TS},
+ ]
+ }
+ ],
+ },
+ description="Cancel multiple runs (tasks or workflows) in a single request",
+ summary="Bulk cancel runs",
+)
+@base_router.post("/runs/cancel/", include_in_schema=False)
+async def bulk_cancel_runs(
+ data: BulkCancelRunsRequest = Body(...),
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+ x_api_key: Annotated[str | None, Header()] = None,
+) -> BulkCancelRunsResponse:
+ analytics.capture("skyvern-oss-agent-bulk-cancel-runs")
+
+ return await run_service.bulk_cancel_runs(
+ data.run_ids, organization_id=current_org.organization_id, api_key=x_api_key
+ )
+
+
@legacy_base_router.post(
"/workflows",
openapi_extra={
diff --git a/skyvern/forge/sdk/routes/code_samples.py b/skyvern/forge/sdk/routes/code_samples.py
index 71ee7dc69..43df0a27a 100644
--- a/skyvern/forge/sdk/routes/code_samples.py
+++ b/skyvern/forge/sdk/routes/code_samples.py
@@ -50,6 +50,16 @@ CANCEL_RUN_CODE_SAMPLE_TS = """import { SkyvernClient } from "@skyvern/client";
const skyvern = new SkyvernClient({ apiKey: "YOUR_API_KEY" });
await skyvern.cancelRun("tsk_v2_123");
"""
+BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON = """from skyvern import Skyvern
+
+skyvern = Skyvern(api_key="YOUR_API_KEY")
+await skyvern.bulk_cancel_runs(run_ids=["tsk_v2_123", "wr_456"])
+"""
+BULK_CANCEL_RUNS_CODE_SAMPLE_TS = """import { SkyvernClient } from "@skyvern/client";
+
+const skyvern = new SkyvernClient({ apiKey: "YOUR_API_KEY" });
+await skyvern.bulkCancelRuns({ run_ids: ["tsk_v2_123", "wr_456"] });
+"""
RETRY_RUN_WEBHOOK_CODE_SAMPLE_PYTHON = """from skyvern import Skyvern
skyvern = Skyvern(api_key="YOUR_API_KEY")
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 25bbeba33..19b829661 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -11,6 +11,7 @@ from typing import Any, Iterator
import structlog
import yaml
from fastapi import Depends, HTTPException, Request, status
+from opentelemetry import trace as otel_trace
from pydantic import ValidationError
from sse_starlette import EventSourceResponse
@@ -27,6 +28,12 @@ from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import AgentResult, ProposalDisposition
from skyvern.forge.sdk.copilot.output_utils import truncate_output
+from skyvern.forge.sdk.copilot.recoverable_failure import (
+ RecoverableFailure,
+ build_recoverable_failure,
+ format_recoverable_failure_reply,
+ merge_failure_into_context,
+)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream, FastAPIEventSourceStream
@@ -292,6 +299,41 @@ def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: o
return not _effective_auto_accept(auto_accept, agent_result)
+def _record_recoverable_failure_span_attrs(
+ failure: RecoverableFailure,
+ *,
+ proposal_disposition: ProposalDisposition,
+) -> None:
+ current_span = otel_trace.get_current_span()
+ current_span.set_attribute("copilot.error_recovered", True)
+ current_span.set_attribute("copilot.error_failure_kind", failure.failure_kind)
+ current_span.set_attribute("copilot.error_id", failure.internal_error_id)
+ if failure.exception_type:
+ current_span.set_attribute("copilot.error_exception_type", failure.exception_type)
+ current_span.set_attribute("copilot.error_workflow_modified", failure.workflow_modified)
+ current_span.set_attribute("copilot.error_reply_proposal_disposition", proposal_disposition)
+
+
+def _build_recoverable_route_agent_result(
+ error: BaseException,
+ *,
+ workflow_modified: bool,
+ clear_proposed_workflow: bool,
+ global_llm_context: str | None,
+) -> tuple[AgentResult, RecoverableFailure]:
+ failure = build_recoverable_failure(error, workflow_modified=workflow_modified)
+ agent_result = AgentResult(
+ user_response=format_recoverable_failure_reply(failure),
+ updated_workflow=None,
+ global_llm_context=merge_failure_into_context(global_llm_context, failure),
+ workflow_was_persisted=False,
+ proposal_disposition="no_proposal",
+ clear_proposed_workflow=clear_proposed_workflow,
+ )
+ _record_recoverable_failure_span_attrs(failure, proposal_disposition="no_proposal")
+ return agent_result, failure
+
+
async def _clear_proposed_workflow(chat: Any) -> None:
await app.DATABASE.workflow_params.update_workflow_copilot_chat(
organization_id=chat.organization_id,
@@ -1315,6 +1357,7 @@ async def _new_copilot_chat_post(
original_workflow: Workflow | None = None
chat = None
agent_result: AgentResult | None = None
+ global_llm_context: str | None = None
terminal_frame_emitted = False
cancel_watcher: asyncio.Task[None] | None = None
# Single-element list used as a closure flag (mutable bool by reference).
@@ -1353,7 +1396,6 @@ async def _new_copilot_chat_post(
chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
)
- global_llm_context = None
for message in reversed(chat_messages):
if message.global_llm_context is not None:
global_llm_context = message.global_llm_context
@@ -1527,21 +1569,51 @@ async def _new_copilot_chat_post(
)
)
except LLMProviderError as exc:
- if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
+ restored = chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result)
+ if restored:
await _restore_workflow_definition(original_workflow, organization.organization_id)
- LOG.error(
- "LLM provider error (copilot v2)",
- organization_id=organization.organization_id,
- error=str(exc),
- exc_info=True,
- )
- terminal_frame_emitted = True
- await stream.send(
- WorkflowCopilotStreamErrorUpdate(
- type=WorkflowCopilotStreamMessageType.ERROR,
- error="Failed to process your request. Please try again.",
+ if chat is not None:
+ workflow_modified = bool(getattr(agent_result, "workflow_was_persisted", False)) and not restored
+ recovered_result, failure = _build_recoverable_route_agent_result(
+ exc,
+ workflow_modified=workflow_modified,
+ clear_proposed_workflow=restored or workflow_modified,
+ global_llm_context=global_llm_context,
+ )
+ LOG.error(
+ "LLM provider error translated to recoverable workflow copilot v2 reply",
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ failure_kind=failure.failure_kind,
+ internal_error_id=failure.internal_error_id,
+ exception_type=failure.exception_type,
+ exc_info=True,
+ )
+ await asyncio.shield(
+ _finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id=organization.organization_id,
+ original_workflow=original_workflow,
+ chat_request=chat_request,
+ agent_result=recovered_result,
+ )
+ )
+ terminal_frame_emitted = True
+ else:
+ LOG.error(
+ "LLM provider error (copilot v2)",
+ organization_id=organization.organization_id,
+ error=str(exc),
+ exc_info=True,
+ )
+ terminal_frame_emitted = True
+ await stream.send(
+ WorkflowCopilotStreamErrorUpdate(
+ type=WorkflowCopilotStreamMessageType.ERROR,
+ error="Failed to process your request. Please try again.",
+ )
)
- )
except asyncio.CancelledError:
if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
await asyncio.shield(_restore_workflow_definition(original_workflow, organization.organization_id))
@@ -1578,21 +1650,51 @@ async def _new_copilot_chat_post(
)
raise
except Exception as exc:
- if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
+ restored = chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result)
+ if restored:
await _restore_workflow_definition(original_workflow, organization.organization_id)
- LOG.error(
- "Unexpected error in workflow copilot v2",
- organization_id=organization.organization_id,
- error=str(exc),
- exc_info=True,
- )
- terminal_frame_emitted = True
- await stream.send(
- WorkflowCopilotStreamErrorUpdate(
- type=WorkflowCopilotStreamMessageType.ERROR,
- error="An error occurred. Please try again.",
+ if chat is not None:
+ workflow_modified = bool(getattr(agent_result, "workflow_was_persisted", False)) and not restored
+ recovered_result, failure = _build_recoverable_route_agent_result(
+ exc,
+ workflow_modified=workflow_modified,
+ clear_proposed_workflow=restored or workflow_modified,
+ global_llm_context=global_llm_context,
+ )
+ LOG.error(
+ "Unexpected workflow copilot v2 error translated to recoverable reply",
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ failure_kind=failure.failure_kind,
+ internal_error_id=failure.internal_error_id,
+ exception_type=failure.exception_type,
+ exc_info=True,
+ )
+ await asyncio.shield(
+ _finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id=organization.organization_id,
+ original_workflow=original_workflow,
+ chat_request=chat_request,
+ agent_result=recovered_result,
+ )
+ )
+ terminal_frame_emitted = True
+ else:
+ LOG.error(
+ "Unexpected error in workflow copilot v2",
+ organization_id=organization.organization_id,
+ error=str(exc),
+ exc_info=True,
+ )
+ terminal_frame_emitted = True
+ await stream.send(
+ WorkflowCopilotStreamErrorUpdate(
+ type=WorkflowCopilotStreamMessageType.ERROR,
+ error="An error occurred. Please try again.",
+ )
)
- )
finally:
if cancel_watcher is not None and not cancel_watcher.done():
cancel_watcher.cancel()
diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.py
index f446aa498..80837c352 100644
--- a/skyvern/schemas/runs.py
+++ b/skyvern/schemas/runs.py
@@ -475,3 +475,12 @@ class TaskRunListItem(BaseModel):
endpoint's Run.script_run field. Do not rely on dict contents here.
"""
return bool(v)
+
+
+class BulkCancelRunsRequest(BaseModel):
+ run_ids: list[str] = Field(max_length=100, description="List of run IDs to cancel")
+
+
+class BulkCancelRunsResponse(BaseModel):
+ cancelled: list[str] = Field(description="Run IDs that were successfully cancelled")
+ failed: list[str] = Field(description="Run IDs that could not be cancelled")
diff --git a/skyvern/services/run_service.py b/skyvern/services/run_service.py
index c732ca4b3..8ead2420e 100644
--- a/skyvern/services/run_service.py
+++ b/skyvern/services/run_service.py
@@ -1,3 +1,6 @@
+import asyncio
+
+import structlog
from fastapi import HTTPException, status
from skyvern.config import settings
@@ -5,10 +8,19 @@ from skyvern.exceptions import OrganizationNotFound, TaskNotFound, WorkflowRunNo
from skyvern.forge import app
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
-from skyvern.schemas.runs import RunEngine, RunResponse, RunType, TaskRunRequest, TaskRunResponse
+from skyvern.schemas.runs import (
+ BulkCancelRunsResponse,
+ RunEngine,
+ RunResponse,
+ RunType,
+ TaskRunRequest,
+ TaskRunResponse,
+)
from skyvern.schemas.webhooks import RunWebhookReplayResponse
from skyvern.services import task_v1_service, task_v2_service, webhook_service, workflow_service
+LOG = structlog.get_logger()
+
async def get_run_response(run_id: str, organization_id: str | None = None) -> RunResponse | None:
run = await app.DATABASE.tasks.get_run(run_id, organization_id=organization_id)
@@ -150,6 +162,24 @@ async def cancel_run(run_id: str, organization_id: str | None = None, api_key: s
)
+async def bulk_cancel_runs(
+ run_ids: list[str], organization_id: str | None = None, api_key: str | None = None
+) -> BulkCancelRunsResponse:
+ cancelled: list[str] = []
+ failed: list[str] = []
+
+ async def _cancel_one(run_id: str) -> None:
+ try:
+ await cancel_run(run_id, organization_id=organization_id, api_key=api_key)
+ cancelled.append(run_id)
+ except Exception:
+ LOG.warning("bulk_cancel_runs: failed to cancel run", run_id=run_id, exc_info=True)
+ failed.append(run_id)
+
+ await asyncio.gather(*[_cancel_one(run_id) for run_id in dict.fromkeys(run_ids)])
+ return BulkCancelRunsResponse(cancelled=cancelled, failed=failed)
+
+
async def retry_run_webhook(
run_id: str,
organization_id: str | None = None,
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 451f02f9f..398ad93b5 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -8,9 +8,17 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
+import yaml
+from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotNonRetriableNavError,
+ CopilotTotalTimeoutError,
+ CopilotUnrecoverableToolError,
+)
+from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
from skyvern.forge.sdk.copilot.request_policy import (
_REDACTED_REFUSED_SECRET_TURN,
TRANSCRIPT_ANCHOR_CHAR_CAP,
@@ -918,6 +926,107 @@ workflow_definition:
)
assert "same IP/workflow shape" in agent_result.user_response
+ def test_unexpected_error_exit_names_failure_and_preserves_context(self) -> None:
+ ctx = _ctx()
+
+ agent_result = agent_module._build_unexpected_error_exit_result(
+ ctx,
+ global_llm_context=None,
+ error=agent_module.CopilotRequestPolicyMissingError(),
+ )
+
+ assert "An unexpected error occurred. Please try again." not in agent_result.user_response
+ assert "Copilot hit an internal error before it could finish this turn" in agent_result.user_response
+ assert "The workflow was not modified" in agent_result.user_response
+ assert "reference cpe_" in agent_result.user_response
+ assert "copilot turn failed: unknown cpe_" in (agent_result.global_llm_context or "")
+ assert agent_result.updated_workflow is None
+
+ def test_unexpected_error_exit_redacts_sensitive_identifiers(self) -> None:
+ ctx = _ctx(workflow_persisted=True)
+
+ agent_result = agent_module._build_unexpected_error_exit_result(
+ ctx,
+ global_llm_context=None,
+ error=RuntimeError("credential cred_12345 was rejected while opening https://example.com/private/path"),
+ )
+
+ assert "cred_12345" not in agent_result.user_response
+ assert "https://example.com/private/path" not in agent_result.user_response
+ assert "credential" not in agent_result.user_response.lower()
+ assert "https://example.com" not in agent_result.user_response
+ assert "Copilot hit an internal error before it could finish this turn" in agent_result.user_response
+ assert "The workflow was preserved" in agent_result.user_response
+ assert "reference cpe_" in agent_result.user_response
+
+ def test_unexpected_error_exit_does_not_persist_tool_output_preview(self) -> None:
+ ctx = _ctx()
+ ctx.tool_activity.append(
+ {
+ "tool": "get_run_results",
+ "summary": "OK",
+ "output_preview": "block_1: [email protected] password=hunter2",
+ }
+ )
+
+ agent_result = agent_module._build_unexpected_error_exit_result(
+ ctx,
+ global_llm_context=None,
+ error=RuntimeError("boom"),
+ )
+
+ assert "[email protected]" not in (agent_result.global_llm_context or "")
+ assert "hunter2" not in (agent_result.global_llm_context or "")
+ assert "copilot turn failed: unknown cpe_" in (agent_result.global_llm_context or "")
+
+ def test_recoverable_failure_maps_expected_exception_families(self) -> None:
+ assert (
+ build_recoverable_failure(
+ CopilotTotalTimeoutError(),
+ workflow_modified=False,
+ internal_error_id="cpe_timeout",
+ ).failure_kind
+ == "timeout"
+ )
+ assert (
+ build_recoverable_failure(
+ CopilotUnrecoverableToolError("click", "browser failed"),
+ workflow_modified=False,
+ internal_error_id="cpe_tool",
+ ).failure_kind
+ == "tool_call"
+ )
+ assert (
+ build_recoverable_failure(
+ yaml.YAMLError("bad yaml"),
+ workflow_modified=False,
+ internal_error_id="cpe_validation",
+ ).failure_kind
+ == "validation"
+ )
+ assert (
+ build_recoverable_failure(
+ LLMProviderError("OPENAI_GPT5_5"),
+ workflow_modified=False,
+ internal_error_id="cpe_external",
+ ).failure_kind
+ == "external_dep"
+ )
+
+ def test_recoverable_failure_uses_chained_navigation_reason(self) -> None:
+ nav_error = CopilotNonRetriableNavError("https://example.com", "net::ERR_NAME_NOT_RESOLVED")
+ wrapper = RuntimeError("wrapped")
+ wrapper.__cause__ = nav_error
+
+ failure = build_recoverable_failure(
+ wrapper,
+ workflow_modified=False,
+ internal_error_id="cpe_nav",
+ )
+
+ assert failure.failure_kind == "tool_call"
+ assert failure.reason_summary == "A browser navigation step could not reach the target URL"
+
def test_reply_still_rewrites_after_failed_test(self) -> None:
ctx = _ctx(
last_update_block_count=2,
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index 27c9a59e1..d51eacc7a 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -14,7 +14,6 @@ from skyvern.forge.sdk.copilot.agent import (
_TIMEOUT_REPLY_DEFAULT,
_TIMEOUT_REPLY_TESTED,
_TIMEOUT_REPLY_UNVALIDATED,
- _UNEXPECTED_ERROR_REPLY_DEFAULT,
_UNEXPECTED_ERROR_REPLY_TESTED,
_UNEXPECTED_ERROR_REPLY_UNVALIDATED,
_build_cancel_exit_result,
@@ -51,6 +50,7 @@ def _ctx(
ctx.total_tokens_used = None
ctx.last_good_workflow = last_good_workflow
ctx.last_good_workflow_yaml = last_good_workflow_yaml
+ ctx.tool_activity = []
ctx.latest_diagnosis_repair_contract = None
ctx.test_after_update_done = last_test_ok is not None
ctx.last_update_block_count = None
@@ -244,7 +244,10 @@ class TestBuildUnexpectedErrorExitResult:
assert result.updated_workflow is None
assert result.workflow_yaml is None
assert result.unvalidated is False
- assert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT
+ assert "An unexpected error occurred. Please try again." not in result.user_response
+ assert "Copilot hit an internal error before it could finish this turn" in result.user_response
+ assert "The workflow was not modified" in result.user_response
+ assert "reference cpe_" in result.user_response
def test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:
wf = MagicMock(name="wf")
@@ -277,7 +280,9 @@ class TestBuildUnexpectedErrorExitResult:
assert result.updated_workflow is None
assert result.workflow_yaml is None
assert result.unvalidated is False
- assert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT
+ assert "Copilot hit an internal error before it could finish this turn" in result.user_response
+ assert "The workflow was preserved" in result.user_response
+ assert "reference cpe_" in result.user_response
def test_failed_test_uses_recorded_blocker_reply(self) -> None:
wf = MagicMock(name="wf")
@@ -346,7 +351,9 @@ class TestBuildUnexpectedErrorExitResult:
assert result.updated_workflow is None
assert result.workflow_yaml is None
assert result.unvalidated is False
- assert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT
+ assert "Copilot hit an internal error before it could finish this turn" in result.user_response
+ assert "The workflow was preserved" in result.user_response
+ assert "reference cpe_" in result.user_response
class TestBuildCancelExitResult:
diff --git a/tests/unit/test_copilot_turn_span.py b/tests/unit/test_copilot_turn_span.py
index 717248a9d..b6ee526d8 100644
--- a/tests/unit/test_copilot_turn_span.py
+++ b/tests/unit/test_copilot_turn_span.py
@@ -57,6 +57,10 @@ async def _stub_build_request_policy_proceed(*_args: Any, **_kwargs: Any) -> Req
return RequestPolicy(user_response_policy="ask_clarification", clarification_question="?")
+async def _raise_unhandled_turn_error(*_args: Any, **_kwargs: Any) -> None:
+ raise RuntimeError("boom")
+
+
@pytest.fixture
def patched_build_request_policy(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
@@ -181,6 +185,35 @@ async def test_copilot_turn_span_preview_is_single_line_and_truncated(
assert preview.endswith("…")
+async def test_outer_turn_recovery_records_error_attrs_on_turn_span(
+ span_exporter: Any,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(copilot_agent, "_run_copilot_turn_impl", _raise_unhandled_turn_error)
+ chat_request = _make_chat_request()
+
+ result = await copilot_agent.run_copilot_agent(
+ stream=object(),
+ organization_id="o_test",
+ chat_request=chat_request,
+ chat_history=[],
+ global_llm_context=None,
+ debug_run_info_text="",
+ llm_api_handler=None,
+ )
+
+ assert result.updated_workflow is None
+ assert "reference cpe_" in result.user_response
+ turn_span = _find_span(span_exporter.get_finished_spans(), "copilot.turn")
+ attrs = dict(turn_span.attributes or {})
+ assert attrs.get("copilot.error_recovered") is True
+ assert attrs.get("copilot.error_failure_kind") == "unknown"
+ assert attrs.get("copilot.error_exception_type") == "RuntimeError"
+ assert attrs.get("copilot.error_reply_proposal_disposition") == result.proposal_disposition
+ assert attrs.get("copilot.error_workflow_modified") is False
+
+
def test_build_user_message_preview_redacts_known_secret_patterns() -> None:
preview = copilot_agent._build_user_message_preview("password: hunter2 please")
assert "hunter2" not in preview
diff --git a/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.py
index ddf3ca2be..45d893fac 100644
--- a/tests/unit/test_quickstart.py
+++ b/tests/unit/test_quickstart.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import subprocess
+from pathlib import Path
import pytest
import typer
@@ -73,3 +74,115 @@ def test_quickstart_reraises_intentional_typer_exit(monkeypatch: pytest.MonkeyPa
)
assert capture_errors == []
+
+
+def test_bootstrap_creates_env_and_rewrites_localhost_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ example = tmp_path / ".env.example"
+ example.write_text('KEY=value\nDATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\n')
+ monkeypatch.chdir(tmp_path)
+
+ quickstart._bootstrap_compose_env_files()
+
+ result = (tmp_path / ".env").read_text()
+ assert "localhost" not in result
+ assert "KEY=value" in result
+ assert 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"' in result
+
+
+def test_bootstrap_creates_env_without_db_string_unchanged(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ example = tmp_path / ".env.example"
+ example.write_text("KEY=value\n")
+ monkeypatch.chdir(tmp_path)
+
+ quickstart._bootstrap_compose_env_files()
+
+ assert (tmp_path / ".env").read_text() == "KEY=value\n"
+
+
+def test_bootstrap_does_not_overwrite_existing_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ (tmp_path / ".env.example").write_text("KEY=example\n")
+ existing = tmp_path / ".env"
+ existing.write_text("KEY=existing\n")
+ monkeypatch.chdir(tmp_path)
+
+ quickstart._bootstrap_compose_env_files()
+
+ assert existing.read_text() == "KEY=existing\n"
+
+
+def test_bootstrap_creates_frontend_env_from_example(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ (tmp_path / ".env.example").write_text("")
+ frontend_dir = tmp_path / "skyvern-frontend"
+ frontend_dir.mkdir()
+ (frontend_dir / ".env.example").write_text("VITE_KEY=val\n")
+ monkeypatch.chdir(tmp_path)
+
+ quickstart._bootstrap_compose_env_files()
+
+ assert (frontend_dir / ".env").read_text() == "VITE_KEY=val\n"
+
+
+def test_bootstrap_does_not_overwrite_existing_frontend_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ (tmp_path / ".env.example").write_text("")
+ frontend_dir = tmp_path / "skyvern-frontend"
+ frontend_dir.mkdir()
+ (frontend_dir / ".env.example").write_text("VITE_KEY=example\n")
+ existing = frontend_dir / ".env"
+ existing.write_text("VITE_KEY=existing\n")
+ monkeypatch.chdir(tmp_path)
+
+ quickstart._bootstrap_compose_env_files()
+
+ assert existing.read_text() == "VITE_KEY=existing\n"
+
+
+def test_bootstrap_rewrites_localhost_db_string_when_confirmed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ env_content = 'ENABLE_OPENAI=false\nDATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\nENV=local\n'
+ (tmp_path / ".env").write_text(env_content)
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *_args, **_kwargs: True)
+
+ quickstart._bootstrap_compose_env_files()
+
+ result = (tmp_path / ".env").read_text()
+ assert "localhost" not in result
+ assert 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"' in result
+ assert "ENABLE_OPENAI=false" in result
+ assert "ENV=local" in result
+
+
+def test_bootstrap_rewrites_export_prefixed_db_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ env_content = 'export DATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\n'
+ (tmp_path / ".env").write_text(env_content)
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *_args, **_kwargs: True)
+
+ quickstart._bootstrap_compose_env_files()
+
+ result = (tmp_path / ".env").read_text()
+ assert "localhost" not in result
+ assert 'DATABASE_STRING="postgresql+psycopg://skyvern:skyvern@postgres/skyvern"' in result
+
+
+def test_bootstrap_keeps_localhost_db_string_when_declined(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ env_content = 'DATABASE_STRING="postgresql+psycopg://skyvern@localhost/skyvern"\n'
+ (tmp_path / ".env").write_text(env_content)
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *_args, **_kwargs: False)
+
+ quickstart._bootstrap_compose_env_files()
+
+ assert (tmp_path / ".env").read_text() == env_content
+
+
+def test_bootstrap_skips_rewrite_for_non_localhost_db_string(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ env_content = 'DATABASE_STRING="postgresql+psycopg://skyvern@postgres/skyvern"\n'
+ (tmp_path / ".env").write_text(env_content)
+ confirm_calls: list[object] = []
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(quickstart.Confirm, "ask", lambda *args, **kwargs: confirm_calls.append(args) or True)
+
+ quickstart._bootstrap_compose_env_files()
+
+ assert confirm_calls == [], "should not prompt when DATABASE_STRING does not point to localhost"
+ assert (tmp_path / ".env").read_text() == env_content
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index c5019d491..94dc14d06 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -23,6 +23,9 @@ import pytest
from skyvern.config import settings
from skyvern.forge import app
+from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
+from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
from skyvern.forge.sdk.routes.workflow_copilot import COPILOT_V2_FLAG_KEY, workflow_copilot_chat_post
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatRequest,
@@ -177,6 +180,7 @@ def _setup_new_copilot_mocks(
get_workflow_run_blocks=AsyncMock(return_value=[]),
)
app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")
+ app.AGENT_FUNCTION.get_copilot_config = MagicMock(return_value=None)
return restore_mock
@@ -260,6 +264,244 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut
restore_mock.assert_not_awaited()
+async def test_flag_on_pre_agent_failure_persists_recoverable_reply(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+
+ async def fake_llm_handler(*args: object, **kwargs: object) -> None:
+ del args, kwargs
+ return None
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.get_llm_handler_for_prompt_type",
+ fake_llm_handler,
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot._restore_workflow_definition",
+ AsyncMock(),
+ )
+ monkeypatch.setattr(agent_module, "ensure_tracing_initialized", lambda: None)
+ monkeypatch.setattr(
+ agent_module,
+ "_run_copilot_turn_impl",
+ AsyncMock(side_effect=agent_module.CopilotRequestPolicyMissingError()),
+ )
+
+ workflow_params = SimpleNamespace(
+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),
+ get_workflow_copilot_chat_messages=AsyncMock(return_value=[]),
+ update_workflow_copilot_chat=AsyncMock(),
+ create_workflow_copilot_chat_message=AsyncMock(
+ return_value=SimpleNamespace(created_at=datetime(2026, 4, 14, tzinfo=timezone.utc))
+ ),
+ )
+ app.DATABASE.workflow_params = workflow_params
+ app.DATABASE.workflows = SimpleNamespace(
+ get_workflow_by_permanent_id=AsyncMock(return_value=original_workflow),
+ )
+ app.DATABASE.observer = SimpleNamespace(
+ get_workflow_run_blocks=AsyncMock(return_value=[]),
+ )
+ app.AGENT_FUNCTION.get_copilot_security_rules = MagicMock(return_value="")
+ app.AGENT_FUNCTION.get_copilot_config = MagicMock(return_value=None)
+ app.AGENT_FUNCTION.resolve_org_api_key = AsyncMock(return_value="sk-test-key")
+
+ request = MagicMock()
+ request.headers = {}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(stream)
+
+ contents = [
+ call.kwargs.get("content") for call in workflow_params.create_workflow_copilot_chat_message.await_args_list
+ ]
+ assert "Please update it" in contents
+ assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]
+ assert len(assistant_contents) == 1
+ assert "An unexpected error occurred. Please try again." not in assistant_contents[0]
+ assert "Copilot hit an internal error before it could finish this turn" in assistant_contents[0]
+ assert "The workflow was not modified" in assistant_contents[0]
+ assert "reference cpe_" in assistant_contents[0]
+
+ frames = [call.args[0] for call in stream.send.await_args_list if call.args]
+ assert any(isinstance(frame, WorkflowCopilotStreamResponseUpdate) for frame in frames)
+ assert not any(isinstance(frame, WorkflowCopilotStreamErrorUpdate) for frame in frames)
+
+
+ ("error", "expected_summary"),
+ [
+ (RuntimeError("route boom"), "Copilot hit an internal error before it could finish this turn"),
+ (LLMProviderError("OPENAI_GPT5_5"), "A Copilot dependency stopped responding"),
+ ],
+)
+async def test_flag_on_route_error_after_chat_persists_recoverable_reply(
+ monkeypatch: pytest.MonkeyPatch,
+ error: BaseException,
+ expected_summary: str,
+) -> None:
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ agent_result = SimpleNamespace(
+ user_response="unused",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=False,
+ clear_proposed_workflow=False,
+ unvalidated=False,
+ )
+ _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",
+ AsyncMock(side_effect=error),
+ )
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test-key"}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(stream)
+
+ contents = [
+ call.kwargs.get("content")
+ for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list
+ ]
+ assert "Please update it" in contents
+ assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]
+ assert len(assistant_contents) == 1
+ assert expected_summary in assistant_contents[0]
+ assert "The workflow was not modified" in assistant_contents[0]
+ assert "reference cpe_" in assistant_contents[0]
+
+ frames = [call.args[0] for call in stream.send.await_args_list if call.args]
+ assert any(isinstance(frame, WorkflowCopilotStreamResponseUpdate) for frame in frames)
+ assert not any(isinstance(frame, WorkflowCopilotStreamErrorUpdate) for frame in frames)
+
+
+async def test_route_error_after_restore_reports_workflow_not_modified(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow={"workflow_id": "stale"},
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ agent_result = SimpleNamespace(
+ user_response="unused",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=True,
+ clear_proposed_workflow=False,
+ unvalidated=False,
+ )
+ restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ finalise_results: list[object] = []
+ original_finalise = workflow_copilot_route._finalise_normal_turn
+
+ async def flaky_finalise(*args: object, **kwargs: object) -> object:
+ finalise_results.append(kwargs["agent_result"])
+ if len(finalise_results) == 1:
+ raise RuntimeError("post-agent route boom")
+ return await original_finalise(*args, **kwargs)
+
+ monkeypatch.setattr(workflow_copilot_route, "_finalise_normal_turn", flaky_finalise)
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test-key"}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(stream)
+
+ restore_mock.assert_awaited_once()
+ assert len(finalise_results) == 2
+ recovered_result = finalise_results[1]
+ assert "The workflow was not modified" in recovered_result.user_response
+ assert "The workflow was preserved" not in recovered_result.user_response
+ assert recovered_result.clear_proposed_workflow is True
+ contents = [
+ call.kwargs.get("content")
+ for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list
+ ]
+ assistant_contents = [content for content in contents if isinstance(content, str) and content != "Please update it"]
+ assert len(assistant_contents) == 1
+ assert "The workflow was not modified" in assistant_contents[0]
+ assert "The workflow was preserved" not in assistant_contents[0]
+ update_calls = app.DATABASE.workflow_params.update_workflow_copilot_chat.await_args_list
+ clear_calls = [c for c in update_calls if c.kwargs.get("proposed_workflow") is None]
+ assert len(clear_calls) == 1
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
(