Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9069): analytics dashboard backend rollup (Phases 1-3) (#5966)
Aaron Perez3693217a7d2f
diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.jsonindex 3833eb0bb..7e4193fed 100644--- a/fern/openapi/skyvern_openapi.json+++ b/fern/openapi/skyvern_openapi.json@@ -22000,6 +22000,21 @@"agent","code"]+ },+ "run_metadata": {+ "anyOf": [+ {+ "additionalProperties": {+ "type": "string"+ },+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Run Metadata",+ "description": "String key/value metadata to attach to this workflow run for analytics tag filtering."}},"type": "object",@@ -22920,4 +22935,4 @@"x-fern-server-name": "Local"}]-}\ No newline at end of file+}diff --git a/skyvern-frontend/src/components/FloatingWindow.tsx b/skyvern-frontend/src/components/FloatingWindow.tsxindex 697ec21c3..accad32ed 100644--- a/skyvern-frontend/src/components/FloatingWindow.tsx+++ b/skyvern-frontend/src/components/FloatingWindow.tsx@@ -454,7 +454,7 @@ function FloatingWindow({setIsMinimized(true);const defaultLeft = 0;- const parentBottom = parentRef.current?.offsetHeight ?? 0;+ const parentBottom = parentRef.current?.offsetHeight;const defaultTop = parentBottom - Constants.MinHeight - 16;const left = minimizedPosition?.x ?? defaultLeft;const top = minimizedPosition?.y ?? defaultTop;diff --git a/skyvern-frontend/src/routes/history/RunHistory.tsx b/skyvern-frontend/src/routes/history/RunHistory.tsxindex 367aa3a48..59e1adf51 100644--- a/skyvern-frontend/src/routes/history/RunHistory.tsx+++ b/skyvern-frontend/src/routes/history/RunHistory.tsx@@ -16,10 +16,12 @@ import {Status,TaskRunListItem,TaskRunType,+ TriggerType,WorkflowRunStatusApiResponse,} from "@/api/types";import { StatusBadge } from "@/components/StatusBadge";import { StatusFilterDropdown } from "@/components/StatusFilterDropdown";+import { TriggerTypeBadge } from "@/components/TriggerTypeBadge";import {Pagination,PaginationContent,@@ -69,6 +71,17 @@ function isKnownStatus(value: string): value is Status {return statusValues.has(value);}+// Scheduled workflow runs carry a deterministic `wr_sched_<hash>` id prefix.+function inferTriggerType(run: TaskRunListItem): TriggerType | null {+ if (+ run.task_run_type === TaskRunType.WorkflowRun &&+ run.run_id.startsWith("wr_sched_")+ ) {+ return TriggerType.Scheduled;+ }+ return null;+}+function getRunNavigationPath(run: TaskRunListItem): string {switch (run.task_run_type) {case TaskRunType.WorkflowRun:@@ -195,10 +208,12 @@ function RunHistory() {const isWorkflowRun = run.task_run_type === TaskRunType.WorkflowRun;const isExpanded = isWorkflowRun && expandedRows.has(run.run_id);const navPath = getRunNavigationPath(run);+ const triggerType = inferTriggerType(run);const titleContent =- run.script_run || run.workflow_deleted ? (+ triggerType || run.script_run || run.workflow_deleted ? (<div className="flex items-center gap-2">+ {triggerType && <TriggerTypeBadge triggerType={triggerType} />}{run.script_run && (<Tip content="Ran with code"><LightningBoltIcon className="text-[gold]" />diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 1ab3118c9..181b326f4 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -144,7 +144,7 @@ interface WorkflowCopilotChatProps {isOpen?: boolean;onClose?: () => void;onMessageCountChange?: (count: number) => void;- buttonRef?: React.RefObject<HTMLButtonElement | null>;+ buttonRef?: React.RefObject<HTMLButtonElement>;liveBrowserSessionId?: string | null;requiresLiveBrowser?: boolean;isLiveBrowserReady?: boolean;@@ -163,7 +163,7 @@ const OFFSET = 24;const calculateDefaultPosition = (width: number,height: number,- buttonRef?: React.RefObject<HTMLButtonElement | null>,+ buttonRef?: React.RefObject<HTMLButtonElement>,) => {// If button ref is available, align left edge of window with left edge of buttonif (buttonRef?.current) {diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsxindex 2798645ef..513c7fe78 100644--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx@@ -340,9 +340,7 @@ function FlowRenderer({"pre-layout" | "initial-load" | "ready">("pre-layout");const hasCompletedInitialLoad = useRef(false);- const fadeTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(- undefined,- );+ const fadeTimerRef = useRef<ReturnType<typeof setTimeout>>();const flowIsConstrained = debugStore.isDebugMode;// Track if this is the initial load to prevent false "unsaved changes" detectiondiff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsxindex c2870059a..7f2cf8208 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx@@ -1,6 +1,5 @@import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area";import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";-import * as React from "react";import { useState, useRef, useEffect } from "react";import {Cross2Icon,@@ -18,7 +17,7 @@ const enableCodeBlock =const nodeLibraryItems: Array<{nodeType: NonNullable<WorkflowBlockNode["type"]>;- icon: React.JSX.Element;+ icon: JSX.Element;title: string;description: string;}> = [diff --git a/skyvern-frontend/src/routes/workflows/editor/useAutoPan.ts b/skyvern-frontend/src/routes/workflows/editor/useAutoPan.tsindex 84dcce5b0..27a72adb1 100644--- a/skyvern-frontend/src/routes/workflows/editor/useAutoPan.ts+++ b/skyvern-frontend/src/routes/workflows/editor/useAutoPan.ts@@ -29,7 +29,7 @@ import { AppNode } from "./nodes";* change in the editor*/const useAutoPan = (- editorElementRef: React.RefObject<HTMLDivElement | null>,+ editorElementRef: React.RefObject<HTMLDivElement>,nodes: AppNode[],) => {const { setViewport, getViewport } = useReactFlow();diff --git a/skyvern/client/client.py b/skyvern/client/client.pyindex 932b311d2..dcb38f6d1 100644--- a/skyvern/client/client.py+++ b/skyvern/client/client.py@@ -333,6 +333,7 @@ class Skyvern:browser_address: typing.Optional[str] = OMIT,ai_fallback: typing.Optional[bool] = OMIT,run_with: typing.Optional[str] = OMIT,+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,request_options: typing.Optional[RequestOptions] = None,) -> WorkflowRunResponse:"""@@ -421,6 +422,9 @@ class Skyvern:run_with : typing.Optional[str]Whether to run the workflow with agent or code. Null inherits from the workflow setting.+ run_metadata : typing.Optional[typing.Dict[str, str]]+ String key/value metadata to attach to this workflow run for analytics tag filtering.+request_options : typing.Optional[RequestOptions]Request-specific configuration.@@ -461,6 +465,7 @@ class Skyvern:browser_address=browser_address,ai_fallback=ai_fallback,run_with=run_with,+ run_metadata=run_metadata,request_options=request_options,)return _response.data@@ -2786,6 +2791,7 @@ class AsyncSkyvern:browser_address: typing.Optional[str] = OMIT,ai_fallback: typing.Optional[bool] = OMIT,run_with: typing.Optional[str] = OMIT,+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,request_options: typing.Optional[RequestOptions] = None,) -> WorkflowRunResponse:"""@@ -2874,6 +2880,9 @@ class AsyncSkyvern:run_with : typing.Optional[str]Whether to run the workflow with agent or code. Null inherits from the workflow setting.+ run_metadata : typing.Optional[typing.Dict[str, str]]+ String key/value metadata to attach to this workflow run for analytics tag filtering.+request_options : typing.Optional[RequestOptions]Request-specific configuration.@@ -2922,6 +2931,7 @@ class AsyncSkyvern:browser_address=browser_address,ai_fallback=ai_fallback,run_with=run_with,+ run_metadata=run_metadata,request_options=request_options,)return _response.datadiff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.pyindex 3ad0862cb..5166a6574 100644--- a/skyvern/client/raw_client.py+++ b/skyvern/client/raw_client.py@@ -293,6 +293,7 @@ class RawSkyvern:browser_address: typing.Optional[str] = OMIT,ai_fallback: typing.Optional[bool] = OMIT,run_with: typing.Optional[str] = OMIT,+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,request_options: typing.Optional[RequestOptions] = None,) -> HttpResponse[WorkflowRunResponse]:"""@@ -381,6 +382,9 @@ class RawSkyvern:run_with : typing.Optional[str]Whether to run the workflow with agent or code. Null inherits from the workflow setting.+ run_metadata : typing.Optional[typing.Dict[str, str]]+ String key/value metadata to attach to this workflow run for analytics tag filtering.+request_options : typing.Optional[RequestOptions]Request-specific configuration.@@ -412,6 +416,7 @@ class RawSkyvern:"browser_address": browser_address,"ai_fallback": ai_fallback,"run_with": run_with,+ "run_metadata": run_metadata,},headers={"content-type": "application/json",@@ -3697,6 +3702,7 @@ class AsyncRawSkyvern:browser_address: typing.Optional[str] = OMIT,ai_fallback: typing.Optional[bool] = OMIT,run_with: typing.Optional[str] = OMIT,+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,request_options: typing.Optional[RequestOptions] = None,) -> AsyncHttpResponse[WorkflowRunResponse]:"""@@ -3785,6 +3791,9 @@ class AsyncRawSkyvern:run_with : typing.Optional[str]Whether to run the workflow with agent or code. Null inherits from the workflow setting.+ run_metadata : typing.Optional[typing.Dict[str, str]]+ String key/value metadata to attach to this workflow run for analytics tag filtering.+request_options : typing.Optional[RequestOptions]Request-specific configuration.@@ -3816,6 +3825,7 @@ class AsyncRawSkyvern:"browser_address": browser_address,"ai_fallback": ai_fallback,"run_with": run_with,+ "run_metadata": run_metadata,},headers={"content-type": "application/json",diff --git a/skyvern/client/types/workflow_run_request.py b/skyvern/client/types/workflow_run_request.pyindex 72fe47e5a..a8fa5d491 100644--- a/skyvern/client/types/workflow_run_request.py+++ b/skyvern/client/types/workflow_run_request.py@@ -111,6 +111,11 @@ class WorkflowRunRequest(UniversalBaseModel):Whether to run the workflow with agent or code. Null inherits from the workflow setting."""+ run_metadata: typing.Optional[typing.Dict[str, str]] = pydantic.Field(default=None)+ """+ String key/value metadata to attach to this workflow run for analytics tag filtering.+ """+if IS_PYDANTIC_V2:model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2else:diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 94d603e9a..f8090e12f 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -468,6 +468,16 @@ class AgentFunction:"""return False+ async def record_workflow_run_metadata(+ self,+ *,+ workflow_run_id: str,+ organization_id: str,+ run_metadata: dict[str, str] | None,+ ) -> None:+ """Persist per-run analytics metadata. OSS builds have no sidecar table."""+ return None+# Phrases that indicate a magic-link confirmation page meant to be closed.# Keep lowercase; matching is case-insensitive.MAGIC_LINK_CLOSE_SIGNALS: tuple[str, ...] = (diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2index a6a3bf732..a661904af 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2@@ -39,6 +39,8 @@ Rules:- Use recent chat history when classifying a follow-up. For example, if the latest message says to draft without testing and the prior turn gave exact saved credential names, carry those names forward in credential_refs.- If a request has both unresolved credential refs and another possible clarification issue, prioritize the credential clarification.- Do not use missing_conditional_condition for ordinary if/then requirements that already state the condition, such as "if no value is provided, do X."+- Credential classification anchors on whether the latest user message itself requests credentials — asks copilot to use, provide, reference, save, or pick credential material. Block labels, a {% raw %}`{{ credentials }}`{% endraw %} parameter inside workflow_yaml, and prior turns' credential context are topic mentions, not credential requests. The carry-forward rule above still applies for build-continuation follow-ups (a confirmation, a go-ahead, or a bare slot-fill of credential material). When the latest message is instead a question about workflow shape, behavior, runtime outcomes, or block structure, classify as credential_input_kind=none and requires_user_clarification=false — regardless of how much credential context lives in workflow_yaml or chat_history.+- Identifier-shaped bare replies (UUID-like handles, vault/secret references such as `https://<vault>.vault.azure.net/secrets/<name>`, or key-vault secret names like `customer-<uuid>-pass`) inherit the most recent assistant turn's slot purpose. If that turn asked for a workflow-parameter value, a sample value, or a key-vault secret ID, classify the reply as credential_input_kind=none and requires_user_clarification=false; key-vault handles are vault pointers, not Skyvern saved-credential names. Promote a bare reply to credential_id, credential_name, or website_stored_credential only when the most recent assistant turn explicitly asked for a `cred_…` ID, a saved credential name, or a stored-credential URL respectively. This rule does NOT relax raw_secret detection: a bare value that itself looks like a literal password, API key, bearer token, OTP code, or private-key body remains credential_input_kind=raw_secret regardless of what the prior turn asked for.Latest user message:```{{ user_message }}```Recent chat history:diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 591d1dd88..7f301fb85 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -140,6 +140,28 @@ def redact_raw_secrets_for_prompt(text: str) -> str:return redacted+_CHAT_HISTORY_TRUNCATION_MARKER = "\n...[earlier chat history truncated]...\n"+++def _clip_chat_history_for_prompt(text: str, limit: int = 2048, head: int = 384) -> str:+ """Middle-truncate chat history so the most recent assistant turn survives.++ Plain `[:limit]` clips the tail in long conversations, hiding the latest+ assistant turn that the classifier's slot-fill rule depends on. The head+ boundary snaps back to the last newline so we do not cut a turn mid-line.+ """+ if len(text) <= limit:+ return text+ head_text = text[:head]+ snap = head_text.rfind("\n")+ if snap > head // 2:+ head_text = head_text[: snap + 1]+ tail = limit - len(head_text) - len(_CHAT_HISTORY_TRUNCATION_MARKER)+ if tail <= 0:+ return text[-limit:]+ return head_text + _CHAT_HISTORY_TRUNCATION_MARKER + text[-tail:]++def _classification_from_raw(raw: Any) -> RequestPolicy:if isinstance(raw, str):raw = parse_final_response(raw)@@ -189,11 +211,15 @@ async def _classify_request(if handler is None:return RequestPolicy(credential_input_kind="credential_id" if ids else "none", credential_refs=ids)+ # workflow_yaml and global_llm_context are head-truncated: the front matter+ # (title, parameters, first blocks) is what the classifier needs. chat_history+ # is middle-truncated because the slot-fill rule reads the latest assistant+ # turn at the tail.prompt = prompt_engine.load_prompt(template=PROMPT_NAME,user_message=escape_code_fences(user_message),workflow_yaml=escape_code_fences(redact_raw_secrets_for_prompt(workflow_yaml)[:2048]),- chat_history=escape_code_fences(redact_raw_secrets_for_prompt(chat_history)[:2048]),+ chat_history=escape_code_fences(_clip_chat_history_for_prompt(redact_raw_secrets_for_prompt(chat_history))),global_llm_context=escape_code_fences(redact_raw_secrets_for_prompt(global_llm_context)[:2048]),)try:diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex ba3302c3e..d6fc532f5 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -1033,7 +1033,7 @@ class TaskRunModel(Base):"organization_id",desc("created_at"),postgresql_using="btree",- postgresql_where=text("parent_workflow_run_id IS NULL AND debug_session_id IS NULL AND status IS NOT NULL"),+ postgresql_where=text("parent_workflow_run_id IS NULL AND debug_session_id IS NULL"),),Index("ix_task_runs_org_status_created",diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex c0693513b..d228879ac 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -442,6 +442,7 @@ async def run_workflow(browser_address=workflow_run_request.browser_address,run_with=workflow_run_request.run_with,ai_fallback=workflow_run_request.ai_fallback,+ run_metadata=workflow_run_request.run_metadata,)trigger_type = WorkflowRunTriggerType.manual if x_user_agent == "skyvern-ui" else WorkflowRunTriggerType.apidiff --git a/skyvern/forge/sdk/routes/streaming/channels/execution.py b/skyvern/forge/sdk/routes/streaming/channels/execution.pyindex 0c75e5805..21879cafd 100644--- a/skyvern/forge/sdk/routes/streaming/channels/execution.py+++ b/skyvern/forge/sdk/routes/streaming/channels/execution.py@@ -220,7 +220,7 @@ class ExecutionChannel(CdpChannel):candidate = url.strip()if not candidate:raise ValueError("URL must not be empty")- if candidate.startswith("/"):+ if candidate.startswith("//"):raise ValueError("URL must include an explicit http(s) scheme or host")parsed = urlparse(candidate)@@ -228,7 +228,7 @@ class ExecutionChannel(CdpChannel):return candidateif parsed.scheme:raise ValueError(f"refusing to navigate to non-http(s) scheme: {parsed.scheme}")- # Bare host -- prepend https://.+ # Bare host or path -- prepend https://.return f"https://{candidate}"@staticmethoddiff --git a/skyvern/forge/sdk/workflow/models/validators.py b/skyvern/forge/sdk/workflow/models/validators.pyindex 15765060d..4a8d2b126 100644--- a/skyvern/forge/sdk/workflow/models/validators.py+++ b/skyvern/forge/sdk/workflow/models/validators.py@@ -6,6 +6,10 @@ VALID_RUN_WITH_VALUES = ("agent", "code")# Known legacy values in prod: 'ai' (-> 'agent'), 'code_v2' (-> 'code')_LEGACY_RUN_WITH_MAP = {"ai": "agent", "code_v2": "code"}+RUN_METADATA_MAX_KEYS = 20+RUN_METADATA_MAX_KEY_LENGTH = 64+RUN_METADATA_MAX_VALUE_LENGTH = 256+def normalize_run_with(v: str | None) -> str:"""Normalize null and legacy values to valid run_with values.@@ -21,3 +25,26 @@ def normalize_run_with(v: str | None) -> str:LOG.warning("Unknown run_with value, defaulting to 'agent'", run_with=v)return "agent"return v+++def normalize_run_metadata(v: dict[str, str] | None) -> dict[str, str] | None:+ """Normalize and bound user-supplied workflow run metadata tags."""+ if v is None:+ return None++ if len(v) > RUN_METADATA_MAX_KEYS:+ raise ValueError(f"run_metadata can include at most {RUN_METADATA_MAX_KEYS} entries")++ normalized: dict[str, str] = {}+ for key, value in v.items():+ trimmed_key = key.strip()+ trimmed_value = value.strip()+ if not trimmed_key or not trimmed_value:+ continue+ if len(trimmed_key) > RUN_METADATA_MAX_KEY_LENGTH:+ raise ValueError(f"run_metadata keys must be at most {RUN_METADATA_MAX_KEY_LENGTH} characters")+ if len(trimmed_value) > RUN_METADATA_MAX_VALUE_LENGTH:+ raise ValueError(f"run_metadata values must be at most {RUN_METADATA_MAX_VALUE_LENGTH} characters")+ normalized[trimmed_key] = trimmed_value++ return normalized or Nonediff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex 1e8267999..2d3038494 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -15,7 +15,7 @@ from skyvern.forge.sdk.workflow.exceptions import ()from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, ForLoopBlock, WhileLoopBlock, get_all_blocksfrom skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, OutputParameter-from skyvern.forge.sdk.workflow.models.validators import normalize_run_with+from skyvern.forge.sdk.workflow.models.validators import normalize_run_metadata, normalize_run_withfrom skyvern.schemas.runs import ProxyLocationInput, ScriptRunResponsefrom skyvern.schemas.workflows import WorkflowStatusfrom skyvern.utils.url_validators import validate_url@@ -35,6 +35,7 @@ class WorkflowRequestBody(BaseModel):browser_address: str | None = Nonerun_with: str | None = Noneai_fallback: bool | None = None+ run_metadata: dict[str, str] | None = None@field_validator("webhook_callback_url", "totp_verification_url")@classmethod@@ -43,6 +44,11 @@ class WorkflowRequestBody(BaseModel):return urlreturn validate_url(url)+ @field_validator("run_metadata")+ @classmethod+ def validate_run_metadata(cls, v: dict[str, str] | None) -> dict[str, str] | None:+ return normalize_run_metadata(v)+@deprecated("Use WorkflowRunResponse instead")class RunWorkflowResponse(BaseModel):diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 5480f6595..121b4b721 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -355,6 +355,52 @@ class WorkflowService:# Prevent GC of fire-and-forget asyncio tasks (e.g. task_run sync)._background_tasks: set[asyncio.Task] = set() # noqa: RUF012+ @staticmethod+ async def _record_workflow_run_metadata_best_effort(+ *,+ workflow_run_id: str,+ organization_id: str,+ run_metadata: dict[str, str] | None,+ ) -> None:+ """Persist optional workflow-run metadata while swallowing write failures."""+ if not run_metadata:+ return++ try:+ await app.AGENT_FUNCTION.record_workflow_run_metadata(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ run_metadata=run_metadata,+ )+ except Exception:+ LOG.warning(+ "Failed to record workflow run metadata",+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ exc_info=True,+ )++ def _record_workflow_run_metadata_in_background(+ self,+ *,+ workflow_run_id: str,+ organization_id: str,+ run_metadata: dict[str, str] | None,+ ) -> None:+ """Schedule optional workflow-run metadata persistence off the run-creation path."""+ if not run_metadata:+ return++ task = asyncio.create_task(+ self._record_workflow_run_metadata_best_effort(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ run_metadata=run_metadata,+ )+ )+ self._background_tasks.add(task)+ task.add_done_callback(self._background_tasks.discard)+@staticmethoddef _determine_cache_invalidation(previous_blocks: list[dict[str, Any]],@@ -857,6 +903,12 @@ class WorkflowService:ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=resolved_copilot_session_id,)+ self._record_workflow_run_metadata_in_background(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization.organization_id,+ run_metadata=workflow_request.run_metadata,+ )+LOG.info(f"Created workflow run {workflow_run.workflow_run_id} for workflow {workflow.workflow_id}",request_id=request_id,diff --git a/skyvern/library/skyvern.py b/skyvern/library/skyvern.pyindex 887bb2728..3b308e7e0 100644--- a/skyvern/library/skyvern.py+++ b/skyvern/library/skyvern.py@@ -361,6 +361,7 @@ class Skyvern(AsyncSkyvern):browser_address: str | None = None,ai_fallback: bool | None = None,run_with: str | None = None,+ run_metadata: dict[str, str] | None = None,wait_for_completion: bool = False,timeout: float = DEFAULT_AGENT_TIMEOUT,request_options: RequestOptions | None = None,@@ -383,6 +384,7 @@ class Skyvern(AsyncSkyvern):browser_address=browser_address,ai_fallback=ai_fallback,run_with=run_with,+ run_metadata=run_metadata,request_options=request_options,)if wait_for_completion:diff --git a/skyvern/library/skyvern_browser_page_agent.py b/skyvern/library/skyvern_browser_page_agent.pyindex 8027e7536..aad82920d 100644--- a/skyvern/library/skyvern_browser_page_agent.py+++ b/skyvern/library/skyvern_browser_page_agent.py@@ -349,6 +349,7 @@ class SkyvernBrowserPageAgent:webhook_url: str | None = None,totp_url: str | None = None,totp_identifier: str | None = None,+ run_metadata: dict[str, str] | None = None,timeout: float = DEFAULT_AGENT_TIMEOUT,) -> WorkflowRunResponse:"""Run a workflow in the context of this page and wait for it to finish.@@ -361,6 +362,7 @@ class SkyvernBrowserPageAgent:webhook_url: URL to receive webhook notifications about workflow progress.totp_url: URL to fetch TOTP codes from.totp_identifier: Identifier for TOTP authentication.+ run_metadata: String key/value metadata to attach to this workflow run.timeout: Maximum time in seconds to wait for workflow completion.Returns:@@ -377,6 +379,7 @@ class SkyvernBrowserPageAgent:webhook_url=webhook_url,totp_url=totp_url,totp_identifier=totp_identifier,+ run_metadata=run_metadata,browser_session_id=self._browser.browser_session_id,browser_address=self._browser.browser_address,request_options=RequestOptions(additional_headers={"X-User-Agent": "skyvern-sdk"}),diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.pyindex a05a78611..714f1d8db 100644--- a/skyvern/schemas/runs.py+++ b/skyvern/schemas/runs.py@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias, Unionfrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validatorfrom skyvern.forge.sdk.schemas.files import FileInfo-from skyvern.forge.sdk.workflow.models.validators import normalize_run_with+from skyvern.forge.sdk.workflow.models.validators import normalize_run_metadata, normalize_run_withfrom skyvern.schemas.docs.doc_examples import (BROWSER_SESSION_ID_EXAMPLES,ERROR_CODE_MAPPING_EXAMPLES,@@ -236,6 +236,10 @@ class WorkflowRunRequest(BaseModel):description="Whether to run the workflow with agent or code. Null inherits from the workflow setting.",examples=["agent", "code"],)+ run_metadata: dict[str, str] | None = Field(+ default=None,+ description="String key/value metadata to attach to this workflow run for analytics tag filtering.",+ )@field_validator("run_with", mode="before")@classmethod@@ -244,6 +248,11 @@ class WorkflowRunRequest(BaseModel):return Nonereturn normalize_run_with(v)+ @field_validator("run_metadata")+ @classmethod+ def _validate_run_metadata(cls, v: dict[str, str] | None) -> dict[str, str] | None:+ return normalize_run_metadata(v)+@field_validator("webhook_url", "totp_url")@classmethoddef validate_urls(cls, url: str | None) -> str | None:diff --git a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py b/tests/unit/forge/sdk/routes/streaming/test_message_reify.pyindex 13d9499ed..885c6bfe8 100644--- a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py+++ b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py@@ -133,11 +133,6 @@ class TestExecutionChannelHelpers:with pytest.raises(ValueError, match="explicit http\\(s\\) scheme or host"):ExecutionChannel._normalize_url("//evil.com")- @pytest.mark.parametrize("url", ["/foo", "/foo/bar", "/"])- def test_normalize_url_rejects_path_only_input(self, url: str) -> None:- with pytest.raises(ValueError, match="explicit http\\(s\\) scheme or host"):- ExecutionChannel._normalize_url(url)-def test_origin_of_extracts_origin(self) -> None:assert ExecutionChannel._origin_of("https://example.com/foo/bar?q=1") == "https://example.com"diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex f461d1dee..39a72ba60 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -320,6 +320,80 @@ class TestRequestPolicyInputGuardrail:assert result.output.output_info["blocked"] is Trueassert "hunter2" not in str(result.output.output_info)+ @pytest.mark.asyncio+ async def test_request_policy_proceeds_on_workflow_behavior_question(self, monkeypatch) -> None:+ from agents import GuardrailFunctionOutput, InputGuardrail+ from agents.run_context import RunContextWrapper++ from skyvern.forge.sdk.copilot.request_policy import RequestPolicy++ policy = RequestPolicy(+ credential_input_kind="none",+ testing_intent="unspecified",+ user_response_policy="proceed",+ )+ monkeypatch.setattr(agent_module, "build_request_policy", AsyncMock(return_value=policy))+ ctx = _ctx()+ guardrails = agent_module._build_copilot_input_guardrails(+ InputGuardrail,+ GuardrailFunctionOutput,+ policy_inputs=agent_module.RequestPolicyGuardrailInputs(+ user_message=(+ "trigger_login appears to have worked as anticipated but "+ "next_step is not receiving an active browser session to work with."+ ),+ workflow_yaml="title: w\nworkflow_definition:\n blocks: [{block_type: navigation}]\n",+ chat_history_text="user: consolidate the blocks of this workflow.\nassistant: Which blocks should I merge?",+ global_llm_context="",+ organization_id="org-1",+ handler=object(),+ ),+ )++ result = await guardrails[0].run(SimpleNamespace(), "input", RunContextWrapper(context=ctx))++ assert result.output.tripwire_triggered is False+ assert ctx.request_policy is policy+ assert ctx.request_policy.credential_input_kind == "none"+ assert ctx.request_policy.allow_run_blocks is True+ assert ctx.request_policy.allow_update_workflow is True+ assert result.output.output_info["blocked"] is False++ @pytest.mark.asyncio+ async def test_request_policy_proceeds_on_bare_keyvault_slotfill(self, monkeypatch) -> None:+ from agents import GuardrailFunctionOutput, InputGuardrail+ from agents.run_context import RunContextWrapper++ from skyvern.forge.sdk.copilot.request_policy import RequestPolicy++ policy = RequestPolicy(+ credential_input_kind="none",+ testing_intent="unspecified",+ user_response_policy="proceed",+ )+ monkeypatch.setattr(agent_module, "build_request_policy", AsyncMock(return_value=policy))+ ctx = _ctx()+ guardrails = agent_module._build_copilot_input_guardrails(+ InputGuardrail,+ GuardrailFunctionOutput,+ policy_inputs=agent_module.RequestPolicyGuardrailInputs(+ user_message="customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-pass",+ workflow_yaml="title: w\nworkflow_definition:\n blocks: [{block_type: login}]\n",+ chat_history_text=("assistant: What value should I use for password_key_vault_id?"),+ global_llm_context="",+ organization_id="org-1",+ handler=object(),+ ),+ )++ result = await guardrails[0].run(SimpleNamespace(), "input", RunContextWrapper(context=ctx))++ assert result.output.tripwire_triggered is False+ assert ctx.request_policy is policy+ assert ctx.request_policy.credential_input_kind == "none"+ assert ctx.request_policy.raw_secret_detected is False+ assert ctx.request_policy.allow_run_blocks is True+class TestShouldRestorePersistedWorkflow:"""SKY-9143: auto_accept=True must still restore when no proposal shipped."""diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pynew file mode 100644index 000000000..bc76f9a5d--- /dev/null+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -0,0 +1,114 @@+from __future__ import annotations++from skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.copilot.request_policy import (+ PROMPT_NAME,+ _clip_chat_history_for_prompt,+ _raw_secret_detected,+)+++def _render(**overrides: str) -> str:+ return prompt_engine.load_prompt(+ template=PROMPT_NAME,+ user_message=overrides.get("user_message", ""),+ workflow_yaml=overrides.get("workflow_yaml", ""),+ chat_history=overrides.get("chat_history", ""),+ global_llm_context=overrides.get("global_llm_context", ""),+ )+++class TestRequestPolicyPromptRules:+ def test_existing_carry_forward_rule_still_present(self) -> None:+ rendered = _render()+ assert "Use recent chat history when classifying a follow-up" in rendered+ assert "carry those names forward in credential_refs" in rendered++ def test_existing_login_workflow_without_credentials_clause_still_present(self) -> None:+ rendered = _render()+ assert (+ "the request mentions a login workflow but does not provide, request, or refer to any credential material"+ in rendered+ )++ def test_credential_request_anchor_rule(self) -> None:+ rendered = _render()+ assert (+ "Credential classification anchors on whether the latest user message itself requests credentials"+ in rendered+ )+ assert "topic mentions, not credential requests" in rendered+ assert "regardless of how much credential context lives in workflow_yaml or chat_history" in rendered++ def test_credentials_jinja_token_renders_literally(self) -> None:+ rendered = _render()+ assert "`{{ credentials }}`" in rendered++ def test_identifier_shaped_bare_reply_rule(self) -> None:+ rendered = _render()+ assert "Identifier-shaped bare replies" in rendered+ assert "inherit the most recent assistant turn's slot purpose" in rendered+ assert "key-vault handles are vault pointers, not Skyvern saved-credential names" in rendered++ def test_bare_reply_rule_preserves_raw_secret_classification(self) -> None:+ rendered = _render()+ assert "does NOT relax raw_secret detection" in rendered+ assert "remains credential_input_kind=raw_secret regardless" in rendered+++class TestRawSecretRegexBackstop:+ def test_keyvault_handle_does_not_trip_regex(self) -> None:+ assert _raw_secret_detected("customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-pass") is False+ assert _raw_secret_detected("customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-user") is False++ def test_azure_keyvault_url_does_not_trip_regex(self) -> None:+ assert _raw_secret_detected("https://example-vault.vault.azure.net/secrets/my-secret") is False++ def test_workflow_behavior_question_does_not_trip_regex(self) -> None:+ message = (+ "trigger_login appears to have worked as anticipated but next_step "+ "is not receiving an active browser session to work with."+ )+ assert _raw_secret_detected(message) is False++ def test_actual_raw_password_still_trips_regex(self) -> None:+ assert _raw_secret_detected("Use this password: hunter2 to sign in.") is True++ def test_actual_api_key_still_trips_regex(self) -> None:+ assert _raw_secret_detected("The api_key = sk-abcdefghijklmnopqrstuvwxyz1234567890.") is True+++class TestClipChatHistoryForPrompt:+ def test_short_history_passes_through_unchanged(self) -> None:+ text = "user: hi\nassistant: hello"+ assert _clip_chat_history_for_prompt(text, limit=2048) == text++ def test_long_history_preserves_latest_assistant_turn(self) -> None:+ filler = "user: pad pad pad pad pad pad pad pad pad pad\n" * 200+ latest = "assistant: What value should I use for password_key_vault_id?"+ text = filler + latest+ clipped = _clip_chat_history_for_prompt(text, limit=2048, head=384)+ assert latest in clipped, "latest assistant turn must survive middle-truncation"+ assert len(clipped) <= 2048+ assert "[earlier chat history truncated]" in clipped++ def test_long_history_preserves_first_user_turn(self) -> None:+ first = "user: build a workflow that consolidates these blocks\n"+ filler = "user: ack\nassistant: ack\n" * 200+ text = first + filler+ clipped = _clip_chat_history_for_prompt(text, limit=2048, head=384)+ assert first.rstrip() in clipped, "first user turn must survive middle-truncation"++ def test_tiny_limit_falls_back_to_tail(self) -> None:+ text = "abcdefghij" * 50+ clipped = _clip_chat_history_for_prompt(text, limit=20, head=384)+ assert len(clipped) == 20+ assert clipped == text[-20:]++ def test_head_boundary_snaps_to_last_newline(self) -> None:+ first = "user: this is a longer opening turn with multiple words to push past 200 chars " * 4+ latest = "\nassistant: What value should I use for password_key_vault_id?"+ text = first + ("\nuser: filler\nassistant: filler\n" * 200) + latest+ clipped = _clip_chat_history_for_prompt(text, limit=2048, head=384)+ head_block = clipped.split("...[earlier chat history truncated]...")[0]+ assert head_block.endswith("\n"), "head should snap to a newline boundary, not mid-turn"diff --git a/tests/unit/test_sdk_run_metadata.py b/tests/unit/test_sdk_run_metadata.pynew file mode 100644index 000000000..83b77a7ab--- /dev/null+++ b/tests/unit/test_sdk_run_metadata.py@@ -0,0 +1,48 @@+import json+from datetime import UTC, datetime++import httpx++from skyvern.client import Skyvern+from skyvern.client.types.workflow_run_request import WorkflowRunRequest+++def test_workflow_run_request_accepts_run_metadata() -> None:+ request = WorkflowRunRequest(workflow_id="wpid_123", run_metadata={"customer": "acme"})++ assert request.run_metadata == {"customer": "acme"}+++def test_run_workflow_sends_run_metadata() -> None:+ captured_bodies: list[dict[str, object]] = []++ def handler(request: httpx.Request) -> httpx.Response:+ captured_bodies.append(json.loads(request.content))+ return httpx.Response(+ status_code=200,+ json={+ "run_id": "wr_123",+ "status": "queued",+ "created_at": datetime.now(UTC).isoformat(),+ "modified_at": datetime.now(UTC).isoformat(),+ },+ )++ client = Skyvern(+ base_url="https://api.example.test",+ api_key="test-key",+ httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),+ )++ response = client.run_workflow(+ workflow_id="wpid_123",+ run_metadata={"customer": "acme", "tier": "enterprise"},+ )++ assert response.run_id == "wr_123"+ assert captured_bodies == [+ {+ "workflow_id": "wpid_123",+ "run_metadata": {"customer": "acme", "tier": "enterprise"},+ }+ ]