Skip to content

Automate browser based workflows with AI

feat(SKY-9069): analytics dashboard backend rollup (Phases 1-3) (#5966)

Aaron Perez3693217a7d2f
diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.json
index 3833eb0bb..7e4193fed 100644
--- a/fern/openapi/skyvern_openapi.json
+++ b/fern/openapi/skyvern_openapi.json
@@ -22000,6 +22000,21 @@
"agent",
"code"
]
+ },
+ "run_metadata": {
+ "anyOf": [
+ {
+ "additionalProperties": {
+ "type": "string"
+ },
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Run Metadata",
+ "description": "String key/value metadata to attach to this workflow run for analytics tag filtering."
}
},
"type": "object",
@@ -22920,4 +22935,4 @@
"x-fern-server-name": "Local"
}
]
-}
\ No newline at end of file
+}
diff --git a/skyvern-frontend/src/components/FloatingWindow.tsx b/skyvern-frontend/src/components/FloatingWindow.tsx
index 697ec21c3..accad32ed 100644
--- a/skyvern-frontend/src/components/FloatingWindow.tsx
+++ b/skyvern-frontend/src/components/FloatingWindow.tsx
@@ -454,7 +454,7 @@ function FloatingWindow({
setIsMinimized(true);
const defaultLeft = 0;
- const parentBottom = parentRef.current?.offsetHeight ?? 0;
+ const parentBottom = parentRef.current?.offsetHeight;
const defaultTop = parentBottom - Constants.MinHeight - 16;
const left = minimizedPosition?.x ?? defaultLeft;
const top = minimizedPosition?.y ?? defaultTop;
diff --git a/skyvern-frontend/src/routes/history/RunHistory.tsx b/skyvern-frontend/src/routes/history/RunHistory.tsx
index 367aa3a48..59e1adf51 100644
--- a/skyvern-frontend/src/routes/history/RunHistory.tsx
+++ b/skyvern-frontend/src/routes/history/RunHistory.tsx
@@ -16,10 +16,12 @@ import {
Status,
TaskRunListItem,
TaskRunType,
+ TriggerType,
WorkflowRunStatusApiResponse,
} from "@/api/types";
import { StatusBadge } from "@/components/StatusBadge";
import { StatusFilterDropdown } from "@/components/StatusFilterDropdown";
+import { TriggerTypeBadge } from "@/components/TriggerTypeBadge";
import {
Pagination,
PaginationContent,
@@ -69,6 +71,17 @@ function isKnownStatus(value: string): value is Status {
return statusValues.has(value);
}
+// Scheduled workflow runs carry a deterministic `wr_sched_<hash>` id prefix.
+function inferTriggerType(run: TaskRunListItem): TriggerType | null {
+ if (
+ run.task_run_type === TaskRunType.WorkflowRun &&
+ run.run_id.startsWith("wr_sched_")
+ ) {
+ return TriggerType.Scheduled;
+ }
+ return null;
+}
+
function getRunNavigationPath(run: TaskRunListItem): string {
switch (run.task_run_type) {
case TaskRunType.WorkflowRun:
@@ -195,10 +208,12 @@ function RunHistory() {
const isWorkflowRun = run.task_run_type === TaskRunType.WorkflowRun;
const isExpanded = isWorkflowRun && expandedRows.has(run.run_id);
const navPath = getRunNavigationPath(run);
+ const triggerType = inferTriggerType(run);
const titleContent =
- run.script_run || run.workflow_deleted ? (
+ triggerType || run.script_run || run.workflow_deleted ? (
<div className="flex items-center gap-2">
+ {triggerType && <TriggerTypeBadge triggerType={triggerType} />}
{run.script_run && (
<Tip content="Ran with code">
<LightningBoltIcon className="text-[gold]" />
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 1ab3118c9..181b326f4 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -144,7 +144,7 @@ interface WorkflowCopilotChatProps {
isOpen?: boolean;
onClose?: () => void;
onMessageCountChange?: (count: number) => void;
- buttonRef?: React.RefObject<HTMLButtonElement | null>;
+ buttonRef?: React.RefObject<HTMLButtonElement>;
liveBrowserSessionId?: string | null;
requiresLiveBrowser?: boolean;
isLiveBrowserReady?: boolean;
@@ -163,7 +163,7 @@ const OFFSET = 24;
const calculateDefaultPosition = (
width: number,
height: number,
- buttonRef?: React.RefObject<HTMLButtonElement | null>,
+ buttonRef?: React.RefObject<HTMLButtonElement>,
) => {
// If button ref is available, align left edge of window with left edge of button
if (buttonRef?.current) {
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
index 2798645ef..513c7fe78 100644
--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
@@ -340,9 +340,7 @@ function FlowRenderer({
"pre-layout" | "initial-load" | "ready"
>("pre-layout");
const hasCompletedInitialLoad = useRef(false);
- const fadeTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(
- undefined,
- );
+ const fadeTimerRef = useRef<ReturnType<typeof setTimeout>>();
const flowIsConstrained = debugStore.isDebugMode;
// Track if this is the initial load to prevent false "unsaved changes" detection
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
index c2870059a..7f2cf8208 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
@@ -1,6 +1,5 @@
import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
-import * as React from "react";
import { useState, useRef, useEffect } from "react";
import {
Cross2Icon,
@@ -18,7 +17,7 @@ const enableCodeBlock =
const nodeLibraryItems: Array<{
nodeType: NonNullable<WorkflowBlockNode["type"]>;
- icon: React.JSX.Element;
+ icon: JSX.Element;
title: string;
description: string;
}> = [
diff --git a/skyvern-frontend/src/routes/workflows/editor/useAutoPan.ts b/skyvern-frontend/src/routes/workflows/editor/useAutoPan.ts
index 84dcce5b0..27a72adb1 100644
--- a/skyvern-frontend/src/routes/workflows/editor/useAutoPan.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/useAutoPan.ts
@@ -29,7 +29,7 @@ import { AppNode } from "./nodes";
* change in the editor
*/
const useAutoPan = (
- editorElementRef: React.RefObject<HTMLDivElement | null>,
+ editorElementRef: React.RefObject<HTMLDivElement>,
nodes: AppNode[],
) => {
const { setViewport, getViewport } = useReactFlow();
diff --git a/skyvern/client/client.py b/skyvern/client/client.py
index 932b311d2..dcb38f6d1 100644
--- a/skyvern/client/client.py
+++ b/skyvern/client/client.py
@@ -333,6 +333,7 @@ class Skyvern:
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
run_with: typing.Optional[str] = OMIT,
+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,
request_options: typing.Optional[RequestOptions] = None,
) -> WorkflowRunResponse:
"""
@@ -421,6 +422,9 @@ class Skyvern:
run_with : typing.Optional[str]
Whether to run the workflow with agent or code. Null inherits from the workflow setting.
+ run_metadata : typing.Optional[typing.Dict[str, str]]
+ String key/value metadata to attach to this workflow run for analytics tag filtering.
+
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
@@ -461,6 +465,7 @@ class Skyvern:
browser_address=browser_address,
ai_fallback=ai_fallback,
run_with=run_with,
+ run_metadata=run_metadata,
request_options=request_options,
)
return _response.data
@@ -2786,6 +2791,7 @@ class AsyncSkyvern:
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
run_with: typing.Optional[str] = OMIT,
+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,
request_options: typing.Optional[RequestOptions] = None,
) -> WorkflowRunResponse:
"""
@@ -2874,6 +2880,9 @@ class AsyncSkyvern:
run_with : typing.Optional[str]
Whether to run the workflow with agent or code. Null inherits from the workflow setting.
+ run_metadata : typing.Optional[typing.Dict[str, str]]
+ String key/value metadata to attach to this workflow run for analytics tag filtering.
+
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
@@ -2922,6 +2931,7 @@ class AsyncSkyvern:
browser_address=browser_address,
ai_fallback=ai_fallback,
run_with=run_with,
+ run_metadata=run_metadata,
request_options=request_options,
)
return _response.data
diff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.py
index 3ad0862cb..5166a6574 100644
--- a/skyvern/client/raw_client.py
+++ b/skyvern/client/raw_client.py
@@ -293,6 +293,7 @@ class RawSkyvern:
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
run_with: typing.Optional[str] = OMIT,
+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,
request_options: typing.Optional[RequestOptions] = None,
) -> HttpResponse[WorkflowRunResponse]:
"""
@@ -381,6 +382,9 @@ class RawSkyvern:
run_with : typing.Optional[str]
Whether to run the workflow with agent or code. Null inherits from the workflow setting.
+ run_metadata : typing.Optional[typing.Dict[str, str]]
+ String key/value metadata to attach to this workflow run for analytics tag filtering.
+
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
@@ -412,6 +416,7 @@ class RawSkyvern:
"browser_address": browser_address,
"ai_fallback": ai_fallback,
"run_with": run_with,
+ "run_metadata": run_metadata,
},
headers={
"content-type": "application/json",
@@ -3697,6 +3702,7 @@ class AsyncRawSkyvern:
browser_address: typing.Optional[str] = OMIT,
ai_fallback: typing.Optional[bool] = OMIT,
run_with: typing.Optional[str] = OMIT,
+ run_metadata: typing.Optional[typing.Dict[str, str]] = OMIT,
request_options: typing.Optional[RequestOptions] = None,
) -> AsyncHttpResponse[WorkflowRunResponse]:
"""
@@ -3785,6 +3791,9 @@ class AsyncRawSkyvern:
run_with : typing.Optional[str]
Whether to run the workflow with agent or code. Null inherits from the workflow setting.
+ run_metadata : typing.Optional[typing.Dict[str, str]]
+ String key/value metadata to attach to this workflow run for analytics tag filtering.
+
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
@@ -3816,6 +3825,7 @@ class AsyncRawSkyvern:
"browser_address": browser_address,
"ai_fallback": ai_fallback,
"run_with": run_with,
+ "run_metadata": run_metadata,
},
headers={
"content-type": "application/json",
diff --git a/skyvern/client/types/workflow_run_request.py b/skyvern/client/types/workflow_run_request.py
index 72fe47e5a..a8fa5d491 100644
--- a/skyvern/client/types/workflow_run_request.py
+++ b/skyvern/client/types/workflow_run_request.py
@@ -111,6 +111,11 @@ class WorkflowRunRequest(UniversalBaseModel):
Whether to run the workflow with agent or code. Null inherits from the workflow setting.
"""
+ run_metadata: typing.Optional[typing.Dict[str, str]] = pydantic.Field(default=None)
+ """
+ String key/value metadata to attach to this workflow run for analytics tag filtering.
+ """
+
if IS_PYDANTIC_V2:
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
else:
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 94d603e9a..f8090e12f 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -468,6 +468,16 @@ class AgentFunction:
"""
return False
+ async def record_workflow_run_metadata(
+ self,
+ *,
+ workflow_run_id: str,
+ organization_id: str,
+ run_metadata: dict[str, str] | None,
+ ) -> None:
+ """Persist per-run analytics metadata. OSS builds have no sidecar table."""
+ return None
+
# Phrases that indicate a magic-link confirmation page meant to be closed.
# Keep lowercase; matching is case-insensitive.
MAGIC_LINK_CLOSE_SIGNALS: tuple[str, ...] = (
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index a6a3bf732..a661904af 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -39,6 +39,8 @@ Rules:
- Use recent chat history when classifying a follow-up. For example, if the latest message says to draft without testing and the prior turn gave exact saved credential names, carry those names forward in credential_refs.
- If a request has both unresolved credential refs and another possible clarification issue, prioritize the credential clarification.
- Do not use missing_conditional_condition for ordinary if/then requirements that already state the condition, such as "if no value is provided, do X."
+- Credential classification anchors on whether the latest user message itself requests credentials — asks copilot to use, provide, reference, save, or pick credential material. Block labels, a {% raw %}`{{ credentials }}`{% endraw %} parameter inside workflow_yaml, and prior turns' credential context are topic mentions, not credential requests. The carry-forward rule above still applies for build-continuation follow-ups (a confirmation, a go-ahead, or a bare slot-fill of credential material). When the latest message is instead a question about workflow shape, behavior, runtime outcomes, or block structure, classify as credential_input_kind=none and requires_user_clarification=false — regardless of how much credential context lives in workflow_yaml or chat_history.
+- Identifier-shaped bare replies (UUID-like handles, vault/secret references such as `https://<vault>.vault.azure.net/secrets/<name>`, or key-vault secret names like `customer-<uuid>-pass`) inherit the most recent assistant turn's slot purpose. If that turn asked for a workflow-parameter value, a sample value, or a key-vault secret ID, classify the reply as credential_input_kind=none and requires_user_clarification=false; key-vault handles are vault pointers, not Skyvern saved-credential names. Promote a bare reply to credential_id, credential_name, or website_stored_credential only when the most recent assistant turn explicitly asked for a `cred_…` ID, a saved credential name, or a stored-credential URL respectively. This rule does NOT relax raw_secret detection: a bare value that itself looks like a literal password, API key, bearer token, OTP code, or private-key body remains credential_input_kind=raw_secret regardless of what the prior turn asked for.
Latest user message:
```{{ user_message }}```
Recent chat history:
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 591d1dd88..7f301fb85 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -140,6 +140,28 @@ def redact_raw_secrets_for_prompt(text: str) -> str:
return redacted
+_CHAT_HISTORY_TRUNCATION_MARKER = "\n...[earlier chat history truncated]...\n"
+
+
+def _clip_chat_history_for_prompt(text: str, limit: int = 2048, head: int = 384) -> str:
+ """Middle-truncate chat history so the most recent assistant turn survives.
+
+ Plain `[:limit]` clips the tail in long conversations, hiding the latest
+ assistant turn that the classifier's slot-fill rule depends on. The head
+ boundary snaps back to the last newline so we do not cut a turn mid-line.
+ """
+ if len(text) <= limit:
+ return text
+ head_text = text[:head]
+ snap = head_text.rfind("\n")
+ if snap > head // 2:
+ head_text = head_text[: snap + 1]
+ tail = limit - len(head_text) - len(_CHAT_HISTORY_TRUNCATION_MARKER)
+ if tail <= 0:
+ return text[-limit:]
+ return head_text + _CHAT_HISTORY_TRUNCATION_MARKER + text[-tail:]
+
+
def _classification_from_raw(raw: Any) -> RequestPolicy:
if isinstance(raw, str):
raw = parse_final_response(raw)
@@ -189,11 +211,15 @@ async def _classify_request(
if handler is None:
return RequestPolicy(credential_input_kind="credential_id" if ids else "none", credential_refs=ids)
+ # workflow_yaml and global_llm_context are head-truncated: the front matter
+ # (title, parameters, first blocks) is what the classifier needs. chat_history
+ # is middle-truncated because the slot-fill rule reads the latest assistant
+ # turn at the tail.
prompt = prompt_engine.load_prompt(
template=PROMPT_NAME,
user_message=escape_code_fences(user_message),
workflow_yaml=escape_code_fences(redact_raw_secrets_for_prompt(workflow_yaml)[:2048]),
- chat_history=escape_code_fences(redact_raw_secrets_for_prompt(chat_history)[:2048]),
+ chat_history=escape_code_fences(_clip_chat_history_for_prompt(redact_raw_secrets_for_prompt(chat_history))),
global_llm_context=escape_code_fences(redact_raw_secrets_for_prompt(global_llm_context)[:2048]),
)
try:
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index ba3302c3e..d6fc532f5 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -1033,7 +1033,7 @@ class TaskRunModel(Base):
"organization_id",
desc("created_at"),
postgresql_using="btree",
- postgresql_where=text("parent_workflow_run_id IS NULL AND debug_session_id IS NULL AND status IS NOT NULL"),
+ postgresql_where=text("parent_workflow_run_id IS NULL AND debug_session_id IS NULL"),
),
Index(
"ix_task_runs_org_status_created",
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index c0693513b..d228879ac 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -442,6 +442,7 @@ async def run_workflow(
browser_address=workflow_run_request.browser_address,
run_with=workflow_run_request.run_with,
ai_fallback=workflow_run_request.ai_fallback,
+ run_metadata=workflow_run_request.run_metadata,
)
trigger_type = WorkflowRunTriggerType.manual if x_user_agent == "skyvern-ui" else WorkflowRunTriggerType.api
diff --git a/skyvern/forge/sdk/routes/streaming/channels/execution.py b/skyvern/forge/sdk/routes/streaming/channels/execution.py
index 0c75e5805..21879cafd 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/execution.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/execution.py
@@ -220,7 +220,7 @@ class ExecutionChannel(CdpChannel):
candidate = url.strip()
if not candidate:
raise ValueError("URL must not be empty")
- if candidate.startswith("/"):
+ if candidate.startswith("//"):
raise ValueError("URL must include an explicit http(s) scheme or host")
parsed = urlparse(candidate)
@@ -228,7 +228,7 @@ class ExecutionChannel(CdpChannel):
return candidate
if parsed.scheme:
raise ValueError(f"refusing to navigate to non-http(s) scheme: {parsed.scheme}")
- # Bare host -- prepend https://.
+ # Bare host or path -- prepend https://.
return f"https://{candidate}"
@staticmethod
diff --git a/skyvern/forge/sdk/workflow/models/validators.py b/skyvern/forge/sdk/workflow/models/validators.py
index 15765060d..4a8d2b126 100644
--- a/skyvern/forge/sdk/workflow/models/validators.py
+++ b/skyvern/forge/sdk/workflow/models/validators.py
@@ -6,6 +6,10 @@ VALID_RUN_WITH_VALUES = ("agent", "code")
# Known legacy values in prod: 'ai' (-> 'agent'), 'code_v2' (-> 'code')
_LEGACY_RUN_WITH_MAP = {"ai": "agent", "code_v2": "code"}
+RUN_METADATA_MAX_KEYS = 20
+RUN_METADATA_MAX_KEY_LENGTH = 64
+RUN_METADATA_MAX_VALUE_LENGTH = 256
+
def normalize_run_with(v: str | None) -> str:
"""Normalize null and legacy values to valid run_with values.
@@ -21,3 +25,26 @@ def normalize_run_with(v: str | None) -> str:
LOG.warning("Unknown run_with value, defaulting to 'agent'", run_with=v)
return "agent"
return v
+
+
+def normalize_run_metadata(v: dict[str, str] | None) -> dict[str, str] | None:
+ """Normalize and bound user-supplied workflow run metadata tags."""
+ if v is None:
+ return None
+
+ if len(v) > RUN_METADATA_MAX_KEYS:
+ raise ValueError(f"run_metadata can include at most {RUN_METADATA_MAX_KEYS} entries")
+
+ normalized: dict[str, str] = {}
+ for key, value in v.items():
+ trimmed_key = key.strip()
+ trimmed_value = value.strip()
+ if not trimmed_key or not trimmed_value:
+ continue
+ if len(trimmed_key) > RUN_METADATA_MAX_KEY_LENGTH:
+ raise ValueError(f"run_metadata keys must be at most {RUN_METADATA_MAX_KEY_LENGTH} characters")
+ if len(trimmed_value) > RUN_METADATA_MAX_VALUE_LENGTH:
+ raise ValueError(f"run_metadata values must be at most {RUN_METADATA_MAX_VALUE_LENGTH} characters")
+ normalized[trimmed_key] = trimmed_value
+
+ return normalized or None
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 1e8267999..2d3038494 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -15,7 +15,7 @@ from skyvern.forge.sdk.workflow.exceptions import (
)
from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, ForLoopBlock, WhileLoopBlock, get_all_blocks
from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, OutputParameter
-from skyvern.forge.sdk.workflow.models.validators import normalize_run_with
+from skyvern.forge.sdk.workflow.models.validators import normalize_run_metadata, normalize_run_with
from skyvern.schemas.runs import ProxyLocationInput, ScriptRunResponse
from skyvern.schemas.workflows import WorkflowStatus
from skyvern.utils.url_validators import validate_url
@@ -35,6 +35,7 @@ class WorkflowRequestBody(BaseModel):
browser_address: str | None = None
run_with: str | None = None
ai_fallback: bool | None = None
+ run_metadata: dict[str, str] | None = None
@field_validator("webhook_callback_url", "totp_verification_url")
@classmethod
@@ -43,6 +44,11 @@ class WorkflowRequestBody(BaseModel):
return url
return validate_url(url)
+ @field_validator("run_metadata")
+ @classmethod
+ def validate_run_metadata(cls, v: dict[str, str] | None) -> dict[str, str] | None:
+ return normalize_run_metadata(v)
+
@deprecated("Use WorkflowRunResponse instead")
class RunWorkflowResponse(BaseModel):
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 5480f6595..121b4b721 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -355,6 +355,52 @@ class WorkflowService:
# Prevent GC of fire-and-forget asyncio tasks (e.g. task_run sync).
_background_tasks: set[asyncio.Task] = set() # noqa: RUF012
+ @staticmethod
+ async def _record_workflow_run_metadata_best_effort(
+ *,
+ workflow_run_id: str,
+ organization_id: str,
+ run_metadata: dict[str, str] | None,
+ ) -> None:
+ """Persist optional workflow-run metadata while swallowing write failures."""
+ if not run_metadata:
+ return
+
+ try:
+ await app.AGENT_FUNCTION.record_workflow_run_metadata(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ run_metadata=run_metadata,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to record workflow run metadata",
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+
+ def _record_workflow_run_metadata_in_background(
+ self,
+ *,
+ workflow_run_id: str,
+ organization_id: str,
+ run_metadata: dict[str, str] | None,
+ ) -> None:
+ """Schedule optional workflow-run metadata persistence off the run-creation path."""
+ if not run_metadata:
+ return
+
+ task = asyncio.create_task(
+ self._record_workflow_run_metadata_best_effort(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ run_metadata=run_metadata,
+ )
+ )
+ self._background_tasks.add(task)
+ task.add_done_callback(self._background_tasks.discard)
+
@staticmethod
def _determine_cache_invalidation(
previous_blocks: list[dict[str, Any]],
@@ -857,6 +903,12 @@ class WorkflowService:
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=resolved_copilot_session_id,
)
+ self._record_workflow_run_metadata_in_background(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization.organization_id,
+ run_metadata=workflow_request.run_metadata,
+ )
+
LOG.info(
f"Created workflow run {workflow_run.workflow_run_id} for workflow {workflow.workflow_id}",
request_id=request_id,
diff --git a/skyvern/library/skyvern.py b/skyvern/library/skyvern.py
index 887bb2728..3b308e7e0 100644
--- a/skyvern/library/skyvern.py
+++ b/skyvern/library/skyvern.py
@@ -361,6 +361,7 @@ class Skyvern(AsyncSkyvern):
browser_address: str | None = None,
ai_fallback: bool | None = None,
run_with: str | None = None,
+ run_metadata: dict[str, str] | None = None,
wait_for_completion: bool = False,
timeout: float = DEFAULT_AGENT_TIMEOUT,
request_options: RequestOptions | None = None,
@@ -383,6 +384,7 @@ class Skyvern(AsyncSkyvern):
browser_address=browser_address,
ai_fallback=ai_fallback,
run_with=run_with,
+ run_metadata=run_metadata,
request_options=request_options,
)
if wait_for_completion:
diff --git a/skyvern/library/skyvern_browser_page_agent.py b/skyvern/library/skyvern_browser_page_agent.py
index 8027e7536..aad82920d 100644
--- a/skyvern/library/skyvern_browser_page_agent.py
+++ b/skyvern/library/skyvern_browser_page_agent.py
@@ -349,6 +349,7 @@ class SkyvernBrowserPageAgent:
webhook_url: str | None = None,
totp_url: str | None = None,
totp_identifier: str | None = None,
+ run_metadata: dict[str, str] | None = None,
timeout: float = DEFAULT_AGENT_TIMEOUT,
) -> WorkflowRunResponse:
"""Run a workflow in the context of this page and wait for it to finish.
@@ -361,6 +362,7 @@ class SkyvernBrowserPageAgent:
webhook_url: URL to receive webhook notifications about workflow progress.
totp_url: URL to fetch TOTP codes from.
totp_identifier: Identifier for TOTP authentication.
+ run_metadata: String key/value metadata to attach to this workflow run.
timeout: Maximum time in seconds to wait for workflow completion.
Returns:
@@ -377,6 +379,7 @@ class SkyvernBrowserPageAgent:
webhook_url=webhook_url,
totp_url=totp_url,
totp_identifier=totp_identifier,
+ run_metadata=run_metadata,
browser_session_id=self._browser.browser_session_id,
browser_address=self._browser.browser_address,
request_options=RequestOptions(additional_headers={"X-User-Agent": "skyvern-sdk"}),
diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.py
index a05a78611..714f1d8db 100644
--- a/skyvern/schemas/runs.py
+++ b/skyvern/schemas/runs.py
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias, Union
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from skyvern.forge.sdk.schemas.files import FileInfo
-from skyvern.forge.sdk.workflow.models.validators import normalize_run_with
+from skyvern.forge.sdk.workflow.models.validators import normalize_run_metadata, normalize_run_with
from skyvern.schemas.docs.doc_examples import (
BROWSER_SESSION_ID_EXAMPLES,
ERROR_CODE_MAPPING_EXAMPLES,
@@ -236,6 +236,10 @@ class WorkflowRunRequest(BaseModel):
description="Whether to run the workflow with agent or code. Null inherits from the workflow setting.",
examples=["agent", "code"],
)
+ run_metadata: dict[str, str] | None = Field(
+ default=None,
+ description="String key/value metadata to attach to this workflow run for analytics tag filtering.",
+ )
@field_validator("run_with", mode="before")
@classmethod
@@ -244,6 +248,11 @@ class WorkflowRunRequest(BaseModel):
return None
return normalize_run_with(v)
+ @field_validator("run_metadata")
+ @classmethod
+ def _validate_run_metadata(cls, v: dict[str, str] | None) -> dict[str, str] | None:
+ return normalize_run_metadata(v)
+
@field_validator("webhook_url", "totp_url")
@classmethod
def validate_urls(cls, url: str | None) -> str | None:
diff --git a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
index 13d9499ed..885c6bfe8 100644
--- a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
+++ b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py
@@ -133,11 +133,6 @@ class TestExecutionChannelHelpers:
with pytest.raises(ValueError, match="explicit http\\(s\\) scheme or host"):
ExecutionChannel._normalize_url("//evil.com")
- @pytest.mark.parametrize("url", ["/foo", "/foo/bar", "/"])
- def test_normalize_url_rejects_path_only_input(self, url: str) -> None:
- with pytest.raises(ValueError, match="explicit http\\(s\\) scheme or host"):
- ExecutionChannel._normalize_url(url)
-
def test_origin_of_extracts_origin(self) -> None:
assert ExecutionChannel._origin_of("https://example.com/foo/bar?q=1") == "https://example.com"
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index f461d1dee..39a72ba60 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -320,6 +320,80 @@ class TestRequestPolicyInputGuardrail:
assert result.output.output_info["blocked"] is True
assert "hunter2" not in str(result.output.output_info)
+ @pytest.mark.asyncio
+ async def test_request_policy_proceeds_on_workflow_behavior_question(self, monkeypatch) -> None:
+ from agents import GuardrailFunctionOutput, InputGuardrail
+ from agents.run_context import RunContextWrapper
+
+ from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+
+ policy = RequestPolicy(
+ credential_input_kind="none",
+ testing_intent="unspecified",
+ user_response_policy="proceed",
+ )
+ monkeypatch.setattr(agent_module, "build_request_policy", AsyncMock(return_value=policy))
+ ctx = _ctx()
+ guardrails = agent_module._build_copilot_input_guardrails(
+ InputGuardrail,
+ GuardrailFunctionOutput,
+ policy_inputs=agent_module.RequestPolicyGuardrailInputs(
+ user_message=(
+ "trigger_login appears to have worked as anticipated but "
+ "next_step is not receiving an active browser session to work with."
+ ),
+ workflow_yaml="title: w\nworkflow_definition:\n blocks: [{block_type: navigation}]\n",
+ chat_history_text="user: consolidate the blocks of this workflow.\nassistant: Which blocks should I merge?",
+ global_llm_context="",
+ organization_id="org-1",
+ handler=object(),
+ ),
+ )
+
+ result = await guardrails[0].run(SimpleNamespace(), "input", RunContextWrapper(context=ctx))
+
+ assert result.output.tripwire_triggered is False
+ assert ctx.request_policy is policy
+ assert ctx.request_policy.credential_input_kind == "none"
+ assert ctx.request_policy.allow_run_blocks is True
+ assert ctx.request_policy.allow_update_workflow is True
+ assert result.output.output_info["blocked"] is False
+
+ @pytest.mark.asyncio
+ async def test_request_policy_proceeds_on_bare_keyvault_slotfill(self, monkeypatch) -> None:
+ from agents import GuardrailFunctionOutput, InputGuardrail
+ from agents.run_context import RunContextWrapper
+
+ from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+
+ policy = RequestPolicy(
+ credential_input_kind="none",
+ testing_intent="unspecified",
+ user_response_policy="proceed",
+ )
+ monkeypatch.setattr(agent_module, "build_request_policy", AsyncMock(return_value=policy))
+ ctx = _ctx()
+ guardrails = agent_module._build_copilot_input_guardrails(
+ InputGuardrail,
+ GuardrailFunctionOutput,
+ policy_inputs=agent_module.RequestPolicyGuardrailInputs(
+ user_message="customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-pass",
+ workflow_yaml="title: w\nworkflow_definition:\n blocks: [{block_type: login}]\n",
+ chat_history_text=("assistant: What value should I use for password_key_vault_id?"),
+ global_llm_context="",
+ organization_id="org-1",
+ handler=object(),
+ ),
+ )
+
+ result = await guardrails[0].run(SimpleNamespace(), "input", RunContextWrapper(context=ctx))
+
+ assert result.output.tripwire_triggered is False
+ assert ctx.request_policy is policy
+ assert ctx.request_policy.credential_input_kind == "none"
+ assert ctx.request_policy.raw_secret_detected is False
+ assert ctx.request_policy.allow_run_blocks is True
+
class TestShouldRestorePersistedWorkflow:
"""SKY-9143: auto_accept=True must still restore when no proposal shipped."""
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
new file mode 100644
index 000000000..bc76f9a5d
--- /dev/null
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.copilot.request_policy import (
+ PROMPT_NAME,
+ _clip_chat_history_for_prompt,
+ _raw_secret_detected,
+)
+
+
+def _render(**overrides: str) -> str:
+ return prompt_engine.load_prompt(
+ template=PROMPT_NAME,
+ user_message=overrides.get("user_message", ""),
+ workflow_yaml=overrides.get("workflow_yaml", ""),
+ chat_history=overrides.get("chat_history", ""),
+ global_llm_context=overrides.get("global_llm_context", ""),
+ )
+
+
+class TestRequestPolicyPromptRules:
+ def test_existing_carry_forward_rule_still_present(self) -> None:
+ rendered = _render()
+ assert "Use recent chat history when classifying a follow-up" in rendered
+ assert "carry those names forward in credential_refs" in rendered
+
+ def test_existing_login_workflow_without_credentials_clause_still_present(self) -> None:
+ rendered = _render()
+ assert (
+ "the request mentions a login workflow but does not provide, request, or refer to any credential material"
+ in rendered
+ )
+
+ def test_credential_request_anchor_rule(self) -> None:
+ rendered = _render()
+ assert (
+ "Credential classification anchors on whether the latest user message itself requests credentials"
+ in rendered
+ )
+ assert "topic mentions, not credential requests" in rendered
+ assert "regardless of how much credential context lives in workflow_yaml or chat_history" in rendered
+
+ def test_credentials_jinja_token_renders_literally(self) -> None:
+ rendered = _render()
+ assert "`{{ credentials }}`" in rendered
+
+ def test_identifier_shaped_bare_reply_rule(self) -> None:
+ rendered = _render()
+ assert "Identifier-shaped bare replies" in rendered
+ assert "inherit the most recent assistant turn's slot purpose" in rendered
+ assert "key-vault handles are vault pointers, not Skyvern saved-credential names" in rendered
+
+ def test_bare_reply_rule_preserves_raw_secret_classification(self) -> None:
+ rendered = _render()
+ assert "does NOT relax raw_secret detection" in rendered
+ assert "remains credential_input_kind=raw_secret regardless" in rendered
+
+
+class TestRawSecretRegexBackstop:
+ def test_keyvault_handle_does_not_trip_regex(self) -> None:
+ assert _raw_secret_detected("customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-pass") is False
+ assert _raw_secret_detected("customer-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee-user") is False
+
+ def test_azure_keyvault_url_does_not_trip_regex(self) -> None:
+ assert _raw_secret_detected("https://example-vault.vault.azure.net/secrets/my-secret") is False
+
+ def test_workflow_behavior_question_does_not_trip_regex(self) -> None:
+ message = (
+ "trigger_login appears to have worked as anticipated but next_step "
+ "is not receiving an active browser session to work with."
+ )
+ assert _raw_secret_detected(message) is False
+
+ def test_actual_raw_password_still_trips_regex(self) -> None:
+ assert _raw_secret_detected("Use this password: hunter2 to sign in.") is True
+
+ def test_actual_api_key_still_trips_regex(self) -> None:
+ assert _raw_secret_detected("The api_key = sk-abcdefghijklmnopqrstuvwxyz1234567890.") is True
+
+
+class TestClipChatHistoryForPrompt:
+ def test_short_history_passes_through_unchanged(self) -> None:
+ text = "user: hi\nassistant: hello"
+ assert _clip_chat_history_for_prompt(text, limit=2048) == text
+
+ def test_long_history_preserves_latest_assistant_turn(self) -> None:
+ filler = "user: pad pad pad pad pad pad pad pad pad pad\n" * 200
+ latest = "assistant: What value should I use for password_key_vault_id?"
+ text = filler + latest
+ clipped = _clip_chat_history_for_prompt(text, limit=2048, head=384)
+ assert latest in clipped, "latest assistant turn must survive middle-truncation"
+ assert len(clipped) <= 2048
+ assert "[earlier chat history truncated]" in clipped
+
+ def test_long_history_preserves_first_user_turn(self) -> None:
+ first = "user: build a workflow that consolidates these blocks\n"
+ filler = "user: ack\nassistant: ack\n" * 200
+ text = first + filler
+ clipped = _clip_chat_history_for_prompt(text, limit=2048, head=384)
+ assert first.rstrip() in clipped, "first user turn must survive middle-truncation"
+
+ def test_tiny_limit_falls_back_to_tail(self) -> None:
+ text = "abcdefghij" * 50
+ clipped = _clip_chat_history_for_prompt(text, limit=20, head=384)
+ assert len(clipped) == 20
+ assert clipped == text[-20:]
+
+ def test_head_boundary_snaps_to_last_newline(self) -> None:
+ first = "user: this is a longer opening turn with multiple words to push past 200 chars " * 4
+ latest = "\nassistant: What value should I use for password_key_vault_id?"
+ text = first + ("\nuser: filler\nassistant: filler\n" * 200) + latest
+ clipped = _clip_chat_history_for_prompt(text, limit=2048, head=384)
+ head_block = clipped.split("...[earlier chat history truncated]...")[0]
+ assert head_block.endswith("\n"), "head should snap to a newline boundary, not mid-turn"
diff --git a/tests/unit/test_sdk_run_metadata.py b/tests/unit/test_sdk_run_metadata.py
new file mode 100644
index 000000000..83b77a7ab
--- /dev/null
+++ b/tests/unit/test_sdk_run_metadata.py
@@ -0,0 +1,48 @@
+import json
+from datetime import UTC, datetime
+
+import httpx
+
+from skyvern.client import Skyvern
+from skyvern.client.types.workflow_run_request import WorkflowRunRequest
+
+
+def test_workflow_run_request_accepts_run_metadata() -> None:
+ request = WorkflowRunRequest(workflow_id="wpid_123", run_metadata={"customer": "acme"})
+
+ assert request.run_metadata == {"customer": "acme"}
+
+
+def test_run_workflow_sends_run_metadata() -> None:
+ captured_bodies: list[dict[str, object]] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ captured_bodies.append(json.loads(request.content))
+ return httpx.Response(
+ status_code=200,
+ json={
+ "run_id": "wr_123",
+ "status": "queued",
+ "created_at": datetime.now(UTC).isoformat(),
+ "modified_at": datetime.now(UTC).isoformat(),
+ },
+ )
+
+ client = Skyvern(
+ base_url="https://api.example.test",
+ api_key="test-key",
+ httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),
+ )
+
+ response = client.run_workflow(
+ workflow_id="wpid_123",
+ run_metadata={"customer": "acme", "tier": "enterprise"},
+ )
+
+ assert response.run_id == "wr_123"
+ assert captured_bodies == [
+ {
+ "workflow_id": "wpid_123",
+ "run_metadata": {"customer": "acme", "tier": "enterprise"},
+ }
+ ]