Skip to content

Automate browser based workflows with AI

Ask for helpful/not-helpful feedback after Record Browser generates steps (SKY-16738) (#8825)

Cindy Li6c1ec08e554d
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index 9624e15cd..392e26d6f 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -1254,7 +1254,10 @@ export type PylonEmailHash = {
hash: string;
};
-export type RunFeedbackTargetType = "workflow_run" | "task";
+export type RunFeedbackTargetType =
+ | "workflow_run"
+ | "task"
+ | "browser_recording";
export type FeedbackRating = "up" | "down";
export type RunFeedbackApiResponse = {
diff --git a/skyvern-frontend/src/components/feedback/FeedbackThumbs.tsx b/skyvern-frontend/src/components/feedback/FeedbackThumbs.tsx
index a6119d00a..0bae99f5b 100644
--- a/skyvern-frontend/src/components/feedback/FeedbackThumbs.tsx
+++ b/skyvern-frontend/src/components/feedback/FeedbackThumbs.tsx
@@ -1,13 +1,17 @@
-import { useEffect, useState } from "react";
+import { useEffect, useId, useState } from "react";
import { Checkbox } from "@/components/ui/checkbox";
+import { Label } from "@/components/ui/label";
+import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
import { Textarea } from "@/components/ui/textarea";
import { cn } from "@/util/utils";
export type FeedbackRating = "up" | "down";
export interface FeedbackRateOptions {
- needsSupport: boolean;
+ needsSupport?: boolean;
+ // Typed detail for the "other" reason option.
+ detail?: string;
}
interface FeedbackThumbsProps {
@@ -26,6 +30,9 @@ interface FeedbackThumbsProps {
subtle?: boolean;
// When set, the reason panel also offers a checkbox that flags the rating for human follow-up.
supportOption?: { label: string };
+ // When set, a thumbs down offers these optional choices instead of free text; the chosen value is the reason.
+ // Choosing "other" opens a text box whose content arrives as `options.detail`.
+ reasonOptions?: Array<{ value: string; label: string }>;
// "report" replaces the thumbs pair with a single thumbs-down action for runs that already failed.
variant?: "thumbs" | "report";
reportLabel?: string;
@@ -60,6 +67,7 @@ export function FeedbackThumbs({
prompt,
subtle = false,
supportOption,
+ reasonOptions,
variant = "thumbs",
reportLabel = "Report this failure",
className,
@@ -70,6 +78,8 @@ export function FeedbackThumbs({
const [reason, setReason] = useState(savedReason ?? "");
const [flash, setFlash] = useState<string | null>(null);
const [needsSupport, setNeedsSupport] = useState(false);
+ const [otherDetail, setOtherDetail] = useState("");
+ const reasonIdPrefix = useId();
useEffect(() => {
if (rating !== "down") {
@@ -91,6 +101,11 @@ export function FeedbackThumbs({
try {
if (supportOption && text !== undefined) {
await onRate(next, text, { needsSupport });
+ } else if (reasonOptions && text !== undefined) {
+ await onRate(next, text, {
+ detail:
+ text === "other" ? otherDetail.trim() || undefined : undefined,
+ });
} else {
await onRate(next, text);
}
@@ -218,14 +233,48 @@ export function FeedbackThumbs({
</div>
{reasonOpen && rating === "down" ? (
<div className="flex flex-col gap-2">
- <Textarea
- value={reason}
- onChange={(event) => setReason(event.target.value)}
- placeholder="What was missing or wrong? (optional)"
- aria-label="Feedback reason"
- maxLength={2000}
- className="min-h-[56px] bg-slate-elevation3 text-xs"
- />
+ {reasonOptions ? (
+ <RadioGroup
+ value={reason}
+ onValueChange={setReason}
+ aria-label="What went wrong? (optional)"
+ className="gap-1.5"
+ >
+ {reasonOptions.map((option) => (
+ <div key={option.value} className="flex items-center gap-2">
+ <RadioGroupItem
+ value={option.value}
+ id={`${reasonIdPrefix}-${option.value}`}
+ />
+ <Label
+ htmlFor={`${reasonIdPrefix}-${option.value}`}
+ className="text-xs font-normal"
+ >
+ {option.label}
+ </Label>
+ </div>
+ ))}
+ </RadioGroup>
+ ) : (
+ <Textarea
+ value={reason}
+ onChange={(event) => setReason(event.target.value)}
+ placeholder="What was missing or wrong? (optional)"
+ aria-label="Feedback reason"
+ maxLength={2000}
+ className="min-h-[56px] bg-slate-elevation3 text-xs"
+ />
+ )}
+ {reasonOptions && reason === "other" ? (
+ <Textarea
+ value={otherDetail}
+ onChange={(event) => setOtherDetail(event.target.value)}
+ placeholder="What went wrong? (optional)"
+ aria-label="Describe what went wrong"
+ maxLength={500}
+ className="min-h-[56px] bg-slate-elevation3 text-xs"
+ />
+ ) : null}
{supportOption ? (
<label className="flex items-center gap-2 text-xs text-foreground">
<Checkbox
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.test.tsx b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.test.tsx
index 4df958fd4..fd4ed14e6 100644
--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.test.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.test.tsx
@@ -8,6 +8,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { MemoryRouter, useLocation } from "react-router-dom";
import { WorkflowPermanentIdContext } from "@/routes/workflows/WorkflowPermanentIdContext";
+import { useRecordingFeedbackStore } from "@/store/RecordingFeedbackStore";
import { useRecordingRefinementEvidenceStore } from "@/store/RecordingRefinementEvidenceStore";
import {
useRecordingStore,
@@ -53,6 +54,11 @@ vi.mock("@/hooks/useCredentialGetter", () => ({
vi.mock("@/util/recordBrowserTelemetry", () => ({
captureRecordBrowser: mocks.captureRecordBrowser,
+ getRecordBrowserContext: () => ({
+ recording_attempt_id: "rra-1",
+ workflow_permanent_id: "wpid-1",
+ browser_session_id: "pbs-1",
+ }),
markRecordBrowserProcessed: mocks.markRecordBrowserProcessed,
setRecordBrowserContext: vi.fn(),
}));
@@ -265,6 +271,13 @@ describe("useProcessRecordingMutation telemetry", () => {
"record_browser.processed",
expect.objectContaining({ recording_id: "br-1" }),
);
+ expect(useRecordingFeedbackStore.getState().target).toMatchObject({
+ recording_attempt_id: "rra-1",
+ recording_id: "br-1",
+ workflow_permanent_id: "wpid-1",
+ browser_session_id: "pbs-1",
+ rating: null,
+ });
});
it("lands recorded blocks when an older backend omits the recording id", async () => {
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
index 3fc07a6fe..b0a436ad1 100644
--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
@@ -19,8 +19,10 @@ import {
useWorkflowYamlEditorStore,
type YamlCommitOwner,
} from "@/store/WorkflowYamlEditorStore";
+import { useRecordingFeedbackStore } from "@/store/RecordingFeedbackStore";
import {
captureRecordBrowser,
+ getRecordBrowserContext,
markRecordBrowserProcessed,
} from "@/util/recordBrowserTelemetry";
@@ -218,6 +220,11 @@ const useProcessRecordingMutation = ({
if (blocks && blocks.length > 0) {
if (!isRecordingOwnerCurrent(owner)) return;
+ useRecordingFeedbackStore.getState().show({
+ ...getRecordBrowserContext(),
+ workflow_permanent_id: owner.workflowPermanentId,
+ recording_id: recordingId ?? undefined,
+ });
if (recordingId) {
useWorkflowHasChangesStore
.getState()
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.runGrounding.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.runGrounding.test.tsx
index 824ecf882..355f05490 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.runGrounding.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.runGrounding.test.tsx
@@ -14,6 +14,7 @@ import type { ComponentProps } from "react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { FeatureFlagContext } from "@/hooks/useFeatureFlag";
+import { useRecordingFeedbackStore } from "@/store/RecordingFeedbackStore";
import { useRecordingRefinementEvidenceStore } from "@/store/RecordingRefinementEvidenceStore";
import {
beginYamlCommit,
@@ -1300,6 +1301,15 @@ describe("WorkflowCopilotChat — run grounding bridge", () => {
),
).toBeNull();
expect(screen.getByTestId("recording-refinement-progress")).toBeTruthy();
+ // A refinement rebuilt from history is not the live recording, so it asks for no rating.
+ expect(screen.queryByTestId("recording-feedback-prompt")).toBeNull();
+ act(() =>
+ useRecordingFeedbackStore
+ .getState()
+ .show({ workflow_permanent_id: "wpid_1" }),
+ );
+ expect(screen.getByTestId("recording-feedback-prompt")).toBeTruthy();
+ act(() => useRecordingFeedbackStore.getState().dismiss());
});
it("hydrates an agent-loop cancellation as cancelled", async () => {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index cad6c05b1..6c684c3f9 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -255,6 +255,7 @@ import { useStudioPanes } from "@/routes/workflows/studio/useStudioPanes";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useRecordingRefinementEvidenceStore } from "@/store/RecordingRefinementEvidenceStore";
import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";
+import { RecordingFeedbackPrompt } from "@/routes/workflows/editor/recording/RecordingFeedbackPrompt";
import { useWorkflowBlockSearchStore } from "@/store/WorkflowBlockSearchStore";
import { resolveTimelineBlockJumpNodeId } from "@/routes/workflows/studio/runview/timelineBlockJump";
import { TooltipProvider } from "@/components/ui/tooltip";
@@ -7204,6 +7205,9 @@ export function WorkflowCopilotChat({
return () => window.clearTimeout(timer);
}, [nextPauseExpiry]);
const lastTurnIndex = findLastTurnIndex(messages);
+ const lastRefinementIndex = messages
+ .map((message) => message.kind)
+ .lastIndexOf("recording_refinement");
// Only the tail turn's ask docks; a stranded one further up stays actionable inline.
const tailTerminalAsk =
trayPauseFrame || isLoadingHistory || lastTurnIndex < 0
@@ -10443,15 +10447,22 @@ export function WorkflowCopilotChat({
message.recordingRefinement
) {
return (
- <RecordingRefinementProgressCard
- key={message.id}
- awaitingReview={Boolean(
- proposedWorkflow &&
- pendingProposalTurnId ===
- message.recordingRefinement.turnId,
- )}
- {...message.recordingRefinement}
- />
+ <div key={message.id} className="flex flex-col gap-3">
+ <RecordingRefinementProgressCard
+ awaitingReview={Boolean(
+ proposedWorkflow &&
+ pendingProposalTurnId ===
+ message.recordingRefinement.turnId,
+ )}
+ {...message.recordingRefinement}
+ />
+ {index === lastRefinementIndex &&
+ message.recordingRefinement.status !== "working" ? (
+ <RecordingFeedbackPrompt
+ workflowPermanentId={workflowPermanentId}
+ />
+ ) : null}
+ </div>
);
}
if (
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingFeedbackPrompt.test.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingFeedbackPrompt.test.tsx
new file mode 100644
index 000000000..191eb0701
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingFeedbackPrompt.test.tsx
@@ -0,0 +1,220 @@
+// @vitest-environment jsdom
+
+import {
+ act,
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+} from "@testing-library/react";
+import posthog from "posthog-js";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { useRecordingFeedbackStore } from "@/store/RecordingFeedbackStore";
+import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";
+import { setRecordBrowserContext } from "@/util/recordBrowserTelemetry";
+
+import { RecordingFeedbackPrompt } from "./RecordingFeedbackPrompt";
+
+const post = vi.hoisted(() => vi.fn());
+
+vi.mock("posthog-js", () => ({ default: { capture: vi.fn() } }));
+vi.mock("@/api/AxiosClient", () => ({ getClient: async () => ({ post }) }));
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => null,
+}));
+vi.mock("@/hooks/useUser", () => ({
+ useUser: () => ({ get: () => ({ email: "[email protected]" }) }),
+}));
+
+const ALLOWED_KEYS = [
+ "recording_attempt_id",
+ "recording_id",
+ "workflow_permanent_id",
+ "browser_session_id",
+ "rating",
+ "reason_code",
+ "has_reason",
+ "updated",
+];
+
+async function click(name: string, role = "button") {
+ await act(async () => fireEvent.click(screen.getByRole(role, { name })));
+}
+
+const feedbackEvents = () =>
+ vi
+ .mocked(posthog.capture)
+ .mock.calls.filter(
+ ([event]) => event === "record_browser.feedback_submitted",
+ )
+ .map(([, properties]) => properties);
+
+function renderPrompt() {
+ render(<RecordingFeedbackPrompt workflowPermanentId="wpid-1" />);
+}
+
+describe("RecordingFeedbackPrompt", () => {
+ afterEach(() => {
+ cleanup();
+ setRecordBrowserContext({});
+ useRecordingFeedbackStore.setState({ target: null });
+ vi.mocked(posthog.capture).mockClear();
+ post.mockReset();
+ });
+
+ it("stores typed reasons in the feedback table, sends analytics only categories, and flags changes as updates", async () => {
+ useRecordingFeedbackStore.getState().show({
+ recording_attempt_id: "rra-1",
+ recording_id: "br-1",
+ workflow_permanent_id: "wpid-1",
+ browser_session_id: "pbs-1",
+ });
+ // A newer attempt's live context must not relabel feedback for the processed one.
+ setRecordBrowserContext({
+ recording_attempt_id: "rra-2",
+ workflow_permanent_id: "wpid-1",
+ browser_session_id: "pbs-2",
+ });
+ renderPrompt();
+
+ await click("Thumbs up");
+ await click("Thumbs down");
+ await click("Other", "radio");
+ fireEvent.change(
+ screen.getByRole("textbox", { name: "Describe what went wrong" }),
+ { target: { value: " Skipped the export step " } },
+ );
+ await click("Send");
+
+ const ids = {
+ recording_attempt_id: "rra-1",
+ recording_id: "br-1",
+ workflow_permanent_id: "wpid-1",
+ browser_session_id: "pbs-1",
+ };
+ expect(feedbackEvents()).toEqual([
+ {
+ ...ids,
+ rating: "up",
+ reason_code: undefined,
+ has_reason: false,
+ updated: false,
+ },
+ {
+ ...ids,
+ rating: "down",
+ reason_code: undefined,
+ has_reason: false,
+ updated: true,
+ },
+ {
+ ...ids,
+ rating: "down",
+ reason_code: "other",
+ has_reason: true,
+ updated: true,
+ },
+ ]);
+ expect(JSON.stringify(feedbackEvents())).not.toContain("export step");
+ expect(post).toHaveBeenLastCalledWith("/feedback", {
+ target_type: "browser_recording",
+ target_id: "br-1",
+ rating: "down",
+ reason: "Skipped the export step",
+ submitted_by: "[email protected]",
+ });
+ for (const properties of feedbackEvents()) {
+ expect(
+ Object.keys(properties ?? {}).every((key) =>
+ ALLOWED_KEYS.includes(key),
+ ),
+ ).toBe(true);
+ }
+ });
+
+ it("accepts a reasonless thumbs down without identifiers, and retires on dismiss but not on a Copilot persist", async () => {
+ useRecordingFeedbackStore
+ .getState()
+ .show({ workflow_permanent_id: "wpid-1" });
+ renderPrompt();
+
+ await click("Thumbs down");
+ await click("Skip");
+ expect(post).not.toHaveBeenCalled();
+ expect(feedbackEvents()).toEqual([
+ expect.objectContaining({
+ recording_attempt_id: undefined,
+ recording_id: undefined,
+ rating: "down",
+ reason_code: undefined,
+ updated: false,
+ }),
+ ]);
+
+ await click("Dismiss recording feedback");
+ expect(screen.queryByTestId("recording-feedback-prompt")).toBeNull();
+
+ act(() =>
+ useRecordingFeedbackStore
+ .getState()
+ .show({ workflow_permanent_id: "wpid-1" }),
+ );
+ expect(screen.getByTestId("recording-feedback-prompt")).toBeTruthy();
+ // A Copilot persist (retitle, auto-accept) bumps the save count but is not the user's save.
+ act(() =>
+ useWorkflowHasChangesStore.getState().recordPersistedSave("wpid-1"),
+ );
+ expect(screen.getByTestId("recording-feedback-prompt")).toBeTruthy();
+ expect(feedbackEvents()).toHaveLength(1);
+ });
+
+ it("does not count a response whose save failed", async () => {
+ post.mockRejectedValue(new Error("offline"));
+ useRecordingFeedbackStore
+ .getState()
+ .show({ recording_id: "br-1", workflow_permanent_id: "wpid-1" });
+ renderPrompt();
+
+ await click("Thumbs up");
+
+ expect(screen.getByText("Couldn't save. Try again.")).toBeTruthy();
+ expect(feedbackEvents()).toEqual([]);
+ });
+
+ it("credits a slow save to the recording that was rated, not a newer one", async () => {
+ let finishSave!: () => void;
+ post.mockReturnValue(
+ new Promise<void>((resolve) => (finishSave = resolve)),
+ );
+ useRecordingFeedbackStore.getState().show({
+ recording_attempt_id: "rra-1",
+ recording_id: "br-1",
+ workflow_permanent_id: "wpid-1",
+ });
+ renderPrompt();
+
+ await click("Thumbs up");
+ act(() =>
+ useRecordingFeedbackStore.getState().show({
+ recording_attempt_id: "rra-2",
+ recording_id: "br-2",
+ workflow_permanent_id: "wpid-1",
+ }),
+ );
+ await act(async () => finishSave());
+
+ expect(feedbackEvents()).toEqual([
+ expect.objectContaining({
+ recording_attempt_id: "rra-1",
+ recording_id: "br-1",
+ rating: "up",
+ }),
+ ]);
+ expect(useRecordingFeedbackStore.getState().target).toMatchObject({
+ recording_attempt_id: "rra-2",
+ rating: null,
+ responded: false,
+ });
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingFeedbackPrompt.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingFeedbackPrompt.tsx
new file mode 100644
index 000000000..4ec7aeabb
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingFeedbackPrompt.tsx
@@ -0,0 +1,79 @@
+import { Cross2Icon } from "@radix-ui/react-icons";
+
+import { getClient } from "@/api/AxiosClient";
+import {
+ FeedbackThumbs,
+ type FeedbackRateOptions,
+ type FeedbackRating,
+} from "@/components/feedback/FeedbackThumbs";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { useUser } from "@/hooks/useUser";
+import { useRecordingFeedbackStore } from "@/store/RecordingFeedbackStore";
+
+const REASON_OPTIONS = [
+ { value: "missing_steps", label: "Missing steps" },
+ { value: "extra_or_wrong_steps", label: "Extra or wrong steps" },
+ { value: "wrong_values", label: "Wrong values or parameters" },
+ { value: "credential_handling", label: "Credential handling" },
+ { value: "navigation", label: "Navigation" },
+ { value: "workflow_failed", label: "Generated workflow failed" },
+ { value: "other", label: "Other" },
+];
+
+export function RecordingFeedbackPrompt({
+ workflowPermanentId,
+}: {
+ workflowPermanentId?: string;
+}) {
+ const target = useRecordingFeedbackStore((state) => state.target);
+ const { rate, dismiss } = useRecordingFeedbackStore.getState();
+ const credentialGetter = useCredentialGetter();
+ const { get: getUser } = useUser();
+
+ if (!target || target.workflow_permanent_id !== workflowPermanentId) {
+ return null;
+ }
+
+ return (
+ <div
+ role="group"
+ aria-label="Recording feedback"
+ className="flex items-start justify-between gap-2 rounded-lg border border-border bg-slate-elevation2 px-3 py-2"
+ data-testid="recording-feedback-prompt"
+ >
+ <FeedbackThumbs
+ key={target.recording_attempt_id ?? target.recording_id}
+ rating={target.rating}
+ onRate={async (
+ rating: FeedbackRating | null,
+ reasonCode?: string,
+ options?: FeedbackRateOptions,
+ ) => {
+ // Without a durable recording id there is no row to attach the reason to; analytics still records the rating.
+ if (target.recording_id) {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ await client.post("/feedback", {
+ target_type: "browser_recording",
+ target_id: target.recording_id,
+ rating,
+ reason: options?.detail || reasonCode || null,
+ submitted_by: getUser()?.email ?? null,
+ });
+ }
+ rate(target, rating, reasonCode, options);
+ }}
+ prompt="Were these recorded steps helpful?"
+ reasonOptions={REASON_OPTIONS}
+ className="min-w-0 flex-1"
+ />
+ <button
+ type="button"
+ aria-label="Dismiss recording feedback"
+ onClick={dismiss}
+ className="ml-1 inline-flex h-6 w-6 items-center justify-center rounded-md text-muted-foreground hover:bg-accent hover:text-foreground"
+ >
+ <Cross2Icon className="h-3.5 w-3.5" />
+ </button>
+ </div>
+ );
+}
diff --git a/skyvern-frontend/src/store/RecordingFeedbackStore.ts b/skyvern-frontend/src/store/RecordingFeedbackStore.ts
new file mode 100644
index 000000000..039b8b5bb
--- /dev/null
+++ b/skyvern-frontend/src/store/RecordingFeedbackStore.ts
@@ -0,0 +1,68 @@
+import { create } from "zustand";
+
+import type {
+ FeedbackRateOptions,
+ FeedbackRating,
+} from "@/components/feedback/FeedbackThumbs";
+import { captureRecordBrowser } from "@/util/recordBrowserTelemetry";
+
+export type RecordingFeedbackTarget = {
+ recording_attempt_id?: string;
+ recording_id?: string;
+ workflow_permanent_id: string;
+ browser_session_id?: string;
+ rating: FeedbackRating | null;
+ responded: boolean;
+};
+
+type RecordingFeedbackStore = {
+ target: RecordingFeedbackTarget | null;
+ show: (ids: Omit<RecordingFeedbackTarget, "rating" | "responded">) => void;
+ // Takes the target the user rated: the save is async and the prompt may change before it lands.
+ rate: (
+ target: RecordingFeedbackTarget,
+ rating: FeedbackRating | null,
+ reasonCode?: string,
+ options?: FeedbackRateOptions,
+ ) => void;
+ dismiss: () => void;
+ // The editor's own save ends the prompt; Copilot persists (retitles, auto-accepts) must not.
+ retire: (workflowPermanentId: string) => void;
+};
+
+export const useRecordingFeedbackStore = create<RecordingFeedbackStore>(
+ (set, get) => ({
+ target: null,
+ show: (ids) =>
+ set({
+ target: {
+ ...ids,
+ rating: null,
+ responded: false,
+ },
+ }),
+ rate: (target, rating, reasonCode, options) => {
+ // Every id is passed explicitly so a newer attempt's live context can't leak in.
+ captureRecordBrowser("record_browser.feedback_submitted", {
+ recording_attempt_id: target.recording_attempt_id,
+ recording_id: target.recording_id,
+ workflow_permanent_id: target.workflow_permanent_id,
+ browser_session_id: target.browser_session_id,
+ rating,
+ reason_code: rating === "down" ? reasonCode || undefined : undefined,
+ // Typed text stays in our feedback table; analytics only learns that a reason exists.
+ has_reason: rating === "down" && Boolean(reasonCode || options?.detail),
+ updated: target.responded,
+ });
+ if (get().target === target)
+ set({ target: { ...target, rating, responded: true } });
+ },
+ dismiss: () => set({ target: null }),
+ retire: (workflowPermanentId) =>
+ set((state) =>
+ state.target?.workflow_permanent_id === workflowPermanentId
+ ? { target: null }
+ : {},
+ ),
+ }),
+);
diff --git a/skyvern-frontend/src/store/WorkflowHasChangesStore.recording.test.tsx b/skyvern-frontend/src/store/WorkflowHasChangesStore.recording.test.tsx
index a0d80a4d5..853448ab4 100644
--- a/skyvern-frontend/src/store/WorkflowHasChangesStore.recording.test.tsx
+++ b/skyvern-frontend/src/store/WorkflowHasChangesStore.recording.test.tsx
@@ -37,6 +37,7 @@ import {
type WorkflowSaveData,
} from "./WorkflowHasChangesStore";
import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
+import { useRecordingFeedbackStore } from "./RecordingFeedbackStore";
import { useRecordingStore } from "./useRecordingStore";
import {
confirmCodeCacheDeletion,
@@ -86,6 +87,7 @@ vi.mock("@/routes/workflows/studio/useStudioPanes", () => ({
}));
vi.mock("@/util/recordBrowserTelemetry", () => ({
captureRecordBrowser: vi.fn(),
+ getRecordBrowserContext: () => ({}),
markRecordBrowserProcessed: vi.fn(),
setRecordBrowserContext: vi.fn(),
}));
@@ -602,6 +604,9 @@ describe("workflow recording attachment", () => {
);
it("writes the pending-save marker without a degraded-mode notice when storage works", async () => {
+ useRecordingFeedbackStore
+ .getState()
+ .show({ workflow_permanent_id: "wpid-1" });
let resolvePut!: (value: unknown) => void;
mocks.put.mockReset().mockImplementationOnce(
() =>
@@ -638,6 +643,7 @@ describe("workflow recording attachment", () => {
await saving;
});
expect(sessionStorage.getItem("workflow-pending-save:wpid-1")).toBeNull();
+ expect(useRecordingFeedbackStore.getState().target).toBeNull();
});
it.each([false, true])(
diff --git a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
index 518790a38..ea6a3b34f 100644
--- a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
+++ b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
@@ -1,4 +1,5 @@
import { useRecordedBlocksStore } from "./RecordedBlocksStore";
+import { useRecordingFeedbackStore } from "./RecordingFeedbackStore";
import { useRecordingStore } from "./useRecordingStore";
import { useWorkflowParametersStore } from "./WorkflowParametersStore";
import { getInitialParameters } from "@/routes/workflows/editor/utils";
@@ -483,6 +484,9 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
useWorkflowHasChangesStore
.getState()
.recordPersistedSave(saveData.workflow.workflow_permanent_id);
+ useRecordingFeedbackStore
+ .getState()
+ .retire(saveData.workflow.workflow_permanent_id);
const activeOwner = useWorkflowYamlEditorStore.getState().editorOwner;
if (
owner &&
diff --git a/skyvern/forge/sdk/routes/feedback.py b/skyvern/forge/sdk/routes/feedback.py
index 7b21e343b..0ed0b7775 100644
--- a/skyvern/forge/sdk/routes/feedback.py
+++ b/skyvern/forge/sdk/routes/feedback.py
@@ -21,6 +21,11 @@ async def _resolve_target_context(
if workflow_run is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Run not found")
return workflow_run.workflow_permanent_id
+ if target_type == "browser_recording":
+ recording = await app.DATABASE.browser_recordings.get_recording(target_id, organization_id)
+ if recording is None:
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Recording not found")
+ return recording.workflow_permanent_id
task = await app.DATABASE.tasks.get_task(target_id, organization_id=organization_id)
if task is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Run not found")
diff --git a/skyvern/forge/sdk/schemas/feedback.py b/skyvern/forge/sdk/schemas/feedback.py
index ebbe272e3..ba120dc32 100644
--- a/skyvern/forge/sdk/schemas/feedback.py
+++ b/skyvern/forge/sdk/schemas/feedback.py
@@ -3,8 +3,8 @@ from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
-FeedbackTargetType = Literal["workflow_run", "task", "copilot_message"]
-RunFeedbackTargetType = Literal["workflow_run", "task"]
+FeedbackTargetType = Literal["workflow_run", "task", "copilot_message", "browser_recording"]
+RunFeedbackTargetType = Literal["workflow_run", "task", "browser_recording"]
FeedbackRating = Literal["up", "down"]
diff --git a/tests/unit/test_feedback_route.py b/tests/unit/test_feedback_route.py
index 8df56a4aa..c0f32875e 100644
--- a/tests/unit/test_feedback_route.py
+++ b/tests/unit/test_feedback_route.py
@@ -38,6 +38,7 @@ def _install_fakes(
workflow_run: object | None,
hook: AsyncMock,
previous: RunFeedback | None = None,
+ recording: object | None = None,
) -> SimpleNamespace:
run_feedback = SimpleNamespace(
upsert_run_feedback=AsyncMock(return_value=_stored()),
@@ -47,6 +48,7 @@ def _install_fakes(
database = SimpleNamespace(
workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=workflow_run)),
tasks=SimpleNamespace(get_task=AsyncMock(return_value=None)),
+ browser_recordings=SimpleNamespace(get_recording=AsyncMock(return_value=recording)),
run_feedback=run_feedback,
)
monkeypatch.setattr(app, "DATABASE", database)
@@ -69,6 +71,27 @@ async def test_feedback_on_a_run_outside_the_org_is_404_and_stores_nothing(monke
hook.assert_not_awaited()
+async def test_recording_feedback_is_scoped_to_the_orgs_recording_and_stores_its_workflow(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ hook = AsyncMock()
+ request = RunFeedbackRequest(target_type="browser_recording", target_id="br_1", rating="down", reason="other")
+ run_feedback = _install_fakes(monkeypatch, workflow_run=None, hook=hook, recording=None)
+
+ with pytest.raises(HTTPException) as exc:
+ await feedback_routes.submit_run_feedback(request, BackgroundTasks(), ORG)
+ assert exc.value.status_code == 404
+ run_feedback.upsert_run_feedback.assert_not_awaited()
+
+ run_feedback = _install_fakes(
+ monkeypatch, workflow_run=None, hook=hook, recording=SimpleNamespace(workflow_permanent_id="wpid_1")
+ )
+ await feedback_routes.submit_run_feedback(request, BackgroundTasks(), ORG)
+ stored = run_feedback.upsert_run_feedback.await_args.kwargs
+ assert (stored["target_type"], stored["context_id"], stored["reason"]) == ("browser_recording", "wpid_1", "other")
+
+
@pytest.mark.asyncio
async def test_rating_is_stored_with_run_context_and_fan_out_runs_after_the_response_and_swallows_failures(
monkeypatch: pytest.MonkeyPatch,