Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix browser session proxy dropdown behavior and complete create-session config support (#4855)
Suchintanb2653ebd8036
diff --git a/fern/credentials/totp.mdx b/fern/credentials/totp.mdxindex b102e35dd..322737699 100644--- a/fern/credentials/totp.mdx+++ b/fern/credentials/totp.mdx@@ -178,13 +178,14 @@ Here's the TOTP endpoint contract you should use:Request (POST):| Parameter | Type | Required? | Sample Value | Description || --- | --- | --- | --- | --- |-| task_id | String | yes | tsk_123 | The task ID that needs the verification to be done |+| task_id | String | no | tsk_123 | The task ID that needs the verification to be done |+| workflow_run_id | String | no | wr_123456 | The workflow run ID that needs the verification to be done |+| workflow_permanent_id | String | no | wpid_123456 | The permanent workflow ID that needs the verification to be done |Response:| Parameter | Type | Required? | Sample Value | Description || --- | --- | --- | --- | --- |-| task_id | String | yes | tsk_123 | The task ID that needs the verification to be done |-| verification_code | String | no | 123456 | The verification code |+| verification_code | String | yes | 123456 | The verification code |### Validate The Sender of The RequestSame as the webhook API, your server needs to make sure it’s Skyvern that’s making the request.diff --git a/skyvern-frontend/src/components/GeoTargetSelector.tsx b/skyvern-frontend/src/components/GeoTargetSelector.tsxindex 9dc695bab..15454cb2c 100644--- a/skyvern-frontend/src/components/GeoTargetSelector.tsx+++ b/skyvern-frontend/src/components/GeoTargetSelector.tsx@@ -28,12 +28,16 @@ interface GeoTargetSelectorProps {value: GeoTarget | null;onChange: (value: GeoTarget) => void;className?: string;+ allowGranularSearch?: boolean;+ modalPopover?: boolean;}export function GeoTargetSelector({value,onChange,className,+ allowGranularSearch = true,+ modalPopover = false,}: GeoTargetSelectorProps) {const [open, setOpen] = React.useState(false);const [query, setQuery] = React.useState("");@@ -47,7 +51,9 @@ export function GeoTargetSelector({const handleSearch = useDebouncedCallback(async (searchQuery: string) => {setLoading(true);try {- const data = await searchGeoData(searchQuery);+ const data = await searchGeoData(searchQuery, {+ includeGranularResults: allowGranularSearch,+ });setResults(data);} catch (error) {console.error("Failed to search geo data", error);@@ -84,7 +90,7 @@ export function GeoTargetSelector({};return (- <Popover open={open} onOpenChange={setOpen}>+ <Popover open={open} onOpenChange={setOpen} modal={modalPopover}><PopoverTrigger asChild><Buttonvariant="outline"@@ -98,10 +104,14 @@ export function GeoTargetSelector({<CaretSortIcon className="ml-2 h-4 w-4 shrink-0 opacity-50" /></Button></PopoverTrigger>- <PopoverContent className="w-[400px] p-0" align="start">+ <PopoverContent className="z-[100] w-[400px] p-0" align="start"><Command shouldFilter={false}><CommandInput- placeholder="Search country, state, or city..."+ placeholder={+ allowGranularSearch+ ? "Search country, state, or city..."+ : "Search country..."+ }value={query}onValueChange={onInput}/>@@ -143,7 +153,7 @@ export function GeoTargetSelector({</CommandGroup>)}- {results.subdivisions.length > 0 && (+ {allowGranularSearch && results.subdivisions.length > 0 && (<CommandGroup heading="States / Regions">{results.subdivisions.map((item) => (<CommandItem@@ -171,7 +181,7 @@ export function GeoTargetSelector({</CommandGroup>)}- {results.cities.length > 0 && (+ {allowGranularSearch && results.cities.length > 0 && (<CommandGroup heading="Cities">{results.cities.map((item) => (<CommandItemdiff --git a/skyvern-frontend/src/components/ProxySelector.tsx b/skyvern-frontend/src/components/ProxySelector.tsxindex 38b762d08..8d6d818d8 100644--- a/skyvern-frontend/src/components/ProxySelector.tsx+++ b/skyvern-frontend/src/components/ProxySelector.tsx@@ -9,9 +9,17 @@ type Props = {value: ProxyLocation;onChange: (value: ProxyLocation) => void;className?: string;+ allowGranularSearch?: boolean;+ modalPopover?: boolean;};-function ProxySelector({ value, onChange, className }: Props) {+function ProxySelector({+ value,+ onChange,+ className,+ allowGranularSearch = true,+ modalPopover = false,+}: Props) {// Convert input (string enum or object) to GeoTarget for the selectorconst geoTargetValue = proxyLocationToGeoTarget(value);@@ -19,6 +27,8 @@ function ProxySelector({ value, onChange, className }: Props) {<GeoTargetSelectorclassName={className}value={geoTargetValue}+ allowGranularSearch={allowGranularSearch}+ modalPopover={modalPopover}onChange={(newTarget) => {// Convert back to ProxyLocation enum if possible (for simple countries)// or keep as GeoTarget objectdiff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsxindex 64cc27d43..e8edbcf01 100644--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsx+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsx@@ -6,6 +6,7 @@ import { ProxyLocation } from "@/api/types";import { Badge } from "@/components/ui/badge";import { Button } from "@/components/ui/button";+import { Checkbox } from "@/components/ui/checkbox";import {Drawer,DrawerContent,@@ -25,6 +26,13 @@ import {PaginationPrevious,} from "@/components/ui/pagination";import { ProxySelector } from "@/components/ProxySelector";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";import {Table,TableBody,@@ -35,7 +43,11 @@ import {} from "@/components/ui/table";import { useBrowserSessionsQuery } from "@/routes/browserSessions/hooks/useBrowserSessionsQuery";import { useCreateBrowserSessionMutation } from "@/routes/browserSessions/hooks/useCreateBrowserSessionMutation";-import { type BrowserSession } from "@/routes/workflows/types/browserSessionTypes";+import {+ type BrowserSession,+ type BrowserSessionExtension,+ type BrowserSessionType,+} from "@/routes/workflows/types/browserSessionTypes";import { CopyText } from "@/routes/workflows/editor/Workspace";import { basicTimeFormat } from "@/util/timeFormat";import { cn, formatMs, toDate } from "@/util/utils";@@ -58,6 +70,31 @@ const Yes = () => (</Badge>);+const BROWSER_TYPE_OPTIONS: Array<{+ value: BrowserSessionType;+ label: string;+}> = [+ { value: "msedge", label: "Microsoft Edge" },+ { value: "chrome", label: "Google Chrome" },+];++const EXTENSION_OPTIONS: Array<{+ value: BrowserSessionExtension;+ label: string;+ description: string;+}> = [+ {+ value: "ad-blocker",+ label: "Ad Blocker",+ description: "Blocks ads and common trackers in session pages.",+ },+ {+ value: "captcha-solver",+ label: "Captcha Solver",+ description: "Enables automated captcha solving when available.",+ },+];+function BrowserSessions() {const navigate = useNavigate();const [searchParams, setSearchParams] = useSearchParams();@@ -65,9 +102,13 @@ function BrowserSessions() {const [sessionOptions, setSessionOptions] = useState<{proxyLocation: ProxyLocation;timeoutMinutes: number | null;+ browserType: BrowserSessionType | null;+ extensions: BrowserSessionExtension[];}>({proxyLocation: ProxyLocation.Residential,timeoutMinutes: 60,+ browserType: null,+ extensions: [],});const page = searchParams.get("page") ? Number(searchParams.get("page")) : 1;@@ -123,6 +164,18 @@ function BrowserSessions() {}}+ function toggleExtension(extension: BrowserSessionExtension) {+ setSessionOptions((prev) => {+ const exists = prev.extensions.includes(extension);+ return {+ ...prev,+ extensions: exists+ ? prev.extensions.filter((item) => item !== extension)+ : [...prev.extensions, extension],+ };+ });+ }+return (<div className="px-8">{/* header */}@@ -336,11 +389,13 @@ function BrowserSessions() {</div><ProxySelectorvalue={sessionOptions.proxyLocation}+ allowGranularSearch={false}+ modalPopoveronChange={(value) => {- setSessionOptions({- ...sessionOptions,+ setSessionOptions((prev) => ({+ ...prev,proxyLocation: value,- });+ }));}}/></div>@@ -367,6 +422,76 @@ function BrowserSessions() {}}/></div>+ <div className="space-y-2">+ <div className="flex items-center gap-2">+ <Label>Browser Type</Label>+ <HelpTooltip content="Choose the browser engine for this session. Leave default to use server defaults." />+ </div>+ <Select+ value={sessionOptions.browserType ?? "default"}+ onValueChange={(value) => {+ setSessionOptions((prev) => ({+ ...prev,+ browserType:+ value === "default"+ ? null+ : (value as BrowserSessionType),+ }));+ }}+ >+ <SelectTrigger>+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="default">+ Default (Microsoft Edge)+ </SelectItem>+ {BROWSER_TYPE_OPTIONS.map((browserType) => (+ <SelectItem+ key={browserType.value}+ value={browserType.value}+ >+ {browserType.label}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>+ <div className="space-y-2">+ <div className="flex items-center gap-2">+ <Label>Extensions</Label>+ <HelpTooltip content="Optional browser extensions to install when the session starts." />+ </div>+ <div className="space-y-2 rounded-md border p-3">+ {EXTENSION_OPTIONS.map((extension) => (+ <div+ key={extension.value}+ className="flex items-start space-x-2"+ >+ <Checkbox+ id={`extension-${extension.value}`}+ checked={sessionOptions.extensions.includes(+ extension.value,+ )}+ onCheckedChange={() => {+ toggleExtension(extension.value);+ }}+ />+ <div className="grid gap-1">+ <Label+ htmlFor={`extension-${extension.value}`}+ className="font-medium"+ >+ {extension.label}+ </Label>+ <p className="text-xs text-muted-foreground">+ {extension.description}+ </p>+ </div>+ </div>+ ))}+ </div>+ </div><Buttondisabled={createBrowserSessionMutation.isPending ||@@ -380,6 +505,8 @@ function BrowserSessions() {createBrowserSessionMutation.mutate({proxyLocation: sessionOptions.proxyLocation,timeout: sessionOptions.timeoutMinutes,+ browserType: sessionOptions.browserType,+ extensions: sessionOptions.extensions,});}}>diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.tsindex 05504c6c9..6fe5f4a22 100644--- a/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.ts+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.ts@@ -5,7 +5,11 @@ import { useMutation } from "@tanstack/react-query";import { getClient } from "@/api/AxiosClient";import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";-import { BrowserSession } from "@/routes/workflows/types/browserSessionTypes";+import {+ BrowserSession,+ BrowserSessionExtension,+ BrowserSessionType,+} from "@/routes/workflows/types/browserSessionTypes";import { ProxyLocation } from "@/api/types";function useCreateBrowserSessionMutation() {@@ -17,9 +21,13 @@ function useCreateBrowserSessionMutation() {mutationFn: async ({proxyLocation = null,timeout = null,+ extensions = [],+ browserType = null,}: {proxyLocation: ProxyLocation | null;timeout: number | null;+ extensions?: BrowserSessionExtension[];+ browserType?: BrowserSessionType | null;}) => {const client = await getClient(credentialGetter, "sans-api-v1");return client.post<string, { data: BrowserSession }>(@@ -27,6 +35,8 @@ function useCreateBrowserSessionMutation() {{proxy_location: proxyLocation,timeout,+ extensions,+ browser_type: browserType,},);},diff --git a/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.ts b/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.tsindex 579212ecc..6bb487dc1 100644--- a/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.ts@@ -1,3 +1,6 @@+type BrowserSessionExtension = "ad-blocker" | "captcha-solver";+type BrowserSessionType = "msedge" | "chrome";+interface BrowserSession {browser_address: string | null;browser_session_id: string;@@ -8,6 +11,8 @@ interface BrowserSession {started_at: string | null;status: string;timeout: number | null;+ extensions?: BrowserSessionExtension[] | null;+ browser_type?: BrowserSessionType | null;vnc_streaming_supported: boolean;}@@ -18,4 +23,9 @@ interface Recording {modified_at: string;}-export { type BrowserSession, type Recording };+export {+ type BrowserSession,+ type BrowserSessionExtension,+ type BrowserSessionType,+ type Recording,+};diff --git a/skyvern-frontend/src/util/geoSearch.ts b/skyvern-frontend/src/util/geoSearch.tsindex f54b9e68b..ff3c648e4 100644--- a/skyvern-frontend/src/util/geoSearch.ts+++ b/skyvern-frontend/src/util/geoSearch.ts@@ -20,6 +20,10 @@ export type GroupedSearchResults = {cities: SearchResultItem[];};+type SearchGeoDataOptions = {+ includeGranularResults?: boolean;+};+// Store the promise so concurrent calls share one importlet cscModulePromise: Promise<typeof import("country-state-city")> | null =null;@@ -37,6 +41,7 @@ function loadCsc() {export async function searchGeoData(query: string,+ { includeGranularResults = true }: SearchGeoDataOptions = {},): Promise<GroupedSearchResults> {const normalizedQuery = query.trim().toLowerCase();const queryMatchesISP = normalizedQuery.includes("isp");@@ -82,8 +87,9 @@ export async function searchGeoData(}}- // If query is very short, just return countries to save perf- if (normalizedQuery.length < 2) {+ // Browser Sessions create currently supports country-level proxy locations only.+ // Allow callers to disable subdivisions/cities so the dropdown matches API support.+ if (!includeGranularResults || normalizedQuery.length < 2) {return results;}