Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Revert "Move Run Workflow button to top of page" (#4614)
Suchintan1fcd659c38b4
diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsxindex a9c8e1180..4bfebb274 100644--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx@@ -504,60 +504,6 @@ function RunWorkflowForm({onSubmit={form.handleSubmit(onSubmit, handleInvalid)}className="space-y-8">- <header className="flex items-end justify-between gap-4">- <div className="space-y-5">- <h1 className="text-3xl">- Parameters{workflow?.title ? ` - ${workflow.title}` : ""}- </h1>- <h2 className="text-lg text-slate-400">- Fill the placeholder values that you have linked throughout your- workflow.- </h2>- </div>- <div className="flex shrink-0 gap-2">- <CopyApiCommandDropdown- getOptions={() => {- const values = form.getValues();- const body = getRunWorkflowRequestBody(- values,- workflowParameters,- );- const transformedBody = transformToWorkflowRunRequest(- body,- workflowPermanentId,- );-- // Build headers - x-max-steps-override is optional and can be added manually if needed- const headers: Record<string, string> = {- "Content-Type": "application/json",- "x-api-key": apiCredential ?? "<your-api-key>",- };-- return {- method: "POST",- url: `${runsApiBaseUrl}/run/workflows`,- body: transformedBody,- headers,- } satisfies ApiCommandOptions;- }}- />- <Button- type="submit"- disabled={- runWorkflowMutation.isPending || hasLoginBlockValidationError- }- >- {runWorkflowMutation.isPending && (- <ReloadIcon className="mr-2 h-4 w-4 animate-spin" />- )}- {!runWorkflowMutation.isPending && (- <PlayIcon className="mr-2 h-4 w-4" />- )}- Run workflow- </Button>- </div>- </header>-{hasLoginBlockValidationError && (<Alert variant="destructive"><ExclamationTriangleIcon className="h-4 w-4" />@@ -1105,6 +1051,49 @@ function RunWorkflowForm({</AccordionItem></Accordion></div>++ <div className="flex justify-end gap-2">+ <CopyApiCommandDropdown+ getOptions={() => {+ const values = form.getValues();+ const body = getRunWorkflowRequestBody(+ values,+ workflowParameters,+ );+ const transformedBody = transformToWorkflowRunRequest(+ body,+ workflowPermanentId,+ );++ // Build headers - x-max-steps-override is optional and can be added manually if needed+ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ "x-api-key": apiCredential ?? "<your-api-key>",+ };++ return {+ method: "POST",+ url: `${runsApiBaseUrl}/run/workflows`,+ body: transformedBody,+ headers,+ } satisfies ApiCommandOptions;+ }}+ />+ <Button+ type="submit"+ disabled={+ runWorkflowMutation.isPending || hasLoginBlockValidationError+ }+ >+ {runWorkflowMutation.isPending && (+ <ReloadIcon className="mr-2 h-4 w-4 animate-spin" />+ )}+ {!runWorkflowMutation.isPending && (+ <PlayIcon className="mr-2 h-4 w-4" />+ )}+ Run workflow+ </Button>+ </div></form></Form>);diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsxindex 72660a33d..61dd25cfd 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx@@ -44,16 +44,22 @@ function WorkflowRunParameters() {const initialValues = getInitialValues(location, workflowParameters ?? []);+ const header = (+ <header className="space-y-5">+ <h1 className="text-3xl">+ Parameters{workflow?.title ? ` - ${workflow.title}` : ""}+ </h1>+ <h2 className="text-lg text-slate-400">+ Fill the placeholder values that you have linked throughout your+ workflow.+ </h2>+ </header>+ );+if (isFetching) {return (<div className="space-y-8">- <header className="space-y-5">- <h1 className="text-3xl">Parameters</h1>- <h2 className="text-lg text-slate-400">- Fill the placeholder values that you have linked throughout your- workflow.- </h2>- </header>+ {header}<Skeleton className="h-96 w-full" /></div>);@@ -64,21 +70,26 @@ function WorkflowRunParameters() {}return (- <RunWorkflowForm- initialValues={initialValues}- workflowParameters={workflowParameters}- initialSettings={{- proxyLocation:- proxyLocation ?? workflow.proxy_location ?? ProxyLocation.Residential,- webhookCallbackUrl:- webhookCallbackUrl ?? workflow.webhook_callback_url ?? "",- maxScreenshotScrolls:- maxScreenshotScrolls ?? workflow.max_screenshot_scrolls ?? null,- extraHttpHeaders:- extraHttpHeaders ?? workflow.extra_http_headers ?? null,- cdpAddress: null,- }}- />+ <div className="space-y-8">+ {header}+ <RunWorkflowForm+ initialValues={initialValues}+ workflowParameters={workflowParameters}+ initialSettings={{+ proxyLocation:+ proxyLocation ??+ workflow.proxy_location ??+ ProxyLocation.Residential,+ webhookCallbackUrl:+ webhookCallbackUrl ?? workflow.webhook_callback_url ?? "",+ maxScreenshotScrolls:+ maxScreenshotScrolls ?? workflow.max_screenshot_scrolls ?? null,+ extraHttpHeaders:+ extraHttpHeaders ?? workflow.extra_http_headers ?? null,+ cdpAddress: null,+ }}+ />+ </div>);}diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsxindex 99f6cc720..992f87e7f 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx@@ -188,7 +188,7 @@ const nodeLibraryItems: Array<{/>),title: "File Parser Block",- description: "Parse PDFs, CSVs, and Excel files",+ description: "Parse PDFs, CSVs, Excel files, and Images",},// {// nodeType: "pdfParser",diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.tsindex 5b615a948..0607c2356 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts@@ -415,7 +415,7 @@ export type SendEmailBlock = WorkflowBlockBase & {export type FileURLParserBlock = WorkflowBlockBase & {block_type: "file_url_parser";file_url: string;- file_type: "csv" | "excel" | "pdf";+ file_type: "csv" | "excel" | "pdf" | "image";json_schema: Record<string, unknown> | null;};diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.tsindex 81ced5740..fa2ca1739 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts@@ -350,7 +350,7 @@ export type SendEmailBlockYAML = BlockYAMLBase & {export type FileUrlParserBlockYAML = BlockYAMLBase & {block_type: "file_url_parser";file_url: string;- file_type: "csv" | "excel" | "pdf";+ file_type: "csv" | "excel" | "pdf" | "image";json_schema?: Record<string, unknown> | null;};diff --git a/skyvern/client/types/file_type.py b/skyvern/client/types/file_type.pyindex fade3f1db..02decc32a 100644--- a/skyvern/client/types/file_type.py+++ b/skyvern/client/types/file_type.py@@ -2,4 +2,4 @@import typing-FileType = typing.Union[typing.Literal["csv", "excel", "pdf"], typing.Any]+FileType = typing.Union[typing.Literal["csv", "excel", "pdf", "image"], typing.Any]diff --git a/skyvern/forge/prompts/skyvern/extract-text-from-image.j2 b/skyvern/forge/prompts/skyvern/extract-text-from-image.j2new file mode 100644index 000000000..a577d1e9a--- /dev/null+++ b/skyvern/forge/prompts/skyvern/extract-text-from-image.j2@@ -0,0 +1,19 @@+Extract all visible text from this image.++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments, no unnecessary quotes.++Reply in JSON format with the following keys:+{+ "extracted_text": str // All text extracted from the image+}++TEXT EXTRACTION GUIDELINES:+- Preserve reading order (top to bottom, left to right)+- For tables: format as rows separated by newlines, columns separated by " | "+- For multi-column layouts: extract each column separately, separated by blank lines+- For forms: format as "Label: Value" on each line+- Preserve line breaks where they appear meaningful (paragraphs, list items)+- Include all visible text: headers, body text, labels, captions, watermarks+- For handwritten text: do your best to transcribe, use [illegible] for unclear parts++If no text is visible in the image, return an empty string for extracted_text.diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 3c1df7d04..e128558ee 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -3063,6 +3063,8 @@ class FileParserBlock(Block):return FileType.PDFelif suffix == ".tsv":return FileType.CSV # TSV files are handled by the CSV parser+ elif suffix in (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".tiff", ".tif"):+ return FileType.IMAGEelse:return FileType.CSV # Default to CSV for .csv and any other extensions@@ -3112,6 +3114,12 @@ class FileParserBlock(Block):validate_pdf_file(file_path, file_identifier=file_url_used)except PDFParsingError as e:raise InvalidFileType(file_url=file_url_used, file_type=self.file_type, error=str(e))+ elif self.file_type == FileType.IMAGE:+ kind = filetype.guess(file_path)+ if kind is None or not kind.mime.startswith("image/"):+ raise InvalidFileType(+ file_url=file_url_used, file_type=self.file_type, error="File is not a valid image"+ )async def _parse_csv_file(self, file_path: str) -> list[dict[str, Any]]:"""Parse CSV/TSV file and return list of dictionaries."""@@ -3184,6 +3192,27 @@ class FileParserBlock(Block):except PDFParsingError as e:raise InvalidFileType(file_url=self.file_url, file_type=self.file_type, error=str(e))+ async def _parse_image_file(self, file_path: str) -> str:+ """Parse image file using vision LLM for OCR."""+ try:+ with open(file_path, "rb") as f:+ image_bytes = f.read()++ llm_prompt = prompt_engine.load_prompt("extract-text-from-image")+ llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(+ self.override_llm_key, default=app.LLM_API_HANDLER+ )+ llm_response = await llm_api_handler(+ prompt=llm_prompt,+ prompt_name="extract-text-from-image",+ screenshots=[image_bytes],+ force_dict=True,+ )+ return llm_response.get("extracted_text", "")+ except Exception:+ LOG.exception("Failed to extract text from image via OCR", file_url=self.file_url)+ raise+async def _extract_with_ai(self, content: str | list[dict[str, Any]], workflow_run_context: WorkflowRunContext) -> dict[str, Any]:@@ -3210,9 +3239,8 @@ class FileParserBlock(Block):"extract-information-from-file-text", extracted_text_content=content_str, json_schema=schema_to_use)- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key, default=app.LLM_API_HANDLER- )+ llm_key = self.override_llm_key+ llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=app.LLM_API_HANDLER)llm_response = await llm_api_handler(prompt=llm_prompt, prompt_name="extract-information-from-file-text", force_dict=False@@ -3261,9 +3289,9 @@ class FileParserBlock(Block):else:file_path = await download_file(self.file_url)- # Auto-detect file type based on file extension- detected_file_type = self._detect_file_type_from_url(self.file_url)- self.file_type = detected_file_type+ # Auto-detect file type if not explicitly set (IMAGE/EXCEL/PDF are explicit choices)+ if self.file_type not in (FileType.IMAGE, FileType.EXCEL, FileType.PDF):+ self.file_type = self._detect_file_type_from_url(self.file_url)# Validate the file typeself.validate_file_type(self.file_url, file_path)@@ -3283,6 +3311,8 @@ class FileParserBlock(Block):parsed_data = await self._parse_excel_file(file_path)elif self.file_type == FileType.PDF:parsed_data = await self._parse_pdf_file(file_path)+ elif self.file_type == FileType.IMAGE:+ parsed_data = await self._parse_image_file(file_path)else:return await self.build_block_result(success=False,diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 8f8faf2ef..4b568a177 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -67,6 +67,7 @@ class FileType(StrEnum):CSV = "csv"EXCEL = "excel"PDF = "pdf"+ IMAGE = "image"class PDFFormat(StrEnum):