Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add schema-directed Parquet workflow exports (#8368)
Aaron Perez1f0454a8d47b
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex a41eda243..771960a26 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -9770,7 +9770,8 @@"google_sheets_write","pdf_fill","split_pdf",- "email_inbox"+ "email_inbox",+ "data_export"],"title": "BlockType"},@@ -12508,6 +12509,249 @@],"title": "CustomLLMUpdateRequest"},+ "DataExportBlock": {+ "properties": {+ "label": {+ "type": "string",+ "title": "Label",+ "description": "Author-facing identifier for a block; unique within a workflow."+ },+ "next_block_label": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Next Block Label",+ "description": "Optional pointer to the next block label when constructing a DAG. Defaults to sequential order when omitted."+ },+ "block_type": {+ "type": "string",+ "const": "data_export",+ "title": "Block Type",+ "default": "data_export"+ },+ "output_parameter": {+ "$ref": "#/components/schemas/OutputParameter"+ },+ "continue_on_failure": {+ "type": "boolean",+ "title": "Continue On Failure",+ "default": false+ },+ "model": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Model"+ },+ "disable_cache": {+ "type": "boolean",+ "title": "Disable Cache",+ "default": false+ },+ "ignore_workflow_system_prompt": {+ "type": "boolean",+ "title": "Ignore Workflow System Prompt",+ "default": false+ },+ "next_loop_on_failure": {+ "type": "boolean",+ "title": "Next Loop On Failure",+ "default": false+ },+ "data": {+ "type": "string",+ "title": "Data"+ },+ "data_schema": {+ "additionalProperties": true,+ "type": "object",+ "title": "Data Schema"+ },+ "file_name": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "File Name"+ },+ "parameters": {+ "items": {+ "oneOf": [+ {+ "$ref": "#/components/schemas/WorkflowParameter"+ },+ {+ "$ref": "#/components/schemas/ContextParameter"+ },+ {+ "$ref": "#/components/schemas/AWSSecretParameter"+ },+ {+ "$ref": "#/components/schemas/AzureSecretParameter"+ },+ {+ "$ref": "#/components/schemas/BitwardenLoginCredentialParameter"+ },+ {+ "$ref": "#/components/schemas/BitwardenSensitiveInformationParameter"+ },+ {+ "$ref": "#/components/schemas/BitwardenCreditCardDataParameter"+ },+ {+ "$ref": "#/components/schemas/OnePasswordCredentialParameter"+ },+ {+ "$ref": "#/components/schemas/AzureVaultCredentialParameter"+ },+ {+ "$ref": "#/components/schemas/OutputParameter"+ },+ {+ "$ref": "#/components/schemas/CredentialParameter"+ }+ ],+ "discriminator": {+ "propertyName": "parameter_type",+ "mapping": {+ "aws_secret": "#/components/schemas/AWSSecretParameter",+ "azure_secret": "#/components/schemas/AzureSecretParameter",+ "azure_vault_credential": "#/components/schemas/AzureVaultCredentialParameter",+ "bitwarden_credit_card_data": "#/components/schemas/BitwardenCreditCardDataParameter",+ "bitwarden_login_credential": "#/components/schemas/BitwardenLoginCredentialParameter",+ "bitwarden_sensitive_information": "#/components/schemas/BitwardenSensitiveInformationParameter",+ "context": "#/components/schemas/ContextParameter",+ "credential": "#/components/schemas/CredentialParameter",+ "onepassword": "#/components/schemas/OnePasswordCredentialParameter",+ "output": "#/components/schemas/OutputParameter",+ "workflow": "#/components/schemas/WorkflowParameter"+ }+ }+ },+ "type": "array",+ "title": "Parameters",+ "default": []+ }+ },+ "type": "object",+ "required": [+ "label",+ "output_parameter",+ "data",+ "data_schema"+ ],+ "title": "DataExportBlock"+ },+ "DataExportBlockYAML": {+ "properties": {+ "block_type": {+ "type": "string",+ "const": "data_export",+ "title": "Block Type",+ "default": "data_export"+ },+ "label": {+ "type": "string",+ "title": "Label",+ "description": "Author-facing identifier; must be unique per workflow."+ },+ "next_block_label": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Next Block Label",+ "description": "Optional pointer to the label of the next block. When omitted, it will default to sequential order. See [[s-4bnl]]."+ },+ "continue_on_failure": {+ "type": "boolean",+ "title": "Continue On Failure",+ "default": false+ },+ "model": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Model"+ },+ "ignore_workflow_system_prompt": {+ "type": "boolean",+ "title": "Ignore Workflow System Prompt",+ "default": false+ },+ "next_loop_on_failure": {+ "type": "boolean",+ "title": "Next Loop On Failure",+ "default": false+ },+ "data": {+ "type": "string",+ "title": "Data"+ },+ "data_schema": {+ "additionalProperties": true,+ "type": "object",+ "title": "Data Schema"+ },+ "file_name": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "File Name"+ },+ "parameter_keys": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Parameter Keys"+ }+ },+ "type": "object",+ "required": [+ "label",+ "data",+ "data_schema"+ ],+ "title": "DataExportBlockYAML"+ },"DeleteScheduleResponse": {"properties": {"ok": {@@ -15964,6 +16208,9 @@},{"$ref": "#/components/schemas/SplitPdfBlock"+ },+ {+ "$ref": "#/components/schemas/DataExportBlock"}],"discriminator": {@@ -15972,6 +16219,7 @@"action": "#/components/schemas/ActionBlock","code": "#/components/schemas/CodeBlock","conditional": "#/components/schemas/ConditionalBlock",+ "data_export": "#/components/schemas/DataExportBlock","download_to_s3": "#/components/schemas/DownloadToS3Block","email_inbox": "#/components/schemas/EmailInboxBlock","extraction": "#/components/schemas/ExtractionBlock",@@ -16215,6 +16463,9 @@{"$ref": "#/components/schemas/FileDownloadBlockYAML"},+ {+ "$ref": "#/components/schemas/DataExportBlockYAML"+ },{"$ref": "#/components/schemas/UrlBlockYAML"},@@ -27061,6 +27312,9 @@},{"$ref": "#/components/schemas/SplitPdfBlock"+ },+ {+ "$ref": "#/components/schemas/DataExportBlock"}],"discriminator": {@@ -27069,6 +27323,7 @@"action": "#/components/schemas/ActionBlock","code": "#/components/schemas/CodeBlock","conditional": "#/components/schemas/ConditionalBlock",+ "data_export": "#/components/schemas/DataExportBlock","download_to_s3": "#/components/schemas/DownloadToS3Block","email_inbox": "#/components/schemas/EmailInboxBlock","extraction": "#/components/schemas/ExtractionBlock",@@ -27240,6 +27495,9 @@{"$ref": "#/components/schemas/FileDownloadBlockYAML"},+ {+ "$ref": "#/components/schemas/DataExportBlockYAML"+ },{"$ref": "#/components/schemas/UrlBlockYAML"},@@ -28099,6 +28357,9 @@},{"$ref": "#/components/schemas/SplitPdfBlock"+ },+ {+ "$ref": "#/components/schemas/DataExportBlock"}],"discriminator": {@@ -28107,6 +28368,7 @@"action": "#/components/schemas/ActionBlock","code": "#/components/schemas/CodeBlock","conditional": "#/components/schemas/ConditionalBlock",+ "data_export": "#/components/schemas/DataExportBlock","download_to_s3": "#/components/schemas/DownloadToS3Block","email_inbox": "#/components/schemas/EmailInboxBlock","extraction": "#/components/schemas/ExtractionBlock",@@ -28320,6 +28582,9 @@{"$ref": "#/components/schemas/FileDownloadBlockYAML"},+ {+ "$ref": "#/components/schemas/DataExportBlockYAML"+ },{"$ref": "#/components/schemas/UrlBlockYAML"},@@ -28363,6 +28628,7 @@"action": "#/components/schemas/ActionBlockYAML","code": "#/components/schemas/CodeBlockYAML","conditional": "#/components/schemas/ConditionalBlockYAML",+ "data_export": "#/components/schemas/DataExportBlockYAML","download_to_s3": "#/components/schemas/DownloadToS3BlockYAML","email_inbox": "#/components/schemas/EmailInboxBlockYAML","extraction": "#/components/schemas/ExtractionBlockYAML",diff --git a/pyproject.toml b/pyproject.tomlindex 8d6c9b3f8..f2374bfb6 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -70,6 +70,7 @@ local = ["libcst>=1.9.0,<2","curlparser>=0.1.0,<0.2","pandas>=3.0.5,<4",+ "pyarrow>=18.0.0,<24","azure-identity>=1.24.0,<2","azure-keyvault-secrets>=4.11.1,<5","google-cloud-secret-manager>=2.30.0,<3",@@ -151,6 +152,7 @@ server = ["libcst>=1.9.0,<2","curlparser>=0.1.0,<0.2","pandas>=3.0.5,<4",+ "pyarrow>=18.0.0,<24","azure-identity>=1.24.0,<2","azure-keyvault-secrets>=4.11.1,<5","google-cloud-secret-manager>=2.30.0,<3",diff --git a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.ts b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.tsindex bfab35668..55b01b2cf 100644--- a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.ts@@ -22,4 +22,10 @@ describe("collapsibleBlockTypes — loop registration", () => {test("toWorkflowBlockType translates loop -> for_loop", () => {expect(toWorkflowBlockType("loop")).toBe("for_loop");});++ test("data export is collapsible", () => {+ expect(collapsibleRfNodeTypes.has("dataExport")).toBe(true);+ expect(collapsibleWorkflowBlockTypes.has("data_export")).toBe(true);+ expect(toWorkflowBlockType("dataExport")).toBe("data_export");+ });});diff --git a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.ts b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.tsindex 6a9ebab72..e8b6b449c 100644--- a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.ts+++ b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.ts@@ -32,6 +32,7 @@ const COLLAPSIBLE_NODE_TYPE_TO_BLOCK_TYPE: Record<string, WorkflowBlockType> = {googleSheetsWrite: "google_sheets_write",pdfFill: "pdf_fill",splitPdf: "split_pdf",+ dataExport: "data_export",loop: "for_loop",conditional: "conditional",};diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportEditor.tsxnew file mode 100644index 000000000..2a8464eaf--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportEditor.tsx@@ -0,0 +1,88 @@+import { useEdges, useNodes, useNodesData } from "@xyflow/react";++import { Input } from "@/components/ui/input";+import { Label } from "@/components/ui/label";+import { Textarea } from "@/components/ui/textarea";+import { WorkflowDataSchemaInputGroup } from "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup";++import { type AppNode } from "..";+import { getAvailableOutputParameterKeys } from "../../workflowEditorUtils";+import { useUpdate } from "../../useUpdate";+import { ParametersMultiSelect } from "../TaskNode/ParametersMultiSelect";+import { type DataExportNode, type DataExportNodeData } from "./types";++const dataSchemaExample = {+ type: "array",+ items: {+ type: "object",+ properties: { value: { type: "string" } },+ },+};++function DataExportEditor({ blockId }: { blockId: string }) {+ const nodeSlice = useNodesData<DataExportNode>(blockId);+ if (!nodeSlice || nodeSlice.type !== "dataExport") {+ return null;+ }+ return <DataExportEditorBody blockId={blockId} data={nodeSlice.data} />;+}++function DataExportEditorBody({+ blockId,+ data,+}: {+ blockId: string;+ data: DataExportNodeData;+}) {+ const nodes = useNodes<AppNode>();+ const edges = useEdges();+ const update = useUpdate<DataExportNodeData>({+ id: blockId,+ editable: data.editable,+ });+ const availableOutputParameters = getAvailableOutputParameterKeys(+ nodes,+ edges,+ blockId,+ );++ return (+ <div data-testid="data-export-block-form" className="space-y-4 p-4">+ <div className="space-y-2">+ <Label className="text-xs text-tertiary-foreground">Records</Label>+ <Textarea+ className="nopan min-h-24 font-mono text-xs"+ disabled={!data.editable}+ onChange={(event) => update({ data: event.target.value })}+ placeholder="{{ extraction_output.extracted_information }}"+ value={data.data}+ />+ </div>+ <WorkflowDataSchemaInputGroup+ exampleValue={dataSchemaExample}+ onChange={(dataSchema) => update({ dataSchema })}+ suggestionContext={{ current_schema: data.dataSchema }}+ value={data.dataSchema}+ />+ <div className="space-y-2">+ <Label className="text-xs text-tertiary-foreground">+ Filename (optional)+ </Label>+ <Input+ className="nopan text-xs"+ disabled={!data.editable}+ onChange={(event) => update({ fileName: event.target.value })}+ placeholder="records"+ value={data.fileName}+ />+ </div>+ <ParametersMultiSelect+ availableOutputParameters={availableOutputParameters}+ onParametersChange={(parameterKeys) => update({ parameterKeys })}+ parameters={data.parameterKeys}+ />+ </div>+ );+}++export { DataExportEditor };diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportNode.tsxnew file mode 100644index 000000000..491a82dff--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportNode.tsx@@ -0,0 +1,71 @@+import { Handle, NodeProps, Position } from "@xyflow/react";+import { useParams } from "react-router-dom";++import { statusIsRunningOrQueued } from "@/routes/tasks/types";+import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";+import { useRecordingStore } from "@/store/useRecordingStore";+import { cn } from "@/util/utils";++import { useCollapseContext } from "../../collapse/CollapseContext";+import { NodeBody } from "../../collapse/NodeBody";+import { BuildModeOnly } from "../BuildModeOnly";+import { NodeHeader } from "../components/NodeHeader";+import { DataExportEditor } from "./DataExportEditor";+import type { DataExportNode } from "./types";++function DataExportNode({ id, data }: NodeProps<DataExportNode>) {+ const { editable, label } = data;+ const { blockLabel: urlBlockLabel } = useParams();+ const { data: workflowRun } = useWorkflowRunQuery();+ const recordingStore = useRecordingStore();+ const { open } = useCollapseContext();+ const isTargeted = urlBlockLabel !== undefined && urlBlockLabel === label;+ const isRunning = workflowRun && statusIsRunningOrQueued(workflowRun);++ return (+ <div+ className={cn({+ "pointer-events-none opacity-50": recordingStore.isRecording,+ })}+ >+ <Handle+ type="source"+ position={Position.Bottom}+ id="a"+ className="opacity-0"+ />+ <Handle+ type="target"+ position={Position.Top}+ id="b"+ className="opacity-0"+ />+ <div+ className={cn(+ "transform-origin-center w-[30rem] space-y-4 rounded-lg bg-slate-elevation3 px-6 py-4 transition-shadow motion-reduce:transition-none",+ open ? "shadow-md" : "shadow-sm",+ {+ "pointer-events-none": isRunning && isTargeted,+ "bg-background outline outline-2 outline-ring": isTargeted,+ },+ )}+ >+ <NodeHeader+ blockLabel={label}+ editable={editable}+ nodeId={id}+ totpIdentifier={null}+ totpUrl={null}+ type="data_export"+ />+ <NodeBody>+ <BuildModeOnly>+ <DataExportEditor blockId={id} />+ </BuildModeOnly>+ </NodeBody>+ </div>+ </div>+ );+}++export { DataExportNode };diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/types.tsnew file mode 100644index 000000000..1c40c161f--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/types.ts@@ -0,0 +1,36 @@+import type { Node } from "@xyflow/react";++import { debuggableWorkflowBlockTypes } from "@/routes/workflows/types/workflowTypes";++import { NodeBaseData } from "../types";++export type DataExportNodeData = NodeBaseData & {+ data: string;+ dataSchema: string;+ fileName: string;+ parameterKeys: Array<string>;+};++export type DataExportNode = Node<DataExportNodeData, "dataExport">;++export const dataExportNodeDefaultData: DataExportNodeData = {+ debuggable: debuggableWorkflowBlockTypes.has("data_export"),+ editable: true,+ label: "",+ data: "[]",+ dataSchema: JSON.stringify(+ {+ type: "array",+ items: {+ type: "object",+ properties: { value: { type: "string" } },+ },+ },+ null,+ 2,+ ),+ fileName: "",+ parameterKeys: [],+ continueOnFailure: false,+ model: null,+} as const;diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsxindex ec9f64e6f..f2fbd470d 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsx@@ -97,6 +97,9 @@ function WorkflowBlockIcon({ workflowBlockType, className }: Props) {case "split_pdf": {return <FileTextIcon className={className} />;}+ case "data_export": {+ return <FileTextIcon className={className} />;+ }case "goto_url": {return <ExternalLinkIcon className={className} />;}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/index.tsindex 92f45d7b8..7fc079638 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts@@ -62,6 +62,8 @@ import { PdfFillNode } from "./PdfFillNode/types";import { PdfFillNode as PdfFillNodeComponent } from "./PdfFillNode/PdfFillNode";import { SplitPdfNode } from "./SplitPdfNode/types";import { SplitPdfNode as SplitPdfNodeComponent } from "./SplitPdfNode/SplitPdfNode";+import { DataExportNode } from "./DataExportNode/types";+import { DataExportNode as DataExportNodeComponent } from "./DataExportNode/DataExportNode";import { withSortableBlock } from "../sortable/withSortableBlock";import { withCollapsible } from "../collapse/withCollapsible";import { withSelectableBlock } from "../selection/withSelectableBlock";@@ -97,7 +99,8 @@ export type WorkflowBlockNode =| GoogleSheetsReadNode| GoogleSheetsWriteNode| PdfFillNode- | SplitPdfNode;+ | SplitPdfNode+ | DataExportNode;export function isUtilityNode(node: AppNode): node is UtilityNode {return node.type === "nodeAdder" || node.type === "start";@@ -162,4 +165,5 @@ export const nodeTypes = {googleSheetsWrite: wrapBlock(GoogleSheetsWriteNodeComponent),pdfFill: wrapBlock(PdfFillNodeComponent),splitPdf: wrapBlock(SplitPdfNodeComponent),+ dataExport: wrapBlock(DataExportNodeComponent),} as const;diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/types.tsindex 303439d45..d59b9f530 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/types.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/types.ts@@ -101,4 +101,5 @@ export const workflowBlockTitle: {google_sheets_write: "Google Sheets Write",pdf_fill: "PDF Fill",split_pdf: "Split PDF",+ data_export: "Data Export",};diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsxindex a26a11a6f..35853cd9d 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsx@@ -45,6 +45,7 @@ describe("nodeTypes composition order", () => {"googleSheetsWrite","pdfFill","splitPdf",+ "dataExport","emailInbox",]),);diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsxindex 35a2d5e43..34b8ed9b9 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsx@@ -84,7 +84,7 @@ describe("BLOCK_FORMS dispatcher", () => {}});- test("contains the 29 expected node.type keys", () => {+ test("contains the 30 expected node.type keys", () => {expect(new Set(BLOCK_FORM_KEYS)).toEqual(new Set(["task",@@ -115,10 +115,11 @@ describe("BLOCK_FORMS dispatcher", () => {"googleSheetsWrite","pdfFill","splitPdf",+ "dataExport","emailInbox",]),);- expect(BLOCK_FORM_KEYS).toHaveLength(29);+ expect(BLOCK_FORM_KEYS).toHaveLength(30);});test("conditional routes to a sidebar form that shows branch prompts", () => {diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsxindex 04d49a462..f2ad12883 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsx@@ -8,6 +8,7 @@ import { ActionBlockForm } from "./BlockConfigForm/ActionBlockForm";import { CodeBlockBlockForm } from "./BlockConfigForm/CodeBlockBlockForm";import { ConditionalBlockForm } from "./BlockConfigForm/ConditionalBlockForm";import { DownloadBlockForm } from "./BlockConfigForm/DownloadBlockForm";+import { DataExportBlockForm } from "./BlockConfigForm/DataExportBlockForm";import { EmailInboxBlockForm } from "./BlockConfigForm/EmailInboxBlockForm";import { ExtractionBlockForm } from "./BlockConfigForm/ExtractionBlockForm";import { FileDownloadBlockForm } from "./BlockConfigForm/FileDownloadBlockForm";@@ -69,6 +70,7 @@ const BLOCK_FORMS: Record<WorkflowBlockNodeType, BlockFormComponent> = {googleSheetsRead: GoogleSheetsReadBlockForm,googleSheetsWrite: GoogleSheetsWriteBlockForm,splitPdf: SplitPdfBlockForm,+ dataExport: DataExportBlockForm,};function BlockConfigForm({ blockId }: Readonly<{ blockId: string }>) {diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/DataExportBlockForm.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/DataExportBlockForm.tsxnew file mode 100644index 000000000..921949763--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/DataExportBlockForm.tsx@@ -0,0 +1,51 @@+import { useReactFlow } from "@xyflow/react";+import { useEffect, useMemo } from "react";++import { usePendingCommitsStore } from "@/store/PendingCommitsStore";++import { type AppNode, isWorkflowBlockNode } from "../../nodes";+import { DataExportEditor } from "../../nodes/DataExportNode/DataExportEditor";+import type { DataExportNode } from "../../nodes/DataExportNode/types";+import { useDebouncedSidebarSave } from "../useDebouncedSidebarSave";++function DataExportBlockForm({ blockId }: { blockId: string }) {+ const reactFlow = useReactFlow<AppNode>();+ const node = reactFlow.getNode(blockId);+ if (!node || !isWorkflowBlockNode(node) || node.type !== "dataExport") {+ return null;+ }+ return <DataExportBlockFormBody blockId={blockId} node={node} />;+}++function DataExportBlockFormBody({+ blockId,+ node,+}: {+ blockId: string;+ node: DataExportNode;+}) {+ const value = useMemo(+ () => ({+ data: node.data.data,+ dataSchema: node.data.dataSchema,+ fileName: node.data.fileName,+ parameterKeys: node.data.parameterKeys,+ }),+ [+ node.data.data,+ node.data.dataSchema,+ node.data.fileName,+ node.data.parameterKeys,+ ],+ );+ const { commit } = useDebouncedSidebarSave({ blockId, value });++ useEffect(() => {+ usePendingCommitsStore.getState().register(blockId, commit);+ return () => usePendingCommitsStore.getState().unregister(blockId);+ }, [blockId, commit]);++ return <DataExportEditor blockId={blockId} />;+}++export { DataExportBlockForm };diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsxindex 57b49478e..933351431 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsx@@ -72,6 +72,7 @@ const NODE_TYPE_TO_BLOCK_TYPE: Record<googleSheetsWrite: "google_sheets_write",pdfFill: "pdf_fill",splitPdf: "split_pdf",+ dataExport: "data_export",};function getBlockTypeFromNode(node: AppNode): WorkflowBlockType | null {diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsxindex c3dee3e63..0ac1cffdb 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx@@ -83,6 +83,17 @@ const nodeLibraryItems: Array<{title: "Human Interaction Block",description: "Pause agent for human review and approval",},+ {+ nodeType: "dataExport",+ icon: (+ <WorkflowBlockIcon+ workflowBlockType={WorkflowBlockTypes.DataExport}+ className="size-6"+ />+ ),+ title: "Data Export Block",+ description: "Write schema-defined records to a Parquet file",+ },// {// nodeType: "task",// icon: (diff --git a/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.ts b/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.tsindex baed22b7b..e20a03269 100644--- a/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.ts@@ -2,7 +2,11 @@ import { describe, expect, test } from "vitest";import { ProxyLocation, RunEngine } from "@/api/types";-import type { WorkflowBlock, WorkflowSettings } from "../types/workflowTypes";+import type {+ DataExportBlock,+ WorkflowBlock,+ WorkflowSettings,+} from "../types/workflowTypes";import { getElements, getWorkflowBlocks } from "./workflowEditorUtils";@@ -73,3 +77,44 @@ describe("engine round-trips through node data", () => {expect(savedBlock).toMatchObject({ engine: "skyvern-3.0" });});});++describe("data export blocks", () => {+ test("an API-authored export block loads and saves without changing its contract", () => {+ const block = {+ label: "export_records",+ block_type: "data_export",+ continue_on_failure: false,+ model: null,+ next_block_label: null,+ parameters: [],+ data: "{{ extraction_output.extracted_information }}",+ data_schema: {+ type: "array",+ items: {+ type: "object",+ properties: { id: { type: "integer" } },+ },+ },+ file_name: "records",+ } as unknown as DataExportBlock;++ const { nodes, edges } = getElements([block], DEFAULT_SETTINGS, true);+ const exportNode = nodes.find(+ (node) => node.data.label === "export_records",+ );++ expect(exportNode?.type).toBe("dataExport");+ expect(exportNode?.data).toMatchObject({+ data: "{{ extraction_output.extracted_information }}",+ fileName: "records",+ });+ expect(getWorkflowBlocks(nodes, edges)).toEqual([+ expect.objectContaining({+ block_type: "data_export",+ data: "{{ extraction_output.extracted_information }}",+ data_schema: block.data_schema,+ file_name: "records",+ }),+ ]);+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 78eac53b9..9031d97b0 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -29,6 +29,7 @@ import {BlockYAML,CodeBlockYAML,ConditionalBlockYAML,+ DataExportBlockYAML,DownloadToS3BlockYAML,FileUrlParserBlockYAML,ForLoopBlockYAML,@@ -74,6 +75,7 @@ import {import { ParametersState } from "./types";import { AppNode, isWorkflowBlockNode, WorkflowBlockNode } from "./nodes";import { codeBlockNodeDefaultData } from "./nodes/CodeBlockNode/types";+import { dataExportNodeDefaultData } from "./nodes/DataExportNode/types";import { downloadNodeDefaultData } from "./nodes/DownloadNode/types";import {isFileParserNode,@@ -1006,6 +1008,20 @@ function convertToNode(},};}+ case "data_export": {+ return {+ ...identifiers,+ ...common,+ type: "dataExport",+ data: {+ ...commonData,+ data: block.data,+ dataSchema: JSON.stringify(block.data_schema, null, 2),+ fileName: block.file_name ?? "",+ parameterKeys: (block.parameters ?? []).map((p) => p.key),+ },+ };+ }case "for_loop": {const loopVariableReference =block.loop_variable_reference !== null@@ -2617,6 +2633,17 @@ function createNode(},};}+ case "dataExport": {+ return {+ ...identifiers,+ ...common,+ type: "dataExport",+ data: {+ ...dataExportNodeDefaultData,+ label,+ },+ };+ }case "download": {return {...identifiers,@@ -3278,6 +3305,16 @@ function getWorkflowBlock(steps: node.data.steps,};}+ case "dataExport": {+ return {+ ...base,+ block_type: "data_export",+ data: node.data.data,+ data_schema: JSONParseSafe(node.data.dataSchema) ?? {},+ file_name: node.data.fileName || null,+ parameter_keys: node.data.parameterKeys,+ };+ }case "download": {return {...base,@@ -4708,6 +4745,17 @@ function convertBlocksToBlockYAML(};return blockYaml;}+ case "data_export": {+ const blockYaml: DataExportBlockYAML = {+ ...base,+ block_type: "data_export",+ data: block.data,+ data_schema: block.data_schema,+ file_name: block.file_name,+ parameter_keys: (block.parameters ?? []).map((p) => p.key),+ };+ return blockYaml;+ }case "download_to_s3": {const blockYaml: DownloadToS3BlockYAML = {...base,diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.tsindex 884e12d95..2899d28cd 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts@@ -211,6 +211,7 @@ export type WorkflowBlock =| FileURLParserBlock| ValidationBlock| HumanInteractionBlock+ | DataExportBlock| ActionBlock| NavigationBlock| ExtractionBlock@@ -243,6 +244,7 @@ export const WorkflowBlockTypes = {FileURLParser: "file_url_parser",Validation: "validation",HumanInteraction: "human_interaction",+ DataExport: "data_export",Action: "action",Navigation: "navigation",Extraction: "extraction",@@ -498,6 +500,14 @@ export type HumanInteractionBlock = WorkflowBlockBase & {body: string;};+export type DataExportBlock = WorkflowBlockBase & {+ block_type: "data_export";+ data: string;+ data_schema: Record<string, unknown>;+ file_name: string | null;+ parameters: Array<WorkflowParameter>;+};+export type ActionBlock = WorkflowBlockBase & {block_type: "action";url: string | null;diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.tsindex bd054482b..09c359bae 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts@@ -155,6 +155,7 @@ export type BlockYAML =| ConditionalBlockYAML| ValidationBlockYAML| HumanInteractionBlockYAML+ | DataExportBlockYAML| ActionBlockYAML| NavigationBlockYAML| ExtractionBlockYAML@@ -236,6 +237,14 @@ export type HumanInteractionBlockYAML = BlockYAMLBase & {body: string;};+export type DataExportBlockYAML = BlockYAMLBase & {+ block_type: "data_export";+ data: string;+ data_schema: Record<string, unknown>;+ file_name: string | null;+ parameter_keys?: Array<string> | null;+};+export type ActionBlockYAML = BlockYAMLBase & {block_type: "action";url: string | null;diff --git a/skyvern/cli/mcp_tools/blocks.py b/skyvern/cli/mcp_tools/blocks.pyindex a078d48f8..24dc8272f 100644--- a/skyvern/cli/mcp_tools/blocks.py+++ b/skyvern/cli/mcp_tools/blocks.py@@ -22,6 +22,7 @@ from skyvern.schemas.workflows import (BlockYAML,CodeBlockYAML,ConditionalBlockYAML,+ DataExportBlockYAML,DownloadToS3BlockYAML,EmailInboxBlockYAML,ExtractionBlockYAML,@@ -79,6 +80,7 @@ BLOCK_TYPE_MAP: dict[str, type[BlockYAML]] = {BlockType.FILE_UPLOAD.value: FileUploadBlockYAML,BlockType.GOTO_URL.value: UrlBlockYAML,BlockType.DOWNLOAD_TO_S3.value: DownloadToS3BlockYAML,+ BlockType.DATA_EXPORT.value: DataExportBlockYAML,BlockType.UPLOAD_TO_S3.value: UploadToS3BlockYAML,BlockType.FILE_URL_PARSER.value: FileParserBlockYAML,BlockType.PDF_PARSER.value: PDFParserBlockYAML,@@ -114,6 +116,7 @@ BLOCK_SUMMARIES: dict[str, str] = {"file_upload": "Upload a file from S3/Azure to a page element","goto_url": "Navigate directly to a URL without additional instructions","download_to_s3": "Download a URL directly to S3 storage",+ "data_export": "Write schema-defined workflow records to a Parquet file","upload_to_s3": "Upload local content to S3","file_url_parser": "Parse a file (CSV/Excel/PDF/image/DOCX) from a URL; ZIP archives are unzipped to a file list","pdf_parser": "Extract structured data from a PDF document",@@ -157,6 +160,22 @@ BLOCK_EXAMPLES: dict[str, dict[str, Any]] = {},},},+ "data_export": {+ "block_type": "data_export",+ "label": "export_records",+ "data": "{{ extract_records_output.extracted_information }}",+ "data_schema": {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "name": {"type": "string"},+ "price": {"type": "number"},+ },+ },+ },+ "file_name": "records",+ },"for_loop": {"block_type": "for_loop","label": "process_each_url",diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 5a5060802..bcaab9c97 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -15387,6 +15387,7 @@ def get_all_blocks(blocks: list[BlockTypeVar]) -> list[BlockTypeVar]:# Late import: google_sheets_blocks imports Block from this module, so top-level import would cycle.+from skyvern.forge.sdk.workflow.models.data_export_block import DataExportBlock # noqa: E402from skyvern.forge.sdk.workflow.models.email_inbox_block import EmailInboxBlock # noqa: E402from skyvern.forge.sdk.workflow.models.google_sheets_blocks import ( # noqa: E402GoogleSheetsReadBlock,@@ -15426,6 +15427,7 @@ BlockSubclasses = Union[GoogleSheetsWriteBlock,PdfFillBlock,SplitPdfBlock,+ DataExportBlock,]BlockTypeVar = Annotated[BlockSubclasses, Field(discriminator="block_type")]diff --git a/skyvern/forge/sdk/workflow/models/data_export_block.py b/skyvern/forge/sdk/workflow/models/data_export_block.pynew file mode 100644index 000000000..1c10b61e5--- /dev/null+++ b/skyvern/forge/sdk/workflow/models/data_export_block.py@@ -0,0 +1,196 @@+"""Workflow block that exports schema-defined records as Parquet."""++from __future__ import annotations++import asyncio+import json+from typing import Any, Literal++import structlog++from skyvern.constants import GET_DOWNLOADED_FILES_TIMEOUT, SAVE_DOWNLOADED_FILES_TIMEOUT+from skyvern.exceptions import DownloadSaveIncompleteError+from skyvern.forge import app+from skyvern.forge.sdk.api.files import get_path_for_workflow_download_directory, resolve_run_download_id+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.schemas.files import FileInfo+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext+from skyvern.forge.sdk.workflow.exceptions import FailedToFormatJinjaStyleParameter+from skyvern.forge.sdk.workflow.models._jinja import jinja_json_finalize_required_binding_env+from skyvern.forge.sdk.workflow.models.block import Block, sanitize_filename+from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE+from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType+from skyvern.utils.parquet_export import ParquetExportError, export_parquet_records++LOG = structlog.get_logger()+++class DataExportBlock(Block):+ block_type: Literal[BlockType.DATA_EXPORT] = BlockType.DATA_EXPORT # type: ignore++ data: str+ data_schema: dict[str, Any]+ file_name: str | None = None+ parameters: list[PARAMETER_TYPE] = []++ def get_all_parameters(self, workflow_run_id: str) -> list[PARAMETER_TYPE]:+ return self.parameters++ def format_potential_template_parameters(self, workflow_run_context: WorkflowRunContext) -> None:+ self.data = self.format_block_parameter_template_from_workflow_run_context(+ self.data,+ workflow_run_context,+ env=jinja_json_finalize_required_binding_env,+ skip_missing_variable_preflight=True,+ )+ if self.file_name:+ self.file_name = self.format_block_parameter_template_from_workflow_run_context(+ self.file_name, workflow_run_context+ )++ def _resolve_file_name(self, workflow_run_context: WorkflowRunContext) -> str:+ stem = sanitize_filename(self.file_name or self.label)+ if stem.lower().endswith(".parquet"):+ stem = stem[: -len(".parquet")]+ current_index = workflow_run_context.get_block_metadata(self.label).get("current_index")+ if isinstance(current_index, int) and not isinstance(current_index, bool):+ stem = f"{stem}-{current_index + 1:04d}"+ return f"{stem}.parquet"++ @staticmethod+ def _parse_records(data: str) -> list[Any]:+ try:+ records = json.loads(data)+ except json.JSONDecodeError as exc:+ raise ParquetExportError("data must resolve to a JSON array") from exc+ if not isinstance(records, list):+ raise ParquetExportError("data must resolve to a JSON array of object records")+ return records++ async def _register_download(+ self,+ *,+ organization_id: str | None,+ run_download_id: str | None,+ workflow_run_id: str,+ workflow_run_block_id: str,+ ) -> list[FileInfo]:+ if not organization_id:+ return []+ try:+ async with asyncio.timeout(SAVE_DOWNLOADED_FILES_TIMEOUT):+ await app.STORAGE.save_downloaded_files(organization_id=organization_id, run_id=run_download_id)+ except asyncio.TimeoutError:+ LOG.warning(+ "Timeout saving Parquet export; workflow finalization will retry",+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ )+ return []+ except DownloadSaveIncompleteError:+ pass+ except Exception:+ LOG.warning(+ "Failed to register Parquet export; workflow finalization will retry",+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ exc_info=True,+ )+ return []+ try:+ async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):+ return await app.STORAGE.get_downloaded_files(organization_id=organization_id, run_id=run_download_id)+ except Exception:+ LOG.warning(+ "Failed to read registered Parquet exports",+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ exc_info=True,+ )+ return []++ async def _failure(+ self,+ reason: str,+ workflow_run_block_id: str,+ organization_id: str | None,+ ) -> BlockResult:+ return await self.build_block_result(+ success=False,+ failure_reason=reason,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )++ async def execute(+ self,+ workflow_run_id: str,+ workflow_run_block_id: str,+ organization_id: str | None = None,+ browser_session_id: str | None = None,+ **kwargs: Any,+ ) -> BlockResult:+ workflow_run_context = self.get_workflow_run_context(workflow_run_id)+ try:+ self.format_potential_template_parameters(workflow_run_context)+ records = self._parse_records(self.data)+ parquet_data = export_parquet_records(records, self.data_schema)+ except (FailedToFormatJinjaStyleParameter, ParquetExportError) as exc:+ return await self._failure(str(exc), workflow_run_block_id, organization_id)++ filename = self._resolve_file_name(workflow_run_context)+ run_download_id = resolve_run_download_id(skyvern_context.current(), fallback_run_id=workflow_run_id)+ path = get_path_for_workflow_download_directory(run_download_id) / filename+ file_created = False+ try:+ path.parent.mkdir(parents=True, exist_ok=True)+ stem = path.stem+ suffix = 2+ while True:+ try:+ with path.open("xb") as parquet_file:+ file_created = True+ parquet_file.write(parquet_data)+ break+ except FileExistsError:+ path = path.with_name(f"{stem}-{suffix:04d}{path.suffix}")+ suffix += 1+ except OSError as exc:+ if file_created:+ try:+ path.unlink()+ except FileNotFoundError:+ pass+ except OSError:+ LOG.warning("Failed to remove incomplete Parquet export", exc_info=True)+ return await self._failure(f"failed to write {filename}: {exc}", workflow_run_block_id, organization_id)++ filename = path.name+ downloaded_files = await self._register_download(+ organization_id=organization_id or workflow_run_context.organization_id,+ run_download_id=run_download_id,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ )+ downloaded_files = [file for file in downloaded_files if file.filename == filename]+ output = {+ "file_name": filename,+ "file_path": str(path),+ "file_size": path.stat().st_size,+ "format": "parquet",+ "compression": "snappy",+ "row_count": len(records),+ "columns": list(self.data_schema.get("items", {}).get("properties", {})),+ "downloaded_files": [file.model_dump() for file in downloaded_files],+ "downloaded_file_urls": [file.url for file in downloaded_files],+ }+ await self.record_output_parameter_value(workflow_run_context, workflow_run_id, output)+ return await self.build_block_result(+ success=True,+ failure_reason=None,+ output_parameter_value=output,+ status=BlockStatus.completed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex e071cd93b..2c5a87841 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -63,6 +63,7 @@ from skyvern.forge.sdk.workflow.models.block import (WhileLoopBlock,WorkflowTriggerBlock,)+from skyvern.forge.sdk.workflow.models.data_export_block import DataExportBlockfrom skyvern.forge.sdk.workflow.models.email_inbox_block import EmailInboxBlockfrom skyvern.forge.sdk.workflow.models.google_sheets_blocks import (GoogleSheetsReadBlock,@@ -894,6 +895,15 @@ def block_yaml_to_block(wait_sec=block_yaml.wait_sec,)+ elif block_yaml.block_type == BlockType.DATA_EXPORT:+ return DataExportBlock(+ **base_kwargs,+ data=block_yaml.data,+ data_schema=block_yaml.data_schema,+ file_name=block_yaml.file_name,+ parameters=_resolve_block_parameters(block_yaml, parameters),+ )+elif block_yaml.block_type == BlockType.FILE_DOWNLOAD:file_download_block_parameters = _resolve_block_parameters(block_yaml, parameters)return FileDownloadBlock(diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex b06bda750..eba68c2dc 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -502,6 +502,7 @@ class BlockType(StrEnum):PDF_FILL = "pdf_fill"SPLIT_PDF = "split_pdf"EMAIL_INBOX = "email_inbox"+ DATA_EXPORT = "data_export"class AIFallbackMode(StrEnum):@@ -1273,6 +1274,15 @@ class HumanInteractionBlockYAML(BlockYAML):body: str = "Your interaction is required for a workflow run!"+class DataExportBlockYAML(BlockYAML):+ block_type: Literal[BlockType.DATA_EXPORT] = BlockType.DATA_EXPORT # type: ignore++ data: str+ data_schema: dict[str, Any]+ file_name: str | None = None+ parameter_keys: list[str] | None = None++class FileDownloadBlockYAML(BlockYAML):block_type: Literal[BlockType.FILE_DOWNLOAD] = BlockType.FILE_DOWNLOAD # type: ignore@@ -1494,6 +1504,7 @@ BLOCK_YAML_SUBCLASSES = (| WaitBlockYAML| HumanInteractionBlockYAML| FileDownloadBlockYAML+ | DataExportBlockYAML| UrlBlockYAML| PDFParserBlockYAML| TaskV2BlockYAMLdiff --git a/skyvern/utils/parquet_export.py b/skyvern/utils/parquet_export.pynew file mode 100644index 000000000..98957f60e--- /dev/null+++ b/skyvern/utils/parquet_export.py@@ -0,0 +1,191 @@+"""Schema-directed Parquet serialization for workflow extraction records."""++from __future__ import annotations++import io+import json+import math+from collections import Counter+from collections.abc import Mapping, Sequence+from datetime import UTC, date, datetime+from typing import Any++import pyarrow as pa+import pyarrow.parquet as pq++INVALID_VALUE_COUNTS_METADATA_KEY = b"skyvern.invalid_value_counts"+++class ParquetExportError(ValueError):+ """The configured schema cannot describe a Parquet table."""+++def export_parquet_records(records: Any, data_schema: Mapping[str, Any]) -> bytes:+ """Write a JSON array of object records to a schema-directed Parquet file.++ Missing properties and values incompatible with their declared JSON Schema type+ become null. Incompatible-value counts are written as Parquet metadata without+ preserving raw extracted values.+ """+ item_schema = _record_schema(data_schema)+ properties = _properties(item_schema, path="items")+ warnings: Counter[str] = Counter()++ if not isinstance(records, list):+ raise ParquetExportError("data must resolve to a JSON array of object records")++ malformed_record_count = sum(not isinstance(record, Mapping) for record in records)+ if malformed_record_count:+ warnings["$[]"] = malformed_record_count+ columns = {+ field_name: [_coerce_record_value(record, field_name, field_schema, warnings) for record in records]+ for field_name, field_schema in properties.items()+ }+ fields = [+ pa.field(name, _arrow_type(field_schema, path=name), nullable=True) for name, field_schema in properties.items()+ ]+ metadata = {INVALID_VALUE_COUNTS_METADATA_KEY: json.dumps(dict(sorted(warnings.items()))).encode("utf-8")}+ table = pa.Table.from_arrays(+ [pa.array(columns[field.name], type=field.type) for field in fields],+ schema=pa.schema(fields, metadata=metadata),+ )+ sink = io.BytesIO()+ pq.write_table(table, sink, compression="snappy")+ return sink.getvalue()+++def _record_schema(data_schema: Mapping[str, Any]) -> Mapping[str, Any]:+ if _schema_type(data_schema, path="data_schema") != "array":+ raise ParquetExportError("data_schema must be a JSON Schema array")+ items = data_schema.get("items")+ if not isinstance(items, Mapping) or _schema_type(items, path="data_schema.items") != "object":+ raise ParquetExportError("data_schema.items must be a JSON Schema object")+ return items+++def _properties(schema: Mapping[str, Any], *, path: str) -> dict[str, Mapping[str, Any]]:+ raw_properties = schema.get("properties")+ if not isinstance(raw_properties, Mapping):+ raise ParquetExportError(f"{path}.properties must be an object")+ properties: dict[str, Mapping[str, Any]] = {}+ for name, field_schema in raw_properties.items():+ if not isinstance(name, str) or not isinstance(field_schema, Mapping):+ raise ParquetExportError(f"{path}.properties must map field names to JSON Schema objects")+ properties[name] = field_schema+ if not properties:+ raise ParquetExportError(f"{path}.properties must declare at least one property")+ return properties+++def _schema_type(schema: Mapping[str, Any], *, path: str) -> str:+ raw_type = schema.get("type")+ if isinstance(raw_type, str):+ return raw_type+ if isinstance(raw_type, Sequence) and not isinstance(raw_type, str):+ types = [value for value in raw_type if value != "null"]+ if len(types) == 1 and isinstance(types[0], str):+ return types[0]+ raise ParquetExportError(f"{path} must declare exactly one non-null JSON Schema type")+++def _arrow_type(schema: Mapping[str, Any], *, path: str) -> pa.DataType:+ schema_type = _schema_type(schema, path=path)+ if schema_type == "string":+ if schema.get("format") == "date":+ return pa.date32()+ if schema.get("format") == "date-time":+ return pa.timestamp("us", tz="UTC")+ return pa.string()+ if schema_type == "integer":+ return pa.int64()+ if schema_type == "number":+ return pa.float64()+ if schema_type == "boolean":+ return pa.bool_()+ if schema_type == "object":+ return pa.struct(+ [+ pa.field(name, _arrow_type(field_schema, path=f"{path}.{name}"), nullable=True)+ for name, field_schema in _properties(schema, path=path).items()+ ]+ )+ if schema_type == "array":+ items = schema.get("items")+ if not isinstance(items, Mapping):+ raise ParquetExportError(f"{path}.items must be a JSON Schema object")+ return pa.list_(_arrow_type(items, path=f"{path}[]"))+ raise ParquetExportError(f"{path} has unsupported JSON Schema type {schema_type!r}")+++def _coerce_record_value(+ record: Any,+ field_name: str,+ field_schema: Mapping[str, Any],+ warnings: Counter[str],+) -> Any:+ if not isinstance(record, Mapping):+ return None+ if field_name not in record:+ return None+ return _coerce_value(record[field_name], field_schema, path=field_name, warnings=warnings)+++def _coerce_value(value: Any, schema: Mapping[str, Any], *, path: str, warnings: Counter[str]) -> Any:+ if value is None:+ return None++ schema_type = _schema_type(schema, path=path)+ if schema_type == "string":+ if not isinstance(value, str):+ return _invalid(path, warnings)+ if schema.get("format") == "date":+ try:+ return date.fromisoformat(value)+ except ValueError:+ return _invalid(path, warnings)+ if schema.get("format") == "date-time":+ try:+ parsed = datetime.fromisoformat(value)+ if parsed.tzinfo is None:+ parsed = parsed.replace(tzinfo=UTC)+ return parsed.astimezone(UTC)+ except (OverflowError, ValueError):+ return _invalid(path, warnings)+ return value+ if schema_type == "integer":+ if isinstance(value, int) and not isinstance(value, bool) and -(2**63) <= value < 2**63:+ return value+ return _invalid(path, warnings)+ if schema_type == "number":+ if isinstance(value, (int, float)) and not isinstance(value, bool):+ try:+ number = float(value)+ except OverflowError:+ return _invalid(path, warnings)+ if math.isfinite(number):+ return number+ return _invalid(path, warnings)+ if schema_type == "boolean":+ return value if isinstance(value, bool) else _invalid(path, warnings)+ if schema_type == "object":+ if not isinstance(value, Mapping):+ return _invalid(path, warnings)+ return {+ field_name: _coerce_value(value[field_name], field_schema, path=f"{path}.{field_name}", warnings=warnings)+ if field_name in value+ else None+ for field_name, field_schema in _properties(schema, path=path).items()+ }+ if schema_type == "array":+ if not isinstance(value, list):+ return _invalid(path, warnings)+ item_schema = schema.get("items")+ if not isinstance(item_schema, Mapping):+ raise ParquetExportError(f"{path}.items must be a JSON Schema object")+ return [_coerce_value(item, item_schema, path=f"{path}[]", warnings=warnings) for item in value]+ raise ParquetExportError(f"{path} has unsupported JSON Schema type {schema_type!r}")+++def _invalid(path: str, warnings: Counter[str]) -> None:+ warnings[path] += 1+ return Nonediff --git a/tests/unit/test_data_export_block.py b/tests/unit/test_data_export_block.pynew file mode 100644index 000000000..03c2ffb50--- /dev/null+++ b/tests/unit/test_data_export_block.py@@ -0,0 +1,230 @@+from __future__ import annotations++from datetime import UTC, datetime+from pathlib import Path+from typing import Any, Self+from unittest.mock import AsyncMock++import pyarrow as pa+import pyarrow.parquet as pq+import pytest++from skyvern.forge import app+from skyvern.forge.sdk.schemas.files import FileInfo+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext+from skyvern.forge.sdk.workflow.models.data_export_block import DataExportBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter+from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block+from skyvern.schemas.workflows import BlockStatus, DataExportBlockYAML++_RECORD_SCHEMA = {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "id": {"type": "integer"},+ "name": {"type": "string"},+ },+ },+}+++def _output_parameter() -> OutputParameter:+ now = datetime.now(UTC)+ return OutputParameter(+ output_parameter_id="export_output_id",+ key="export_output",+ workflow_id="workflow-id",+ created_at=now,+ modified_at=now,+ )+++def _block(**overrides: object) -> DataExportBlock:+ return DataExportBlock(+ **{+ "label": "export",+ "output_parameter": _output_parameter(),+ "data": "[]",+ "data_schema": _RECORD_SCHEMA,+ **overrides,+ }+ )+++def test_yaml_conversion_builds_a_parquet_export_block() -> None:+ block = block_yaml_to_block(+ DataExportBlockYAML(+ label="export",+ data="{{ extract_output.extracted_information }}",+ data_schema=_RECORD_SCHEMA,+ ),+ {"export_output": _output_parameter()},+ )++ assert isinstance(block, DataExportBlock)+ assert block.data_schema == _RECORD_SCHEMA+++async def test_export_block_writes_and_registers_one_parquet_file_per_loop_iteration(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ download_directory = tmp_path / "workflow-downloads"+ context = WorkflowRunContext(+ workflow_title="Parquet test",+ workflow_id="workflow-id",+ workflow_permanent_id="wpid",+ workflow_run_id="run-id",+ aws_client=AsyncMock(),+ )+ context.organization_id = "org-id"+ context.values["extract_output"] = {"extracted_information": [{"id": 1, "name": "one"}, {"id": 2}]}+ context.update_block_metadata("export", {"current_index": 2})+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))+ monkeypatch.setattr(app.DATABASE.workflow_runs, "create_or_update_workflow_run_output_parameter", AsyncMock())+ monkeypatch.setattr(+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",+ lambda _run_id: download_directory,+ )+ monkeypatch.setattr(app.STORAGE, "save_downloaded_files", AsyncMock())+ monkeypatch.setattr(+ app.STORAGE,+ "get_downloaded_files",+ AsyncMock(+ return_value=[FileInfo(url="https://example.test/items-0003.parquet", filename="items-0003.parquet")]+ ),+ )++ result = await _block(+ data="{{ extract_output.extracted_information }}",+ file_name="items",+ ).execute("run-id", "block-id", organization_id="org-id")++ assert result.success is True+ assert result.status is BlockStatus.completed+ assert result.output_parameter_value == {+ "file_name": "items-0003.parquet",+ "file_path": str(download_directory / "items-0003.parquet"),+ "file_size": (download_directory / "items-0003.parquet").stat().st_size,+ "format": "parquet",+ "compression": "snappy",+ "row_count": 2,+ "columns": ["id", "name"],+ "downloaded_files": [+ FileInfo(url="https://example.test/items-0003.parquet", filename="items-0003.parquet").model_dump()+ ],+ "downloaded_file_urls": ["https://example.test/items-0003.parquet"],+ }+ table = pq.read_table(pa.BufferReader((download_directory / "items-0003.parquet").read_bytes()))+ assert table.to_pylist() == [{"id": 1, "name": "one"}, {"id": 2, "name": None}]+++async def test_export_block_preserves_parquet_files_with_repeated_loop_indexes(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ download_directory = tmp_path / "workflow-downloads"+ context = WorkflowRunContext(+ workflow_title="Parquet test",+ workflow_id="workflow-id",+ workflow_permanent_id="wpid",+ workflow_run_id="run-id",+ aws_client=AsyncMock(),+ )+ context.update_block_metadata("export", {"current_index": 0})+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))+ monkeypatch.setattr(app.DATABASE.workflow_runs, "create_or_update_workflow_run_output_parameter", AsyncMock())+ monkeypatch.setattr(+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",+ lambda _run_id: download_directory,+ )++ block = _block(file_name="items")+ first_result = await block.execute("run-id", "block-id")+ second_result = await block.execute("run-id", "block-id")++ assert first_result.output_parameter_value["file_name"] == "items-0001.parquet"+ assert second_result.output_parameter_value["file_name"] == "items-0001-0002.parquet"+ assert sorted(path.name for path in download_directory.glob("*.parquet")) == [+ "items-0001-0002.parquet",+ "items-0001.parquet",+ ]+++async def test_export_block_returns_clean_failure_when_writing_parquet_fails(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ context = WorkflowRunContext(+ workflow_title="Parquet test",+ workflow_id="workflow-id",+ workflow_permanent_id="wpid",+ workflow_run_id="run-id",+ aws_client=AsyncMock(),+ )+ download_directory = tmp_path / "workflow-downloads"+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))+ monkeypatch.setattr(+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",+ lambda _run_id: download_directory,+ )++ def fail_write(_path: Path, _mode: str = "r", **_kwargs: object) -> object:+ raise OSError("disk full")++ monkeypatch.setattr(Path, "open", fail_write)++ result = await _block().execute("run-id", "block-id")++ assert download_directory.is_dir()+ assert result.success is False+ assert result.status is BlockStatus.failed+ assert result.failure_reason == "failed to write export.parquet: disk full"+++async def test_export_block_removes_partial_parquet_file_after_write_failure(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ context = WorkflowRunContext(+ workflow_title="Parquet test",+ workflow_id="workflow-id",+ workflow_permanent_id="wpid",+ workflow_run_id="run-id",+ aws_client=AsyncMock(),+ )+ download_directory = tmp_path / "workflow-downloads"+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))+ monkeypatch.setattr(+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",+ lambda _run_id: download_directory,+ )++ original_open = Path.open++ class PartiallyWrittenFile:+ def __init__(self, parquet_file: Any) -> None:+ self.parquet_file = parquet_file++ def __enter__(self) -> Self:+ return self++ def __exit__(self, *_args: object) -> None:+ self.parquet_file.close()++ def write(self, data: bytes) -> int:+ self.parquet_file.write(data[:1])+ raise OSError("disk full")++ def fail_after_partial_write(path: Path, mode: str = "r", **kwargs: object) -> PartiallyWrittenFile:+ return PartiallyWrittenFile(original_open(path, mode, **kwargs))++ monkeypatch.setattr(Path, "open", fail_after_partial_write)++ result = await _block().execute("run-id", "block-id")++ assert result.success is False+ assert result.status is BlockStatus.failed+ assert result.failure_reason == "failed to write export.parquet: disk full"+ assert not (download_directory / "export.parquet").exists()diff --git a/tests/unit/test_mcp_block_tools.py b/tests/unit/test_mcp_block_tools.pyindex ef96fb5bf..124ab446a 100644--- a/tests/unit/test_mcp_block_tools.py+++ b/tests/unit/test_mcp_block_tools.py@@ -222,3 +222,26 @@ def test_block_schema_docstring_routes_full_definitions_to_block_validate() -> Ndef test_block_validate_docstring_cross_refs_block_schema() -> None:doc = skyvern_block_validate.__doc__ or ""assert "skyvern_block_schema" in doc+++async def test_block_validate_data_export() -> None:+ result = await skyvern_block_validate(+ block_json=json.dumps(+ {+ "block_type": "data_export",+ "label": "export_records",+ "data": "{{ extract_output.extracted_information }}",+ "data_schema": {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {"id": {"type": "integer"}},+ },+ },+ }+ )+ )++ assert result["ok"] is True+ assert result["data"]["block_type"] == "data_export"diff --git a/tests/unit/test_mcp_block_type_parity.py b/tests/unit/test_mcp_block_type_parity.pyindex 7f54a23ad..db591b111 100644--- a/tests/unit/test_mcp_block_type_parity.py+++ b/tests/unit/test_mcp_block_type_parity.py@@ -95,9 +95,13 @@ class BlockDriftEntry:# the vendored Fern SDK. Run `fern generate` using the Skyvern Cloud-only# runbook to resync, then remove the entry here._KNOWN_DRIFT_ALLOWLIST: tuple[BlockDriftEntry, ...] = (- # Empty: pdf_fill (SKY-10944), split_pdf (SKY-12071), and email_inbox- # (SKY-12068) were resolved by the Fern SDK regen in the 1.0.47 bump — all- # three now appear in both WorkflowDefinition(Yaml)BlocksItem unions.+ BlockDriftEntry(+ block_type="data_export",+ ticket="SKY-14102",+ owner="@wintonzheng",+ added_at=date(2026, 8, 17),+ notes="Parquet data export awaits the next Fern SDK regeneration.",+ ),)diff --git a/tests/unit/test_mcp_workflow_tools.py b/tests/unit/test_mcp_workflow_tools.pyindex 5f3026851..010fc63ca 100644--- a/tests/unit/test_mcp_workflow_tools.py+++ b/tests/unit/test_mcp_workflow_tools.py@@ -189,6 +189,12 @@ def _known_drift_definition(block_type: str) -> dict[str, object]:block["file_url"] = "{{ source_pdf }}"block["prompt"] = "Fill the PDF using the payload."block["payload"] = {"name": "{{ applicant.name }}"}+ elif block_type == "data_export":+ block["data"] = "{{ extract_output.extracted_information }}"+ block["data_schema"] = {+ "type": "array",+ "items": {"type": "object", "properties": {"id": {"type": "integer"}}},+ }else:raise ValueError(f"Unsupported known drift block type: {block_type}")@@ -251,7 +257,7 @@ def _heavy_workflow_run_payload(*, include_expanded_outputs: bool = True) -> dic}[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill"])[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill", "data_export"])@pytest.mark.asyncioasync def test_workflow_create_sends_known_drift_json_definition_as_raw_dict(monkeypatch: pytest.MonkeyPatch, block_type: str@@ -269,11 +275,13 @@ async def test_workflow_create_sends_known_drift_json_definition_as_raw_dict(assert sent_block["block_type"] == block_typeif block_type.startswith("google_sheets"):assert sent_block["spreadsheet_url"] == "https://docs.google.com/spreadsheets/d/SPREADSHEET_ID/edit"+ elif block_type == "data_export":+ assert sent_block["data_schema"]["items"]["properties"]["id"]["type"] == "integer"else:assert sent_block["file_url"] == "{{ source_pdf }}"[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill"])[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill", "data_export"])@pytest.mark.asyncioasync def test_workflow_update_sends_known_drift_json_definition_as_raw_dict(monkeypatch: pytest.MonkeyPatch, block_type: str@@ -308,6 +316,8 @@ async def test_workflow_update_sends_known_drift_json_definition_as_raw_dict(assert sent_block["block_type"] == block_typeif block_type.startswith("google_sheets"):assert sent_block["spreadsheet_url"] == "https://docs.google.com/spreadsheets/d/SPREADSHEET_ID/edit"+ elif block_type == "data_export":+ assert sent_block["data_schema"]["items"]["properties"]["id"]["type"] == "integer"else:assert sent_block["file_url"] == "{{ source_pdf }}"diff --git a/tests/unit/test_parquet_export.py b/tests/unit/test_parquet_export.pynew file mode 100644index 000000000..68979aa64--- /dev/null+++ b/tests/unit/test_parquet_export.py@@ -0,0 +1,175 @@+from __future__ import annotations++import json++import pyarrow as pa+import pyarrow.parquet as pq+import pytest++from skyvern.utils.parquet_export import ParquetExportError, export_parquet_records++_RECORD_SCHEMA = {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "id": {"type": "integer"},+ "label": {"type": "string"},+ "price": {"type": "number"},+ "active": {"type": "boolean"},+ "tags": {"type": "array", "items": {"type": "string"}},+ "meta": {+ "type": "object",+ "properties": {+ "source": {"type": "string"},+ "rank": {"type": "integer"},+ },+ },+ },+ },+}+++def test_parquet_round_trips_declared_types_and_ragged_records() -> None:+ payload = export_parquet_records(+ [+ {+ "id": 1,+ "label": "first",+ "price": 3.5,+ "active": True,+ "tags": ["kept"],+ "meta": {"source": "page", "rank": 2},+ },+ {"id": 2, "tags": []},+ {+ "id": "not-an-integer",+ "label": 3,+ "price": "not-a-number",+ "active": "yes",+ "tags": ["kept", 7],+ "meta": {"source": 9},+ },+ ],+ _RECORD_SCHEMA,+ )++ table = pq.read_table(pa.BufferReader(payload))++ assert str(table.schema.field("id").type) == "int64"+ assert str(table.schema.field("label").type) == "string"+ assert str(table.schema.field("price").type) == "double"+ assert str(table.schema.field("active").type) == "bool"+ assert str(table.schema.field("tags").type) == "list<element: string>"+ assert str(table.schema.field("meta").type) == "struct<source: string, rank: int64>"+ assert table.to_pylist() == [+ {+ "id": 1,+ "label": "first",+ "price": 3.5,+ "active": True,+ "tags": ["kept"],+ "meta": {"source": "page", "rank": 2},+ },+ {+ "id": 2,+ "label": None,+ "price": None,+ "active": None,+ "tags": [],+ "meta": None,+ },+ {+ "id": None,+ "label": None,+ "price": None,+ "active": None,+ "tags": ["kept", None],+ "meta": {"source": None, "rank": None},+ },+ ]+ assert json.loads(table.schema.metadata[b"skyvern.invalid_value_counts"]) == {+ "active": 1,+ "id": 1,+ "label": 1,+ "meta.source": 1,+ "price": 1,+ "tags[]": 1,+ }+++def test_empty_export_keeps_the_declared_parquet_schema() -> None:+ payload = export_parquet_records([], _RECORD_SCHEMA)++ table = pq.read_table(pa.BufferReader(payload))++ assert table.num_rows == 0+ assert table.schema.names == ["id", "label", "price", "active", "tags", "meta"]+++def test_parquet_counts_each_malformed_record_once() -> None:+ payload = export_parquet_records([None, "not-a-record"], _RECORD_SCHEMA)++ table = pq.read_table(pa.BufferReader(payload))++ assert table.to_pylist() == [+ {"id": None, "label": None, "price": None, "active": None, "tags": None, "meta": None},+ {"id": None, "label": None, "price": None, "active": None, "tags": None, "meta": None},+ ]+ assert json.loads(table.schema.metadata[b"skyvern.invalid_value_counts"]) == {"$[]": 2}+++def test_parquet_coerces_overflowing_numbers_and_datetimes_to_null() -> None:+ payload = export_parquet_records(+ [{"number": 10**400, "timestamp": "9999-12-31T23:59:59-23:59"}],+ {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "number": {"type": "number"},+ "timestamp": {"type": "string", "format": "date-time"},+ },+ },+ },+ )++ table = pq.read_table(pa.BufferReader(payload))++ assert table.to_pylist() == [{"number": None, "timestamp": None}]+ assert json.loads(table.schema.metadata[b"skyvern.invalid_value_counts"]) == {+ "number": 1,+ "timestamp": 1,+ }+++def test_parquet_rejects_schemas_without_columns() -> None:+ with pytest.raises(ParquetExportError, match="must declare at least one property"):+ export_parquet_records(+ [{}],+ {"type": "array", "items": {"type": "object", "properties": {}}},+ )+++ "data_schema",+ [+ {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {"meta": {"type": "object", "properties": {}}},+ },+ },+ {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {"meta": {"type": "array", "items": {"type": "object", "properties": {}}}},+ },+ },+ ],+)+def test_parquet_rejects_nested_empty_object_schemas(data_schema: dict[str, object]) -> None:+ with pytest.raises(ParquetExportError, match="must declare at least one property"):+ export_parquet_records([{"meta": {}}], data_schema)diff --git a/uv.lock b/uv.lockindex 675e59e44..c24619d20 100644--- a/uv.lock+++ b/uv.lock@@ -5448,6 +5448,56 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/9b/bf/7595e817906a29453ba4d99394e781b6fabe55d21f3c15d240f85dd06bb1/py_serializable-2.1.0-py3-none-any.whl", hash = "sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304", size = 23045, upload-time = "2025-07-21T09:56:46.848Z" },]+[[package]]+name = "pyarrow"+version = "23.0.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" },+ { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" },+ { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" },+ { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" },+ { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" },+ { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" },+ { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" },+ { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" },+ { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" },+ { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" },+ { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" },+ { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" },+ { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" },+ { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" },+ { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" },+ { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" },+ { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" },+ { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" },+ { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" },+ { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" },+ { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" },+ { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" },+ { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" },+ { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" },+ { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" },+ { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" },+ { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" },+ { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" },+ { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" },+ { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" },+ { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" },+ { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" },+ { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" },+ { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" },+ { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" },+ { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" },+ { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" },+ { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" },+ { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" },+ { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" },+ { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" },+ { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" },+]+[[package]]name = "pyasn1"version = "0.6.4"@@ -6849,6 +6899,7 @@ local = [{ name = "playwright", version = "1.46.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },{ name = "playwright", version = "1.58.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },{ name = "psutil" },+ { name = "pyarrow" },{ name = "pyasn1" },{ name = "pyjwt", extra = ["crypto"] },{ name = "pyotp" },@@ -6919,6 +6970,7 @@ server = [{ name = "posthog" },{ name = "psutil" },{ name = "psycopg", extra = ["binary", "pool"] },+ { name = "pyarrow" },{ name = "pyasn1" },{ name = "pyjwt", extra = ["crypto"] },{ name = "pyotp" },@@ -7101,6 +7153,8 @@ requires-dist = [{ name = "psutil", marker = "extra == 'local'", specifier = ">=7.0.0" },{ name = "psutil", marker = "extra == 'server'", specifier = ">=7.0.0" },{ name = "psycopg", extras = ["binary", "pool"], marker = "extra == 'server'", specifier = ">=3.2.10,<3.3.0" },+ { name = "pyarrow", marker = "extra == 'local'", specifier = ">=18.0.0,<24" },+ { name = "pyarrow", marker = "extra == 'server'", specifier = ">=18.0.0,<24" },{ name = "pyasn1", marker = "extra == 'local'", specifier = ">=0.6.4" },{ name = "pyasn1", marker = "extra == 'server'", specifier = ">=0.6.4" },{ name = "pydantic", specifier = ">=2.13.4,<3" },