Skip to content

Automate browser based workflows with AI

Add schema-directed Parquet workflow exports (#8368)

Aaron Perez1f0454a8d47b
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index a41eda243..771960a26 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -9770,7 +9770,8 @@
"google_sheets_write",
"pdf_fill",
"split_pdf",
- "email_inbox"
+ "email_inbox",
+ "data_export"
],
"title": "BlockType"
},
@@ -12508,6 +12509,249 @@
],
"title": "CustomLLMUpdateRequest"
},
+ "DataExportBlock": {
+ "properties": {
+ "label": {
+ "type": "string",
+ "title": "Label",
+ "description": "Author-facing identifier for a block; unique within a workflow."
+ },
+ "next_block_label": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Next Block Label",
+ "description": "Optional pointer to the next block label when constructing a DAG. Defaults to sequential order when omitted."
+ },
+ "block_type": {
+ "type": "string",
+ "const": "data_export",
+ "title": "Block Type",
+ "default": "data_export"
+ },
+ "output_parameter": {
+ "$ref": "#/components/schemas/OutputParameter"
+ },
+ "continue_on_failure": {
+ "type": "boolean",
+ "title": "Continue On Failure",
+ "default": false
+ },
+ "model": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Model"
+ },
+ "disable_cache": {
+ "type": "boolean",
+ "title": "Disable Cache",
+ "default": false
+ },
+ "ignore_workflow_system_prompt": {
+ "type": "boolean",
+ "title": "Ignore Workflow System Prompt",
+ "default": false
+ },
+ "next_loop_on_failure": {
+ "type": "boolean",
+ "title": "Next Loop On Failure",
+ "default": false
+ },
+ "data": {
+ "type": "string",
+ "title": "Data"
+ },
+ "data_schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Data Schema"
+ },
+ "file_name": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "File Name"
+ },
+ "parameters": {
+ "items": {
+ "oneOf": [
+ {
+ "$ref": "#/components/schemas/WorkflowParameter"
+ },
+ {
+ "$ref": "#/components/schemas/ContextParameter"
+ },
+ {
+ "$ref": "#/components/schemas/AWSSecretParameter"
+ },
+ {
+ "$ref": "#/components/schemas/AzureSecretParameter"
+ },
+ {
+ "$ref": "#/components/schemas/BitwardenLoginCredentialParameter"
+ },
+ {
+ "$ref": "#/components/schemas/BitwardenSensitiveInformationParameter"
+ },
+ {
+ "$ref": "#/components/schemas/BitwardenCreditCardDataParameter"
+ },
+ {
+ "$ref": "#/components/schemas/OnePasswordCredentialParameter"
+ },
+ {
+ "$ref": "#/components/schemas/AzureVaultCredentialParameter"
+ },
+ {
+ "$ref": "#/components/schemas/OutputParameter"
+ },
+ {
+ "$ref": "#/components/schemas/CredentialParameter"
+ }
+ ],
+ "discriminator": {
+ "propertyName": "parameter_type",
+ "mapping": {
+ "aws_secret": "#/components/schemas/AWSSecretParameter",
+ "azure_secret": "#/components/schemas/AzureSecretParameter",
+ "azure_vault_credential": "#/components/schemas/AzureVaultCredentialParameter",
+ "bitwarden_credit_card_data": "#/components/schemas/BitwardenCreditCardDataParameter",
+ "bitwarden_login_credential": "#/components/schemas/BitwardenLoginCredentialParameter",
+ "bitwarden_sensitive_information": "#/components/schemas/BitwardenSensitiveInformationParameter",
+ "context": "#/components/schemas/ContextParameter",
+ "credential": "#/components/schemas/CredentialParameter",
+ "onepassword": "#/components/schemas/OnePasswordCredentialParameter",
+ "output": "#/components/schemas/OutputParameter",
+ "workflow": "#/components/schemas/WorkflowParameter"
+ }
+ }
+ },
+ "type": "array",
+ "title": "Parameters",
+ "default": []
+ }
+ },
+ "type": "object",
+ "required": [
+ "label",
+ "output_parameter",
+ "data",
+ "data_schema"
+ ],
+ "title": "DataExportBlock"
+ },
+ "DataExportBlockYAML": {
+ "properties": {
+ "block_type": {
+ "type": "string",
+ "const": "data_export",
+ "title": "Block Type",
+ "default": "data_export"
+ },
+ "label": {
+ "type": "string",
+ "title": "Label",
+ "description": "Author-facing identifier; must be unique per workflow."
+ },
+ "next_block_label": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Next Block Label",
+ "description": "Optional pointer to the label of the next block. When omitted, it will default to sequential order. See [[s-4bnl]]."
+ },
+ "continue_on_failure": {
+ "type": "boolean",
+ "title": "Continue On Failure",
+ "default": false
+ },
+ "model": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Model"
+ },
+ "ignore_workflow_system_prompt": {
+ "type": "boolean",
+ "title": "Ignore Workflow System Prompt",
+ "default": false
+ },
+ "next_loop_on_failure": {
+ "type": "boolean",
+ "title": "Next Loop On Failure",
+ "default": false
+ },
+ "data": {
+ "type": "string",
+ "title": "Data"
+ },
+ "data_schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Data Schema"
+ },
+ "file_name": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "File Name"
+ },
+ "parameter_keys": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Parameter Keys"
+ }
+ },
+ "type": "object",
+ "required": [
+ "label",
+ "data",
+ "data_schema"
+ ],
+ "title": "DataExportBlockYAML"
+ },
"DeleteScheduleResponse": {
"properties": {
"ok": {
@@ -15964,6 +16208,9 @@
},
{
"$ref": "#/components/schemas/SplitPdfBlock"
+ },
+ {
+ "$ref": "#/components/schemas/DataExportBlock"
}
],
"discriminator": {
@@ -15972,6 +16219,7 @@
"action": "#/components/schemas/ActionBlock",
"code": "#/components/schemas/CodeBlock",
"conditional": "#/components/schemas/ConditionalBlock",
+ "data_export": "#/components/schemas/DataExportBlock",
"download_to_s3": "#/components/schemas/DownloadToS3Block",
"email_inbox": "#/components/schemas/EmailInboxBlock",
"extraction": "#/components/schemas/ExtractionBlock",
@@ -16215,6 +16463,9 @@
{
"$ref": "#/components/schemas/FileDownloadBlockYAML"
},
+ {
+ "$ref": "#/components/schemas/DataExportBlockYAML"
+ },
{
"$ref": "#/components/schemas/UrlBlockYAML"
},
@@ -27061,6 +27312,9 @@
},
{
"$ref": "#/components/schemas/SplitPdfBlock"
+ },
+ {
+ "$ref": "#/components/schemas/DataExportBlock"
}
],
"discriminator": {
@@ -27069,6 +27323,7 @@
"action": "#/components/schemas/ActionBlock",
"code": "#/components/schemas/CodeBlock",
"conditional": "#/components/schemas/ConditionalBlock",
+ "data_export": "#/components/schemas/DataExportBlock",
"download_to_s3": "#/components/schemas/DownloadToS3Block",
"email_inbox": "#/components/schemas/EmailInboxBlock",
"extraction": "#/components/schemas/ExtractionBlock",
@@ -27240,6 +27495,9 @@
{
"$ref": "#/components/schemas/FileDownloadBlockYAML"
},
+ {
+ "$ref": "#/components/schemas/DataExportBlockYAML"
+ },
{
"$ref": "#/components/schemas/UrlBlockYAML"
},
@@ -28099,6 +28357,9 @@
},
{
"$ref": "#/components/schemas/SplitPdfBlock"
+ },
+ {
+ "$ref": "#/components/schemas/DataExportBlock"
}
],
"discriminator": {
@@ -28107,6 +28368,7 @@
"action": "#/components/schemas/ActionBlock",
"code": "#/components/schemas/CodeBlock",
"conditional": "#/components/schemas/ConditionalBlock",
+ "data_export": "#/components/schemas/DataExportBlock",
"download_to_s3": "#/components/schemas/DownloadToS3Block",
"email_inbox": "#/components/schemas/EmailInboxBlock",
"extraction": "#/components/schemas/ExtractionBlock",
@@ -28320,6 +28582,9 @@
{
"$ref": "#/components/schemas/FileDownloadBlockYAML"
},
+ {
+ "$ref": "#/components/schemas/DataExportBlockYAML"
+ },
{
"$ref": "#/components/schemas/UrlBlockYAML"
},
@@ -28363,6 +28628,7 @@
"action": "#/components/schemas/ActionBlockYAML",
"code": "#/components/schemas/CodeBlockYAML",
"conditional": "#/components/schemas/ConditionalBlockYAML",
+ "data_export": "#/components/schemas/DataExportBlockYAML",
"download_to_s3": "#/components/schemas/DownloadToS3BlockYAML",
"email_inbox": "#/components/schemas/EmailInboxBlockYAML",
"extraction": "#/components/schemas/ExtractionBlockYAML",
diff --git a/pyproject.toml b/pyproject.toml
index 8d6c9b3f8..f2374bfb6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,6 +70,7 @@ local = [
"libcst>=1.9.0,<2",
"curlparser>=0.1.0,<0.2",
"pandas>=3.0.5,<4",
+ "pyarrow>=18.0.0,<24",
"azure-identity>=1.24.0,<2",
"azure-keyvault-secrets>=4.11.1,<5",
"google-cloud-secret-manager>=2.30.0,<3",
@@ -151,6 +152,7 @@ server = [
"libcst>=1.9.0,<2",
"curlparser>=0.1.0,<0.2",
"pandas>=3.0.5,<4",
+ "pyarrow>=18.0.0,<24",
"azure-identity>=1.24.0,<2",
"azure-keyvault-secrets>=4.11.1,<5",
"google-cloud-secret-manager>=2.30.0,<3",
diff --git a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.ts b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.ts
index bfab35668..55b01b2cf 100644
--- a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.test.ts
@@ -22,4 +22,10 @@ describe("collapsibleBlockTypes — loop registration", () => {
test("toWorkflowBlockType translates loop -> for_loop", () => {
expect(toWorkflowBlockType("loop")).toBe("for_loop");
});
+
+ test("data export is collapsible", () => {
+ expect(collapsibleRfNodeTypes.has("dataExport")).toBe(true);
+ expect(collapsibleWorkflowBlockTypes.has("data_export")).toBe(true);
+ expect(toWorkflowBlockType("dataExport")).toBe("data_export");
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.ts b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.ts
index 6a9ebab72..e8b6b449c 100644
--- a/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/collapse/collapsibleBlockTypes.ts
@@ -32,6 +32,7 @@ const COLLAPSIBLE_NODE_TYPE_TO_BLOCK_TYPE: Record<string, WorkflowBlockType> = {
googleSheetsWrite: "google_sheets_write",
pdfFill: "pdf_fill",
splitPdf: "split_pdf",
+ dataExport: "data_export",
loop: "for_loop",
conditional: "conditional",
};
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportEditor.tsx
new file mode 100644
index 000000000..2a8464eaf
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportEditor.tsx
@@ -0,0 +1,88 @@
+import { useEdges, useNodes, useNodesData } from "@xyflow/react";
+
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Textarea } from "@/components/ui/textarea";
+import { WorkflowDataSchemaInputGroup } from "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup";
+
+import { type AppNode } from "..";
+import { getAvailableOutputParameterKeys } from "../../workflowEditorUtils";
+import { useUpdate } from "../../useUpdate";
+import { ParametersMultiSelect } from "../TaskNode/ParametersMultiSelect";
+import { type DataExportNode, type DataExportNodeData } from "./types";
+
+const dataSchemaExample = {
+ type: "array",
+ items: {
+ type: "object",
+ properties: { value: { type: "string" } },
+ },
+};
+
+function DataExportEditor({ blockId }: { blockId: string }) {
+ const nodeSlice = useNodesData<DataExportNode>(blockId);
+ if (!nodeSlice || nodeSlice.type !== "dataExport") {
+ return null;
+ }
+ return <DataExportEditorBody blockId={blockId} data={nodeSlice.data} />;
+}
+
+function DataExportEditorBody({
+ blockId,
+ data,
+}: {
+ blockId: string;
+ data: DataExportNodeData;
+}) {
+ const nodes = useNodes<AppNode>();
+ const edges = useEdges();
+ const update = useUpdate<DataExportNodeData>({
+ id: blockId,
+ editable: data.editable,
+ });
+ const availableOutputParameters = getAvailableOutputParameterKeys(
+ nodes,
+ edges,
+ blockId,
+ );
+
+ return (
+ <div data-testid="data-export-block-form" className="space-y-4 p-4">
+ <div className="space-y-2">
+ <Label className="text-xs text-tertiary-foreground">Records</Label>
+ <Textarea
+ className="nopan min-h-24 font-mono text-xs"
+ disabled={!data.editable}
+ onChange={(event) => update({ data: event.target.value })}
+ placeholder="{{ extraction_output.extracted_information }}"
+ value={data.data}
+ />
+ </div>
+ <WorkflowDataSchemaInputGroup
+ exampleValue={dataSchemaExample}
+ onChange={(dataSchema) => update({ dataSchema })}
+ suggestionContext={{ current_schema: data.dataSchema }}
+ value={data.dataSchema}
+ />
+ <div className="space-y-2">
+ <Label className="text-xs text-tertiary-foreground">
+ Filename (optional)
+ </Label>
+ <Input
+ className="nopan text-xs"
+ disabled={!data.editable}
+ onChange={(event) => update({ fileName: event.target.value })}
+ placeholder="records"
+ value={data.fileName}
+ />
+ </div>
+ <ParametersMultiSelect
+ availableOutputParameters={availableOutputParameters}
+ onParametersChange={(parameterKeys) => update({ parameterKeys })}
+ parameters={data.parameterKeys}
+ />
+ </div>
+ );
+}
+
+export { DataExportEditor };
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportNode.tsx
new file mode 100644
index 000000000..491a82dff
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/DataExportNode.tsx
@@ -0,0 +1,71 @@
+import { Handle, NodeProps, Position } from "@xyflow/react";
+import { useParams } from "react-router-dom";
+
+import { statusIsRunningOrQueued } from "@/routes/tasks/types";
+import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
+import { useRecordingStore } from "@/store/useRecordingStore";
+import { cn } from "@/util/utils";
+
+import { useCollapseContext } from "../../collapse/CollapseContext";
+import { NodeBody } from "../../collapse/NodeBody";
+import { BuildModeOnly } from "../BuildModeOnly";
+import { NodeHeader } from "../components/NodeHeader";
+import { DataExportEditor } from "./DataExportEditor";
+import type { DataExportNode } from "./types";
+
+function DataExportNode({ id, data }: NodeProps<DataExportNode>) {
+ const { editable, label } = data;
+ const { blockLabel: urlBlockLabel } = useParams();
+ const { data: workflowRun } = useWorkflowRunQuery();
+ const recordingStore = useRecordingStore();
+ const { open } = useCollapseContext();
+ const isTargeted = urlBlockLabel !== undefined && urlBlockLabel === label;
+ const isRunning = workflowRun && statusIsRunningOrQueued(workflowRun);
+
+ return (
+ <div
+ className={cn({
+ "pointer-events-none opacity-50": recordingStore.isRecording,
+ })}
+ >
+ <Handle
+ type="source"
+ position={Position.Bottom}
+ id="a"
+ className="opacity-0"
+ />
+ <Handle
+ type="target"
+ position={Position.Top}
+ id="b"
+ className="opacity-0"
+ />
+ <div
+ className={cn(
+ "transform-origin-center w-[30rem] space-y-4 rounded-lg bg-slate-elevation3 px-6 py-4 transition-shadow motion-reduce:transition-none",
+ open ? "shadow-md" : "shadow-sm",
+ {
+ "pointer-events-none": isRunning && isTargeted,
+ "bg-background outline outline-2 outline-ring": isTargeted,
+ },
+ )}
+ >
+ <NodeHeader
+ blockLabel={label}
+ editable={editable}
+ nodeId={id}
+ totpIdentifier={null}
+ totpUrl={null}
+ type="data_export"
+ />
+ <NodeBody>
+ <BuildModeOnly>
+ <DataExportEditor blockId={id} />
+ </BuildModeOnly>
+ </NodeBody>
+ </div>
+ </div>
+ );
+}
+
+export { DataExportNode };
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/types.ts
new file mode 100644
index 000000000..1c40c161f
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/DataExportNode/types.ts
@@ -0,0 +1,36 @@
+import type { Node } from "@xyflow/react";
+
+import { debuggableWorkflowBlockTypes } from "@/routes/workflows/types/workflowTypes";
+
+import { NodeBaseData } from "../types";
+
+export type DataExportNodeData = NodeBaseData & {
+ data: string;
+ dataSchema: string;
+ fileName: string;
+ parameterKeys: Array<string>;
+};
+
+export type DataExportNode = Node<DataExportNodeData, "dataExport">;
+
+export const dataExportNodeDefaultData: DataExportNodeData = {
+ debuggable: debuggableWorkflowBlockTypes.has("data_export"),
+ editable: true,
+ label: "",
+ data: "[]",
+ dataSchema: JSON.stringify(
+ {
+ type: "array",
+ items: {
+ type: "object",
+ properties: { value: { type: "string" } },
+ },
+ },
+ null,
+ 2,
+ ),
+ fileName: "",
+ parameterKeys: [],
+ continueOnFailure: false,
+ model: null,
+} as const;
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsx
index ec9f64e6f..f2fbd470d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/WorkflowBlockIcon.tsx
@@ -97,6 +97,9 @@ function WorkflowBlockIcon({ workflowBlockType, className }: Props) {
case "split_pdf": {
return <FileTextIcon className={className} />;
}
+ case "data_export": {
+ return <FileTextIcon className={className} />;
+ }
case "goto_url": {
return <ExternalLinkIcon className={className} />;
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts
index 92f45d7b8..7fc079638 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts
@@ -62,6 +62,8 @@ import { PdfFillNode } from "./PdfFillNode/types";
import { PdfFillNode as PdfFillNodeComponent } from "./PdfFillNode/PdfFillNode";
import { SplitPdfNode } from "./SplitPdfNode/types";
import { SplitPdfNode as SplitPdfNodeComponent } from "./SplitPdfNode/SplitPdfNode";
+import { DataExportNode } from "./DataExportNode/types";
+import { DataExportNode as DataExportNodeComponent } from "./DataExportNode/DataExportNode";
import { withSortableBlock } from "../sortable/withSortableBlock";
import { withCollapsible } from "../collapse/withCollapsible";
import { withSelectableBlock } from "../selection/withSelectableBlock";
@@ -97,7 +99,8 @@ export type WorkflowBlockNode =
| GoogleSheetsReadNode
| GoogleSheetsWriteNode
| PdfFillNode
- | SplitPdfNode;
+ | SplitPdfNode
+ | DataExportNode;
export function isUtilityNode(node: AppNode): node is UtilityNode {
return node.type === "nodeAdder" || node.type === "start";
@@ -162,4 +165,5 @@ export const nodeTypes = {
googleSheetsWrite: wrapBlock(GoogleSheetsWriteNodeComponent),
pdfFill: wrapBlock(PdfFillNodeComponent),
splitPdf: wrapBlock(SplitPdfNodeComponent),
+ dataExport: wrapBlock(DataExportNodeComponent),
} as const;
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/types.ts
index 303439d45..d59b9f530 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/types.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/types.ts
@@ -101,4 +101,5 @@ export const workflowBlockTitle: {
google_sheets_write: "Google Sheets Write",
pdf_fill: "PDF Fill",
split_pdf: "Split PDF",
+ data_export: "Data Export",
};
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsx
index a26a11a6f..35853cd9d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/wrap.test.tsx
@@ -45,6 +45,7 @@ describe("nodeTypes composition order", () => {
"googleSheetsWrite",
"pdfFill",
"splitPdf",
+ "dataExport",
"emailInbox",
]),
);
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsx
index 35a2d5e43..34b8ed9b9 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.test.tsx
@@ -84,7 +84,7 @@ describe("BLOCK_FORMS dispatcher", () => {
}
});
- test("contains the 29 expected node.type keys", () => {
+ test("contains the 30 expected node.type keys", () => {
expect(new Set(BLOCK_FORM_KEYS)).toEqual(
new Set([
"task",
@@ -115,10 +115,11 @@ describe("BLOCK_FORMS dispatcher", () => {
"googleSheetsWrite",
"pdfFill",
"splitPdf",
+ "dataExport",
"emailInbox",
]),
);
- expect(BLOCK_FORM_KEYS).toHaveLength(29);
+ expect(BLOCK_FORM_KEYS).toHaveLength(30);
});
test("conditional routes to a sidebar form that shows branch prompts", () => {
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsx
index 04d49a462..f2ad12883 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm.tsx
@@ -8,6 +8,7 @@ import { ActionBlockForm } from "./BlockConfigForm/ActionBlockForm";
import { CodeBlockBlockForm } from "./BlockConfigForm/CodeBlockBlockForm";
import { ConditionalBlockForm } from "./BlockConfigForm/ConditionalBlockForm";
import { DownloadBlockForm } from "./BlockConfigForm/DownloadBlockForm";
+import { DataExportBlockForm } from "./BlockConfigForm/DataExportBlockForm";
import { EmailInboxBlockForm } from "./BlockConfigForm/EmailInboxBlockForm";
import { ExtractionBlockForm } from "./BlockConfigForm/ExtractionBlockForm";
import { FileDownloadBlockForm } from "./BlockConfigForm/FileDownloadBlockForm";
@@ -69,6 +70,7 @@ const BLOCK_FORMS: Record<WorkflowBlockNodeType, BlockFormComponent> = {
googleSheetsRead: GoogleSheetsReadBlockForm,
googleSheetsWrite: GoogleSheetsWriteBlockForm,
splitPdf: SplitPdfBlockForm,
+ dataExport: DataExportBlockForm,
};
function BlockConfigForm({ blockId }: Readonly<{ blockId: string }>) {
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/DataExportBlockForm.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/DataExportBlockForm.tsx
new file mode 100644
index 000000000..921949763
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/DataExportBlockForm.tsx
@@ -0,0 +1,51 @@
+import { useReactFlow } from "@xyflow/react";
+import { useEffect, useMemo } from "react";
+
+import { usePendingCommitsStore } from "@/store/PendingCommitsStore";
+
+import { type AppNode, isWorkflowBlockNode } from "../../nodes";
+import { DataExportEditor } from "../../nodes/DataExportNode/DataExportEditor";
+import type { DataExportNode } from "../../nodes/DataExportNode/types";
+import { useDebouncedSidebarSave } from "../useDebouncedSidebarSave";
+
+function DataExportBlockForm({ blockId }: { blockId: string }) {
+ const reactFlow = useReactFlow<AppNode>();
+ const node = reactFlow.getNode(blockId);
+ if (!node || !isWorkflowBlockNode(node) || node.type !== "dataExport") {
+ return null;
+ }
+ return <DataExportBlockFormBody blockId={blockId} node={node} />;
+}
+
+function DataExportBlockFormBody({
+ blockId,
+ node,
+}: {
+ blockId: string;
+ node: DataExportNode;
+}) {
+ const value = useMemo(
+ () => ({
+ data: node.data.data,
+ dataSchema: node.data.dataSchema,
+ fileName: node.data.fileName,
+ parameterKeys: node.data.parameterKeys,
+ }),
+ [
+ node.data.data,
+ node.data.dataSchema,
+ node.data.fileName,
+ node.data.parameterKeys,
+ ],
+ );
+ const { commit } = useDebouncedSidebarSave({ blockId, value });
+
+ useEffect(() => {
+ usePendingCommitsStore.getState().register(blockId, commit);
+ return () => usePendingCommitsStore.getState().unregister(blockId);
+ }, [blockId, commit]);
+
+ return <DataExportEditor blockId={blockId} />;
+}
+
+export { DataExportBlockForm };
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsx
index 57b49478e..933351431 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigSidebar.tsx
@@ -72,6 +72,7 @@ const NODE_TYPE_TO_BLOCK_TYPE: Record<
googleSheetsWrite: "google_sheets_write",
pdfFill: "pdf_fill",
splitPdf: "split_pdf",
+ dataExport: "data_export",
};
function getBlockTypeFromNode(node: AppNode): WorkflowBlockType | null {
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
index c3dee3e63..0ac1cffdb 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx
@@ -83,6 +83,17 @@ const nodeLibraryItems: Array<{
title: "Human Interaction Block",
description: "Pause agent for human review and approval",
},
+ {
+ nodeType: "dataExport",
+ icon: (
+ <WorkflowBlockIcon
+ workflowBlockType={WorkflowBlockTypes.DataExport}
+ className="size-6"
+ />
+ ),
+ title: "Data Export Block",
+ description: "Write schema-defined records to a Parquet file",
+ },
// {
// nodeType: "task",
// icon: (
diff --git a/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.ts b/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.ts
index baed22b7b..e20a03269 100644
--- a/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/undefinedParametersGuard.test.ts
@@ -2,7 +2,11 @@ import { describe, expect, test } from "vitest";
import { ProxyLocation, RunEngine } from "@/api/types";
-import type { WorkflowBlock, WorkflowSettings } from "../types/workflowTypes";
+import type {
+ DataExportBlock,
+ WorkflowBlock,
+ WorkflowSettings,
+} from "../types/workflowTypes";
import { getElements, getWorkflowBlocks } from "./workflowEditorUtils";
@@ -73,3 +77,44 @@ describe("engine round-trips through node data", () => {
expect(savedBlock).toMatchObject({ engine: "skyvern-3.0" });
});
});
+
+describe("data export blocks", () => {
+ test("an API-authored export block loads and saves without changing its contract", () => {
+ const block = {
+ label: "export_records",
+ block_type: "data_export",
+ continue_on_failure: false,
+ model: null,
+ next_block_label: null,
+ parameters: [],
+ data: "{{ extraction_output.extracted_information }}",
+ data_schema: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: { id: { type: "integer" } },
+ },
+ },
+ file_name: "records",
+ } as unknown as DataExportBlock;
+
+ const { nodes, edges } = getElements([block], DEFAULT_SETTINGS, true);
+ const exportNode = nodes.find(
+ (node) => node.data.label === "export_records",
+ );
+
+ expect(exportNode?.type).toBe("dataExport");
+ expect(exportNode?.data).toMatchObject({
+ data: "{{ extraction_output.extracted_information }}",
+ fileName: "records",
+ });
+ expect(getWorkflowBlocks(nodes, edges)).toEqual([
+ expect.objectContaining({
+ block_type: "data_export",
+ data: "{{ extraction_output.extracted_information }}",
+ data_schema: block.data_schema,
+ file_name: "records",
+ }),
+ ]);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index 78eac53b9..9031d97b0 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -29,6 +29,7 @@ import {
BlockYAML,
CodeBlockYAML,
ConditionalBlockYAML,
+ DataExportBlockYAML,
DownloadToS3BlockYAML,
FileUrlParserBlockYAML,
ForLoopBlockYAML,
@@ -74,6 +75,7 @@ import {
import { ParametersState } from "./types";
import { AppNode, isWorkflowBlockNode, WorkflowBlockNode } from "./nodes";
import { codeBlockNodeDefaultData } from "./nodes/CodeBlockNode/types";
+import { dataExportNodeDefaultData } from "./nodes/DataExportNode/types";
import { downloadNodeDefaultData } from "./nodes/DownloadNode/types";
import {
isFileParserNode,
@@ -1006,6 +1008,20 @@ function convertToNode(
},
};
}
+ case "data_export": {
+ return {
+ ...identifiers,
+ ...common,
+ type: "dataExport",
+ data: {
+ ...commonData,
+ data: block.data,
+ dataSchema: JSON.stringify(block.data_schema, null, 2),
+ fileName: block.file_name ?? "",
+ parameterKeys: (block.parameters ?? []).map((p) => p.key),
+ },
+ };
+ }
case "for_loop": {
const loopVariableReference =
block.loop_variable_reference !== null
@@ -2617,6 +2633,17 @@ function createNode(
},
};
}
+ case "dataExport": {
+ return {
+ ...identifiers,
+ ...common,
+ type: "dataExport",
+ data: {
+ ...dataExportNodeDefaultData,
+ label,
+ },
+ };
+ }
case "download": {
return {
...identifiers,
@@ -3278,6 +3305,16 @@ function getWorkflowBlock(
steps: node.data.steps,
};
}
+ case "dataExport": {
+ return {
+ ...base,
+ block_type: "data_export",
+ data: node.data.data,
+ data_schema: JSONParseSafe(node.data.dataSchema) ?? {},
+ file_name: node.data.fileName || null,
+ parameter_keys: node.data.parameterKeys,
+ };
+ }
case "download": {
return {
...base,
@@ -4708,6 +4745,17 @@ function convertBlocksToBlockYAML(
};
return blockYaml;
}
+ case "data_export": {
+ const blockYaml: DataExportBlockYAML = {
+ ...base,
+ block_type: "data_export",
+ data: block.data,
+ data_schema: block.data_schema,
+ file_name: block.file_name,
+ parameter_keys: (block.parameters ?? []).map((p) => p.key),
+ };
+ return blockYaml;
+ }
case "download_to_s3": {
const blockYaml: DownloadToS3BlockYAML = {
...base,
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
index 884e12d95..2899d28cd 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
@@ -211,6 +211,7 @@ export type WorkflowBlock =
| FileURLParserBlock
| ValidationBlock
| HumanInteractionBlock
+ | DataExportBlock
| ActionBlock
| NavigationBlock
| ExtractionBlock
@@ -243,6 +244,7 @@ export const WorkflowBlockTypes = {
FileURLParser: "file_url_parser",
Validation: "validation",
HumanInteraction: "human_interaction",
+ DataExport: "data_export",
Action: "action",
Navigation: "navigation",
Extraction: "extraction",
@@ -498,6 +500,14 @@ export type HumanInteractionBlock = WorkflowBlockBase & {
body: string;
};
+export type DataExportBlock = WorkflowBlockBase & {
+ block_type: "data_export";
+ data: string;
+ data_schema: Record<string, unknown>;
+ file_name: string | null;
+ parameters: Array<WorkflowParameter>;
+};
+
export type ActionBlock = WorkflowBlockBase & {
block_type: "action";
url: string | null;
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
index bd054482b..09c359bae 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
@@ -155,6 +155,7 @@ export type BlockYAML =
| ConditionalBlockYAML
| ValidationBlockYAML
| HumanInteractionBlockYAML
+ | DataExportBlockYAML
| ActionBlockYAML
| NavigationBlockYAML
| ExtractionBlockYAML
@@ -236,6 +237,14 @@ export type HumanInteractionBlockYAML = BlockYAMLBase & {
body: string;
};
+export type DataExportBlockYAML = BlockYAMLBase & {
+ block_type: "data_export";
+ data: string;
+ data_schema: Record<string, unknown>;
+ file_name: string | null;
+ parameter_keys?: Array<string> | null;
+};
+
export type ActionBlockYAML = BlockYAMLBase & {
block_type: "action";
url: string | null;
diff --git a/skyvern/cli/mcp_tools/blocks.py b/skyvern/cli/mcp_tools/blocks.py
index a078d48f8..24dc8272f 100644
--- a/skyvern/cli/mcp_tools/blocks.py
+++ b/skyvern/cli/mcp_tools/blocks.py
@@ -22,6 +22,7 @@ from skyvern.schemas.workflows import (
BlockYAML,
CodeBlockYAML,
ConditionalBlockYAML,
+ DataExportBlockYAML,
DownloadToS3BlockYAML,
EmailInboxBlockYAML,
ExtractionBlockYAML,
@@ -79,6 +80,7 @@ BLOCK_TYPE_MAP: dict[str, type[BlockYAML]] = {
BlockType.FILE_UPLOAD.value: FileUploadBlockYAML,
BlockType.GOTO_URL.value: UrlBlockYAML,
BlockType.DOWNLOAD_TO_S3.value: DownloadToS3BlockYAML,
+ BlockType.DATA_EXPORT.value: DataExportBlockYAML,
BlockType.UPLOAD_TO_S3.value: UploadToS3BlockYAML,
BlockType.FILE_URL_PARSER.value: FileParserBlockYAML,
BlockType.PDF_PARSER.value: PDFParserBlockYAML,
@@ -114,6 +116,7 @@ BLOCK_SUMMARIES: dict[str, str] = {
"file_upload": "Upload a file from S3/Azure to a page element",
"goto_url": "Navigate directly to a URL without additional instructions",
"download_to_s3": "Download a URL directly to S3 storage",
+ "data_export": "Write schema-defined workflow records to a Parquet file",
"upload_to_s3": "Upload local content to S3",
"file_url_parser": "Parse a file (CSV/Excel/PDF/image/DOCX) from a URL; ZIP archives are unzipped to a file list",
"pdf_parser": "Extract structured data from a PDF document",
@@ -157,6 +160,22 @@ BLOCK_EXAMPLES: dict[str, dict[str, Any]] = {
},
},
},
+ "data_export": {
+ "block_type": "data_export",
+ "label": "export_records",
+ "data": "{{ extract_records_output.extracted_information }}",
+ "data_schema": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "price": {"type": "number"},
+ },
+ },
+ },
+ "file_name": "records",
+ },
"for_loop": {
"block_type": "for_loop",
"label": "process_each_url",
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 5a5060802..bcaab9c97 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -15387,6 +15387,7 @@ def get_all_blocks(blocks: list[BlockTypeVar]) -> list[BlockTypeVar]:
# Late import: google_sheets_blocks imports Block from this module, so top-level import would cycle.
+from skyvern.forge.sdk.workflow.models.data_export_block import DataExportBlock # noqa: E402
from skyvern.forge.sdk.workflow.models.email_inbox_block import EmailInboxBlock # noqa: E402
from skyvern.forge.sdk.workflow.models.google_sheets_blocks import ( # noqa: E402
GoogleSheetsReadBlock,
@@ -15426,6 +15427,7 @@ BlockSubclasses = Union[
GoogleSheetsWriteBlock,
PdfFillBlock,
SplitPdfBlock,
+ DataExportBlock,
]
BlockTypeVar = Annotated[BlockSubclasses, Field(discriminator="block_type")]
diff --git a/skyvern/forge/sdk/workflow/models/data_export_block.py b/skyvern/forge/sdk/workflow/models/data_export_block.py
new file mode 100644
index 000000000..1c10b61e5
--- /dev/null
+++ b/skyvern/forge/sdk/workflow/models/data_export_block.py
@@ -0,0 +1,196 @@
+"""Workflow block that exports schema-defined records as Parquet."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any, Literal
+
+import structlog
+
+from skyvern.constants import GET_DOWNLOADED_FILES_TIMEOUT, SAVE_DOWNLOADED_FILES_TIMEOUT
+from skyvern.exceptions import DownloadSaveIncompleteError
+from skyvern.forge import app
+from skyvern.forge.sdk.api.files import get_path_for_workflow_download_directory, resolve_run_download_id
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.schemas.files import FileInfo
+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.exceptions import FailedToFormatJinjaStyleParameter
+from skyvern.forge.sdk.workflow.models._jinja import jinja_json_finalize_required_binding_env
+from skyvern.forge.sdk.workflow.models.block import Block, sanitize_filename
+from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE
+from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType
+from skyvern.utils.parquet_export import ParquetExportError, export_parquet_records
+
+LOG = structlog.get_logger()
+
+
+class DataExportBlock(Block):
+ block_type: Literal[BlockType.DATA_EXPORT] = BlockType.DATA_EXPORT # type: ignore
+
+ data: str
+ data_schema: dict[str, Any]
+ file_name: str | None = None
+ parameters: list[PARAMETER_TYPE] = []
+
+ def get_all_parameters(self, workflow_run_id: str) -> list[PARAMETER_TYPE]:
+ return self.parameters
+
+ def format_potential_template_parameters(self, workflow_run_context: WorkflowRunContext) -> None:
+ self.data = self.format_block_parameter_template_from_workflow_run_context(
+ self.data,
+ workflow_run_context,
+ env=jinja_json_finalize_required_binding_env,
+ skip_missing_variable_preflight=True,
+ )
+ if self.file_name:
+ self.file_name = self.format_block_parameter_template_from_workflow_run_context(
+ self.file_name, workflow_run_context
+ )
+
+ def _resolve_file_name(self, workflow_run_context: WorkflowRunContext) -> str:
+ stem = sanitize_filename(self.file_name or self.label)
+ if stem.lower().endswith(".parquet"):
+ stem = stem[: -len(".parquet")]
+ current_index = workflow_run_context.get_block_metadata(self.label).get("current_index")
+ if isinstance(current_index, int) and not isinstance(current_index, bool):
+ stem = f"{stem}-{current_index + 1:04d}"
+ return f"{stem}.parquet"
+
+ @staticmethod
+ def _parse_records(data: str) -> list[Any]:
+ try:
+ records = json.loads(data)
+ except json.JSONDecodeError as exc:
+ raise ParquetExportError("data must resolve to a JSON array") from exc
+ if not isinstance(records, list):
+ raise ParquetExportError("data must resolve to a JSON array of object records")
+ return records
+
+ async def _register_download(
+ self,
+ *,
+ organization_id: str | None,
+ run_download_id: str | None,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ ) -> list[FileInfo]:
+ if not organization_id:
+ return []
+ try:
+ async with asyncio.timeout(SAVE_DOWNLOADED_FILES_TIMEOUT):
+ await app.STORAGE.save_downloaded_files(organization_id=organization_id, run_id=run_download_id)
+ except asyncio.TimeoutError:
+ LOG.warning(
+ "Timeout saving Parquet export; workflow finalization will retry",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ )
+ return []
+ except DownloadSaveIncompleteError:
+ pass
+ except Exception:
+ LOG.warning(
+ "Failed to register Parquet export; workflow finalization will retry",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+ return []
+ try:
+ async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
+ return await app.STORAGE.get_downloaded_files(organization_id=organization_id, run_id=run_download_id)
+ except Exception:
+ LOG.warning(
+ "Failed to read registered Parquet exports",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+ return []
+
+ async def _failure(
+ self,
+ reason: str,
+ workflow_run_block_id: str,
+ organization_id: str | None,
+ ) -> BlockResult:
+ return await self.build_block_result(
+ success=False,
+ failure_reason=reason,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+
+ async def execute(
+ self,
+ workflow_run_id: str,
+ workflow_run_block_id: str,
+ organization_id: str | None = None,
+ browser_session_id: str | None = None,
+ **kwargs: Any,
+ ) -> BlockResult:
+ workflow_run_context = self.get_workflow_run_context(workflow_run_id)
+ try:
+ self.format_potential_template_parameters(workflow_run_context)
+ records = self._parse_records(self.data)
+ parquet_data = export_parquet_records(records, self.data_schema)
+ except (FailedToFormatJinjaStyleParameter, ParquetExportError) as exc:
+ return await self._failure(str(exc), workflow_run_block_id, organization_id)
+
+ filename = self._resolve_file_name(workflow_run_context)
+ run_download_id = resolve_run_download_id(skyvern_context.current(), fallback_run_id=workflow_run_id)
+ path = get_path_for_workflow_download_directory(run_download_id) / filename
+ file_created = False
+ try:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ stem = path.stem
+ suffix = 2
+ while True:
+ try:
+ with path.open("xb") as parquet_file:
+ file_created = True
+ parquet_file.write(parquet_data)
+ break
+ except FileExistsError:
+ path = path.with_name(f"{stem}-{suffix:04d}{path.suffix}")
+ suffix += 1
+ except OSError as exc:
+ if file_created:
+ try:
+ path.unlink()
+ except FileNotFoundError:
+ pass
+ except OSError:
+ LOG.warning("Failed to remove incomplete Parquet export", exc_info=True)
+ return await self._failure(f"failed to write {filename}: {exc}", workflow_run_block_id, organization_id)
+
+ filename = path.name
+ downloaded_files = await self._register_download(
+ organization_id=organization_id or workflow_run_context.organization_id,
+ run_download_id=run_download_id,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ )
+ downloaded_files = [file for file in downloaded_files if file.filename == filename]
+ output = {
+ "file_name": filename,
+ "file_path": str(path),
+ "file_size": path.stat().st_size,
+ "format": "parquet",
+ "compression": "snappy",
+ "row_count": len(records),
+ "columns": list(self.data_schema.get("items", {}).get("properties", {})),
+ "downloaded_files": [file.model_dump() for file in downloaded_files],
+ "downloaded_file_urls": [file.url for file in downloaded_files],
+ }
+ await self.record_output_parameter_value(workflow_run_context, workflow_run_id, output)
+ return await self.build_block_result(
+ success=True,
+ failure_reason=None,
+ output_parameter_value=output,
+ status=BlockStatus.completed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index e071cd93b..2c5a87841 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -63,6 +63,7 @@ from skyvern.forge.sdk.workflow.models.block import (
WhileLoopBlock,
WorkflowTriggerBlock,
)
+from skyvern.forge.sdk.workflow.models.data_export_block import DataExportBlock
from skyvern.forge.sdk.workflow.models.email_inbox_block import EmailInboxBlock
from skyvern.forge.sdk.workflow.models.google_sheets_blocks import (
GoogleSheetsReadBlock,
@@ -894,6 +895,15 @@ def block_yaml_to_block(
wait_sec=block_yaml.wait_sec,
)
+ elif block_yaml.block_type == BlockType.DATA_EXPORT:
+ return DataExportBlock(
+ **base_kwargs,
+ data=block_yaml.data,
+ data_schema=block_yaml.data_schema,
+ file_name=block_yaml.file_name,
+ parameters=_resolve_block_parameters(block_yaml, parameters),
+ )
+
elif block_yaml.block_type == BlockType.FILE_DOWNLOAD:
file_download_block_parameters = _resolve_block_parameters(block_yaml, parameters)
return FileDownloadBlock(
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index b06bda750..eba68c2dc 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -502,6 +502,7 @@ class BlockType(StrEnum):
PDF_FILL = "pdf_fill"
SPLIT_PDF = "split_pdf"
EMAIL_INBOX = "email_inbox"
+ DATA_EXPORT = "data_export"
class AIFallbackMode(StrEnum):
@@ -1273,6 +1274,15 @@ class HumanInteractionBlockYAML(BlockYAML):
body: str = "Your interaction is required for a workflow run!"
+class DataExportBlockYAML(BlockYAML):
+ block_type: Literal[BlockType.DATA_EXPORT] = BlockType.DATA_EXPORT # type: ignore
+
+ data: str
+ data_schema: dict[str, Any]
+ file_name: str | None = None
+ parameter_keys: list[str] | None = None
+
+
class FileDownloadBlockYAML(BlockYAML):
block_type: Literal[BlockType.FILE_DOWNLOAD] = BlockType.FILE_DOWNLOAD # type: ignore
@@ -1494,6 +1504,7 @@ BLOCK_YAML_SUBCLASSES = (
| WaitBlockYAML
| HumanInteractionBlockYAML
| FileDownloadBlockYAML
+ | DataExportBlockYAML
| UrlBlockYAML
| PDFParserBlockYAML
| TaskV2BlockYAML
diff --git a/skyvern/utils/parquet_export.py b/skyvern/utils/parquet_export.py
new file mode 100644
index 000000000..98957f60e
--- /dev/null
+++ b/skyvern/utils/parquet_export.py
@@ -0,0 +1,191 @@
+"""Schema-directed Parquet serialization for workflow extraction records."""
+
+from __future__ import annotations
+
+import io
+import json
+import math
+from collections import Counter
+from collections.abc import Mapping, Sequence
+from datetime import UTC, date, datetime
+from typing import Any
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+INVALID_VALUE_COUNTS_METADATA_KEY = b"skyvern.invalid_value_counts"
+
+
+class ParquetExportError(ValueError):
+ """The configured schema cannot describe a Parquet table."""
+
+
+def export_parquet_records(records: Any, data_schema: Mapping[str, Any]) -> bytes:
+ """Write a JSON array of object records to a schema-directed Parquet file.
+
+ Missing properties and values incompatible with their declared JSON Schema type
+ become null. Incompatible-value counts are written as Parquet metadata without
+ preserving raw extracted values.
+ """
+ item_schema = _record_schema(data_schema)
+ properties = _properties(item_schema, path="items")
+ warnings: Counter[str] = Counter()
+
+ if not isinstance(records, list):
+ raise ParquetExportError("data must resolve to a JSON array of object records")
+
+ malformed_record_count = sum(not isinstance(record, Mapping) for record in records)
+ if malformed_record_count:
+ warnings["$[]"] = malformed_record_count
+ columns = {
+ field_name: [_coerce_record_value(record, field_name, field_schema, warnings) for record in records]
+ for field_name, field_schema in properties.items()
+ }
+ fields = [
+ pa.field(name, _arrow_type(field_schema, path=name), nullable=True) for name, field_schema in properties.items()
+ ]
+ metadata = {INVALID_VALUE_COUNTS_METADATA_KEY: json.dumps(dict(sorted(warnings.items()))).encode("utf-8")}
+ table = pa.Table.from_arrays(
+ [pa.array(columns[field.name], type=field.type) for field in fields],
+ schema=pa.schema(fields, metadata=metadata),
+ )
+ sink = io.BytesIO()
+ pq.write_table(table, sink, compression="snappy")
+ return sink.getvalue()
+
+
+def _record_schema(data_schema: Mapping[str, Any]) -> Mapping[str, Any]:
+ if _schema_type(data_schema, path="data_schema") != "array":
+ raise ParquetExportError("data_schema must be a JSON Schema array")
+ items = data_schema.get("items")
+ if not isinstance(items, Mapping) or _schema_type(items, path="data_schema.items") != "object":
+ raise ParquetExportError("data_schema.items must be a JSON Schema object")
+ return items
+
+
+def _properties(schema: Mapping[str, Any], *, path: str) -> dict[str, Mapping[str, Any]]:
+ raw_properties = schema.get("properties")
+ if not isinstance(raw_properties, Mapping):
+ raise ParquetExportError(f"{path}.properties must be an object")
+ properties: dict[str, Mapping[str, Any]] = {}
+ for name, field_schema in raw_properties.items():
+ if not isinstance(name, str) or not isinstance(field_schema, Mapping):
+ raise ParquetExportError(f"{path}.properties must map field names to JSON Schema objects")
+ properties[name] = field_schema
+ if not properties:
+ raise ParquetExportError(f"{path}.properties must declare at least one property")
+ return properties
+
+
+def _schema_type(schema: Mapping[str, Any], *, path: str) -> str:
+ raw_type = schema.get("type")
+ if isinstance(raw_type, str):
+ return raw_type
+ if isinstance(raw_type, Sequence) and not isinstance(raw_type, str):
+ types = [value for value in raw_type if value != "null"]
+ if len(types) == 1 and isinstance(types[0], str):
+ return types[0]
+ raise ParquetExportError(f"{path} must declare exactly one non-null JSON Schema type")
+
+
+def _arrow_type(schema: Mapping[str, Any], *, path: str) -> pa.DataType:
+ schema_type = _schema_type(schema, path=path)
+ if schema_type == "string":
+ if schema.get("format") == "date":
+ return pa.date32()
+ if schema.get("format") == "date-time":
+ return pa.timestamp("us", tz="UTC")
+ return pa.string()
+ if schema_type == "integer":
+ return pa.int64()
+ if schema_type == "number":
+ return pa.float64()
+ if schema_type == "boolean":
+ return pa.bool_()
+ if schema_type == "object":
+ return pa.struct(
+ [
+ pa.field(name, _arrow_type(field_schema, path=f"{path}.{name}"), nullable=True)
+ for name, field_schema in _properties(schema, path=path).items()
+ ]
+ )
+ if schema_type == "array":
+ items = schema.get("items")
+ if not isinstance(items, Mapping):
+ raise ParquetExportError(f"{path}.items must be a JSON Schema object")
+ return pa.list_(_arrow_type(items, path=f"{path}[]"))
+ raise ParquetExportError(f"{path} has unsupported JSON Schema type {schema_type!r}")
+
+
+def _coerce_record_value(
+ record: Any,
+ field_name: str,
+ field_schema: Mapping[str, Any],
+ warnings: Counter[str],
+) -> Any:
+ if not isinstance(record, Mapping):
+ return None
+ if field_name not in record:
+ return None
+ return _coerce_value(record[field_name], field_schema, path=field_name, warnings=warnings)
+
+
+def _coerce_value(value: Any, schema: Mapping[str, Any], *, path: str, warnings: Counter[str]) -> Any:
+ if value is None:
+ return None
+
+ schema_type = _schema_type(schema, path=path)
+ if schema_type == "string":
+ if not isinstance(value, str):
+ return _invalid(path, warnings)
+ if schema.get("format") == "date":
+ try:
+ return date.fromisoformat(value)
+ except ValueError:
+ return _invalid(path, warnings)
+ if schema.get("format") == "date-time":
+ try:
+ parsed = datetime.fromisoformat(value)
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=UTC)
+ return parsed.astimezone(UTC)
+ except (OverflowError, ValueError):
+ return _invalid(path, warnings)
+ return value
+ if schema_type == "integer":
+ if isinstance(value, int) and not isinstance(value, bool) and -(2**63) <= value < 2**63:
+ return value
+ return _invalid(path, warnings)
+ if schema_type == "number":
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
+ try:
+ number = float(value)
+ except OverflowError:
+ return _invalid(path, warnings)
+ if math.isfinite(number):
+ return number
+ return _invalid(path, warnings)
+ if schema_type == "boolean":
+ return value if isinstance(value, bool) else _invalid(path, warnings)
+ if schema_type == "object":
+ if not isinstance(value, Mapping):
+ return _invalid(path, warnings)
+ return {
+ field_name: _coerce_value(value[field_name], field_schema, path=f"{path}.{field_name}", warnings=warnings)
+ if field_name in value
+ else None
+ for field_name, field_schema in _properties(schema, path=path).items()
+ }
+ if schema_type == "array":
+ if not isinstance(value, list):
+ return _invalid(path, warnings)
+ item_schema = schema.get("items")
+ if not isinstance(item_schema, Mapping):
+ raise ParquetExportError(f"{path}.items must be a JSON Schema object")
+ return [_coerce_value(item, item_schema, path=f"{path}[]", warnings=warnings) for item in value]
+ raise ParquetExportError(f"{path} has unsupported JSON Schema type {schema_type!r}")
+
+
+def _invalid(path: str, warnings: Counter[str]) -> None:
+ warnings[path] += 1
+ return None
diff --git a/tests/unit/test_data_export_block.py b/tests/unit/test_data_export_block.py
new file mode 100644
index 000000000..03c2ffb50
--- /dev/null
+++ b/tests/unit/test_data_export_block.py
@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Self
+from unittest.mock import AsyncMock
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.schemas.files import FileInfo
+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.models.data_export_block import DataExportBlock
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
+from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block
+from skyvern.schemas.workflows import BlockStatus, DataExportBlockYAML
+
+_RECORD_SCHEMA = {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {"type": "integer"},
+ "name": {"type": "string"},
+ },
+ },
+}
+
+
+def _output_parameter() -> OutputParameter:
+ now = datetime.now(UTC)
+ return OutputParameter(
+ output_parameter_id="export_output_id",
+ key="export_output",
+ workflow_id="workflow-id",
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def _block(**overrides: object) -> DataExportBlock:
+ return DataExportBlock(
+ **{
+ "label": "export",
+ "output_parameter": _output_parameter(),
+ "data": "[]",
+ "data_schema": _RECORD_SCHEMA,
+ **overrides,
+ }
+ )
+
+
+def test_yaml_conversion_builds_a_parquet_export_block() -> None:
+ block = block_yaml_to_block(
+ DataExportBlockYAML(
+ label="export",
+ data="{{ extract_output.extracted_information }}",
+ data_schema=_RECORD_SCHEMA,
+ ),
+ {"export_output": _output_parameter()},
+ )
+
+ assert isinstance(block, DataExportBlock)
+ assert block.data_schema == _RECORD_SCHEMA
+
+
+async def test_export_block_writes_and_registers_one_parquet_file_per_loop_iteration(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ download_directory = tmp_path / "workflow-downloads"
+ context = WorkflowRunContext(
+ workflow_title="Parquet test",
+ workflow_id="workflow-id",
+ workflow_permanent_id="wpid",
+ workflow_run_id="run-id",
+ aws_client=AsyncMock(),
+ )
+ context.organization_id = "org-id"
+ context.values["extract_output"] = {"extracted_information": [{"id": 1, "name": "one"}, {"id": 2}]}
+ context.update_block_metadata("export", {"current_index": 2})
+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "create_or_update_workflow_run_output_parameter", AsyncMock())
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",
+ lambda _run_id: download_directory,
+ )
+ monkeypatch.setattr(app.STORAGE, "save_downloaded_files", AsyncMock())
+ monkeypatch.setattr(
+ app.STORAGE,
+ "get_downloaded_files",
+ AsyncMock(
+ return_value=[FileInfo(url="https://example.test/items-0003.parquet", filename="items-0003.parquet")]
+ ),
+ )
+
+ result = await _block(
+ data="{{ extract_output.extracted_information }}",
+ file_name="items",
+ ).execute("run-id", "block-id", organization_id="org-id")
+
+ assert result.success is True
+ assert result.status is BlockStatus.completed
+ assert result.output_parameter_value == {
+ "file_name": "items-0003.parquet",
+ "file_path": str(download_directory / "items-0003.parquet"),
+ "file_size": (download_directory / "items-0003.parquet").stat().st_size,
+ "format": "parquet",
+ "compression": "snappy",
+ "row_count": 2,
+ "columns": ["id", "name"],
+ "downloaded_files": [
+ FileInfo(url="https://example.test/items-0003.parquet", filename="items-0003.parquet").model_dump()
+ ],
+ "downloaded_file_urls": ["https://example.test/items-0003.parquet"],
+ }
+ table = pq.read_table(pa.BufferReader((download_directory / "items-0003.parquet").read_bytes()))
+ assert table.to_pylist() == [{"id": 1, "name": "one"}, {"id": 2, "name": None}]
+
+
+async def test_export_block_preserves_parquet_files_with_repeated_loop_indexes(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ download_directory = tmp_path / "workflow-downloads"
+ context = WorkflowRunContext(
+ workflow_title="Parquet test",
+ workflow_id="workflow-id",
+ workflow_permanent_id="wpid",
+ workflow_run_id="run-id",
+ aws_client=AsyncMock(),
+ )
+ context.update_block_metadata("export", {"current_index": 0})
+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "create_or_update_workflow_run_output_parameter", AsyncMock())
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",
+ lambda _run_id: download_directory,
+ )
+
+ block = _block(file_name="items")
+ first_result = await block.execute("run-id", "block-id")
+ second_result = await block.execute("run-id", "block-id")
+
+ assert first_result.output_parameter_value["file_name"] == "items-0001.parquet"
+ assert second_result.output_parameter_value["file_name"] == "items-0001-0002.parquet"
+ assert sorted(path.name for path in download_directory.glob("*.parquet")) == [
+ "items-0001-0002.parquet",
+ "items-0001.parquet",
+ ]
+
+
+async def test_export_block_returns_clean_failure_when_writing_parquet_fails(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ context = WorkflowRunContext(
+ workflow_title="Parquet test",
+ workflow_id="workflow-id",
+ workflow_permanent_id="wpid",
+ workflow_run_id="run-id",
+ aws_client=AsyncMock(),
+ )
+ download_directory = tmp_path / "workflow-downloads"
+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",
+ lambda _run_id: download_directory,
+ )
+
+ def fail_write(_path: Path, _mode: str = "r", **_kwargs: object) -> object:
+ raise OSError("disk full")
+
+ monkeypatch.setattr(Path, "open", fail_write)
+
+ result = await _block().execute("run-id", "block-id")
+
+ assert download_directory.is_dir()
+ assert result.success is False
+ assert result.status is BlockStatus.failed
+ assert result.failure_reason == "failed to write export.parquet: disk full"
+
+
+async def test_export_block_removes_partial_parquet_file_after_write_failure(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ context = WorkflowRunContext(
+ workflow_title="Parquet test",
+ workflow_id="workflow-id",
+ workflow_permanent_id="wpid",
+ workflow_run_id="run-id",
+ aws_client=AsyncMock(),
+ )
+ download_directory = tmp_path / "workflow-downloads"
+ monkeypatch.setattr(DataExportBlock, "get_workflow_run_context", staticmethod(lambda _run_id: context))
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.data_export_block.get_path_for_workflow_download_directory",
+ lambda _run_id: download_directory,
+ )
+
+ original_open = Path.open
+
+ class PartiallyWrittenFile:
+ def __init__(self, parquet_file: Any) -> None:
+ self.parquet_file = parquet_file
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(self, *_args: object) -> None:
+ self.parquet_file.close()
+
+ def write(self, data: bytes) -> int:
+ self.parquet_file.write(data[:1])
+ raise OSError("disk full")
+
+ def fail_after_partial_write(path: Path, mode: str = "r", **kwargs: object) -> PartiallyWrittenFile:
+ return PartiallyWrittenFile(original_open(path, mode, **kwargs))
+
+ monkeypatch.setattr(Path, "open", fail_after_partial_write)
+
+ result = await _block().execute("run-id", "block-id")
+
+ assert result.success is False
+ assert result.status is BlockStatus.failed
+ assert result.failure_reason == "failed to write export.parquet: disk full"
+ assert not (download_directory / "export.parquet").exists()
diff --git a/tests/unit/test_mcp_block_tools.py b/tests/unit/test_mcp_block_tools.py
index ef96fb5bf..124ab446a 100644
--- a/tests/unit/test_mcp_block_tools.py
+++ b/tests/unit/test_mcp_block_tools.py
@@ -222,3 +222,26 @@ def test_block_schema_docstring_routes_full_definitions_to_block_validate() -> N
def test_block_validate_docstring_cross_refs_block_schema() -> None:
doc = skyvern_block_validate.__doc__ or ""
assert "skyvern_block_schema" in doc
+
+
+async def test_block_validate_data_export() -> None:
+ result = await skyvern_block_validate(
+ block_json=json.dumps(
+ {
+ "block_type": "data_export",
+ "label": "export_records",
+ "data": "{{ extract_output.extracted_information }}",
+ "data_schema": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {"id": {"type": "integer"}},
+ },
+ },
+ }
+ )
+ )
+
+ assert result["ok"] is True
+ assert result["data"]["block_type"] == "data_export"
diff --git a/tests/unit/test_mcp_block_type_parity.py b/tests/unit/test_mcp_block_type_parity.py
index 7f54a23ad..db591b111 100644
--- a/tests/unit/test_mcp_block_type_parity.py
+++ b/tests/unit/test_mcp_block_type_parity.py
@@ -95,9 +95,13 @@ class BlockDriftEntry:
# the vendored Fern SDK. Run `fern generate` using the Skyvern Cloud-only
# runbook to resync, then remove the entry here.
_KNOWN_DRIFT_ALLOWLIST: tuple[BlockDriftEntry, ...] = (
- # Empty: pdf_fill (SKY-10944), split_pdf (SKY-12071), and email_inbox
- # (SKY-12068) were resolved by the Fern SDK regen in the 1.0.47 bump — all
- # three now appear in both WorkflowDefinition(Yaml)BlocksItem unions.
+ BlockDriftEntry(
+ block_type="data_export",
+ ticket="SKY-14102",
+ owner="@wintonzheng",
+ added_at=date(2026, 8, 17),
+ notes="Parquet data export awaits the next Fern SDK regeneration.",
+ ),
)
diff --git a/tests/unit/test_mcp_workflow_tools.py b/tests/unit/test_mcp_workflow_tools.py
index 5f3026851..010fc63ca 100644
--- a/tests/unit/test_mcp_workflow_tools.py
+++ b/tests/unit/test_mcp_workflow_tools.py
@@ -189,6 +189,12 @@ def _known_drift_definition(block_type: str) -> dict[str, object]:
block["file_url"] = "{{ source_pdf }}"
block["prompt"] = "Fill the PDF using the payload."
block["payload"] = {"name": "{{ applicant.name }}"}
+ elif block_type == "data_export":
+ block["data"] = "{{ extract_output.extracted_information }}"
+ block["data_schema"] = {
+ "type": "array",
+ "items": {"type": "object", "properties": {"id": {"type": "integer"}}},
+ }
else:
raise ValueError(f"Unsupported known drift block type: {block_type}")
@@ -251,7 +257,7 @@ def _heavy_workflow_run_payload(*, include_expanded_outputs: bool = True) -> dic
}
[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill"])
[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill", "data_export"])
@pytest.mark.asyncio
async def test_workflow_create_sends_known_drift_json_definition_as_raw_dict(
monkeypatch: pytest.MonkeyPatch, block_type: str
@@ -269,11 +275,13 @@ async def test_workflow_create_sends_known_drift_json_definition_as_raw_dict(
assert sent_block["block_type"] == block_type
if block_type.startswith("google_sheets"):
assert sent_block["spreadsheet_url"] == "https://docs.google.com/spreadsheets/d/SPREADSHEET_ID/edit"
+ elif block_type == "data_export":
+ assert sent_block["data_schema"]["items"]["properties"]["id"]["type"] == "integer"
else:
assert sent_block["file_url"] == "{{ source_pdf }}"
[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill"])
[email protected]("block_type", ["google_sheets_read", "google_sheets_write", "pdf_fill", "data_export"])
@pytest.mark.asyncio
async def test_workflow_update_sends_known_drift_json_definition_as_raw_dict(
monkeypatch: pytest.MonkeyPatch, block_type: str
@@ -308,6 +316,8 @@ async def test_workflow_update_sends_known_drift_json_definition_as_raw_dict(
assert sent_block["block_type"] == block_type
if block_type.startswith("google_sheets"):
assert sent_block["spreadsheet_url"] == "https://docs.google.com/spreadsheets/d/SPREADSHEET_ID/edit"
+ elif block_type == "data_export":
+ assert sent_block["data_schema"]["items"]["properties"]["id"]["type"] == "integer"
else:
assert sent_block["file_url"] == "{{ source_pdf }}"
diff --git a/tests/unit/test_parquet_export.py b/tests/unit/test_parquet_export.py
new file mode 100644
index 000000000..68979aa64
--- /dev/null
+++ b/tests/unit/test_parquet_export.py
@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import json
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from skyvern.utils.parquet_export import ParquetExportError, export_parquet_records
+
+_RECORD_SCHEMA = {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {"type": "integer"},
+ "label": {"type": "string"},
+ "price": {"type": "number"},
+ "active": {"type": "boolean"},
+ "tags": {"type": "array", "items": {"type": "string"}},
+ "meta": {
+ "type": "object",
+ "properties": {
+ "source": {"type": "string"},
+ "rank": {"type": "integer"},
+ },
+ },
+ },
+ },
+}
+
+
+def test_parquet_round_trips_declared_types_and_ragged_records() -> None:
+ payload = export_parquet_records(
+ [
+ {
+ "id": 1,
+ "label": "first",
+ "price": 3.5,
+ "active": True,
+ "tags": ["kept"],
+ "meta": {"source": "page", "rank": 2},
+ },
+ {"id": 2, "tags": []},
+ {
+ "id": "not-an-integer",
+ "label": 3,
+ "price": "not-a-number",
+ "active": "yes",
+ "tags": ["kept", 7],
+ "meta": {"source": 9},
+ },
+ ],
+ _RECORD_SCHEMA,
+ )
+
+ table = pq.read_table(pa.BufferReader(payload))
+
+ assert str(table.schema.field("id").type) == "int64"
+ assert str(table.schema.field("label").type) == "string"
+ assert str(table.schema.field("price").type) == "double"
+ assert str(table.schema.field("active").type) == "bool"
+ assert str(table.schema.field("tags").type) == "list<element: string>"
+ assert str(table.schema.field("meta").type) == "struct<source: string, rank: int64>"
+ assert table.to_pylist() == [
+ {
+ "id": 1,
+ "label": "first",
+ "price": 3.5,
+ "active": True,
+ "tags": ["kept"],
+ "meta": {"source": "page", "rank": 2},
+ },
+ {
+ "id": 2,
+ "label": None,
+ "price": None,
+ "active": None,
+ "tags": [],
+ "meta": None,
+ },
+ {
+ "id": None,
+ "label": None,
+ "price": None,
+ "active": None,
+ "tags": ["kept", None],
+ "meta": {"source": None, "rank": None},
+ },
+ ]
+ assert json.loads(table.schema.metadata[b"skyvern.invalid_value_counts"]) == {
+ "active": 1,
+ "id": 1,
+ "label": 1,
+ "meta.source": 1,
+ "price": 1,
+ "tags[]": 1,
+ }
+
+
+def test_empty_export_keeps_the_declared_parquet_schema() -> None:
+ payload = export_parquet_records([], _RECORD_SCHEMA)
+
+ table = pq.read_table(pa.BufferReader(payload))
+
+ assert table.num_rows == 0
+ assert table.schema.names == ["id", "label", "price", "active", "tags", "meta"]
+
+
+def test_parquet_counts_each_malformed_record_once() -> None:
+ payload = export_parquet_records([None, "not-a-record"], _RECORD_SCHEMA)
+
+ table = pq.read_table(pa.BufferReader(payload))
+
+ assert table.to_pylist() == [
+ {"id": None, "label": None, "price": None, "active": None, "tags": None, "meta": None},
+ {"id": None, "label": None, "price": None, "active": None, "tags": None, "meta": None},
+ ]
+ assert json.loads(table.schema.metadata[b"skyvern.invalid_value_counts"]) == {"$[]": 2}
+
+
+def test_parquet_coerces_overflowing_numbers_and_datetimes_to_null() -> None:
+ payload = export_parquet_records(
+ [{"number": 10**400, "timestamp": "9999-12-31T23:59:59-23:59"}],
+ {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "number": {"type": "number"},
+ "timestamp": {"type": "string", "format": "date-time"},
+ },
+ },
+ },
+ )
+
+ table = pq.read_table(pa.BufferReader(payload))
+
+ assert table.to_pylist() == [{"number": None, "timestamp": None}]
+ assert json.loads(table.schema.metadata[b"skyvern.invalid_value_counts"]) == {
+ "number": 1,
+ "timestamp": 1,
+ }
+
+
+def test_parquet_rejects_schemas_without_columns() -> None:
+ with pytest.raises(ParquetExportError, match="must declare at least one property"):
+ export_parquet_records(
+ [{}],
+ {"type": "array", "items": {"type": "object", "properties": {}}},
+ )
+
+
+ "data_schema",
+ [
+ {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {"meta": {"type": "object", "properties": {}}},
+ },
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {"meta": {"type": "array", "items": {"type": "object", "properties": {}}}},
+ },
+ },
+ ],
+)
+def test_parquet_rejects_nested_empty_object_schemas(data_schema: dict[str, object]) -> None:
+ with pytest.raises(ParquetExportError, match="must declare at least one property"):
+ export_parquet_records([{"meta": {}}], data_schema)
diff --git a/uv.lock b/uv.lock
index 675e59e44..c24619d20 100644
--- a/uv.lock
+++ b/uv.lock
@@ -5448,6 +5448,56 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9b/bf/7595e817906a29453ba4d99394e781b6fabe55d21f3c15d240f85dd06bb1/py_serializable-2.1.0-py3-none-any.whl", hash = "sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304", size = 23045, upload-time = "2025-07-21T09:56:46.848Z" },
]
+[[package]]
+name = "pyarrow"
+version = "23.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" },
+ { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" },
+ { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" },
+ { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" },
+ { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" },
+ { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" },
+ { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" },
+ { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" },
+ { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" },
+ { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" },
+ { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" },
+ { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" },
+ { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" },
+ { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" },
+ { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" },
+ { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" },
+ { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" },
+]
+
[[package]]
name = "pyasn1"
version = "0.6.4"
@@ -6849,6 +6899,7 @@ local = [
{ name = "playwright", version = "1.46.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
{ name = "playwright", version = "1.58.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "psutil" },
+ { name = "pyarrow" },
{ name = "pyasn1" },
{ name = "pyjwt", extra = ["crypto"] },
{ name = "pyotp" },
@@ -6919,6 +6970,7 @@ server = [
{ name = "posthog" },
{ name = "psutil" },
{ name = "psycopg", extra = ["binary", "pool"] },
+ { name = "pyarrow" },
{ name = "pyasn1" },
{ name = "pyjwt", extra = ["crypto"] },
{ name = "pyotp" },
@@ -7101,6 +7153,8 @@ requires-dist = [
{ name = "psutil", marker = "extra == 'local'", specifier = ">=7.0.0" },
{ name = "psutil", marker = "extra == 'server'", specifier = ">=7.0.0" },
{ name = "psycopg", extras = ["binary", "pool"], marker = "extra == 'server'", specifier = ">=3.2.10,<3.3.0" },
+ { name = "pyarrow", marker = "extra == 'local'", specifier = ">=18.0.0,<24" },
+ { name = "pyarrow", marker = "extra == 'server'", specifier = ">=18.0.0,<24" },
{ name = "pyasn1", marker = "extra == 'local'", specifier = ">=0.6.4" },
{ name = "pyasn1", marker = "extra == 'server'", specifier = ">=0.6.4" },
{ name = "pydantic", specifier = ">=2.13.4,<3" },