Skip to content

Automate browser based workflows with AI

[SKY-8952] CSV with Wide Header Exceede ~1 MB (#5510)

Aaron Perez0e72bdabd77e
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 9b36ca8f9..36255184f 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -2414,555 +2414,6 @@
}
}
}
- },
- "/v1/schedules": {
- "get": {
- "tags": [
- "Schedules"
- ],
- "summary": "List all schedules for the organization",
- "operationId": "schedules_list_all",
- "parameters": [
- {
- "name": "page",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "minimum": 1,
- "default": 1,
- "title": "Page"
- }
- },
- {
- "name": "page_size",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "maximum": 100,
- "minimum": 1,
- "default": 10,
- "title": "Page Size"
- }
- },
- {
- "name": "status",
- "in": "query",
- "required": false,
- "schema": {
- "enum": [
- "active",
- "paused"
- ],
- "type": "string",
- "nullable": true,
- "description": "Filter by status: 'active' or 'paused'",
- "title": "Status"
- },
- "description": "Filter by status: 'active' or 'paused'"
- },
- {
- "name": "search",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Search by workflow title or schedule name",
- "title": "Search"
- },
- "description": "Search by workflow title or schedule name"
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/OrganizationScheduleListResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "list_organization_schedules"
- }
- },
- "/v1/workflows/{workflow_permanent_id}/schedules": {
- "post": {
- "tags": [
- "Schedules"
- ],
- "summary": "Create a schedule for a workflow",
- "operationId": "schedules_create",
- "parameters": [
- {
- "name": "workflow_permanent_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Permanent Id"
- }
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleUpsertRequest"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "create_workflow_schedule"
- },
- "get": {
- "tags": [
- "Schedules"
- ],
- "summary": "List schedules for a workflow",
- "operationId": "schedules_list",
- "parameters": [
- {
- "name": "workflow_permanent_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Permanent Id"
- }
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleListResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "list_workflow_schedules"
- }
- },
- "/v1/workflows/{workflow_permanent_id}/schedules/{workflow_schedule_id}": {
- "get": {
- "tags": [
- "Schedules"
- ],
- "summary": "Get a schedule by ID",
- "operationId": "schedules_get",
- "parameters": [
- {
- "name": "workflow_permanent_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Permanent Id"
- }
- },
- {
- "name": "workflow_schedule_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Schedule Id"
- }
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "get_workflow_schedule"
- },
- "put": {
- "tags": [
- "Schedules"
- ],
- "summary": "Update a schedule",
- "operationId": "schedules_update",
- "parameters": [
- {
- "name": "workflow_permanent_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Permanent Id"
- }
- },
- {
- "name": "workflow_schedule_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Schedule Id"
- }
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleUpsertRequest"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "update_workflow_schedule"
- },
- "delete": {
- "tags": [
- "Schedules"
- ],
- "summary": "Delete a schedule",
- "operationId": "schedules_delete",
- "parameters": [
- {
- "name": "workflow_permanent_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Permanent Id"
- }
- },
- {
- "name": "workflow_schedule_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Schedule Id"
- }
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/DeleteScheduleResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "delete_workflow_schedule_route"
- }
- },
- "/v1/workflows/{workflow_permanent_id}/schedules/{workflow_schedule_id}/enable": {
- "post": {
- "tags": [
- "Schedules"
- ],
- "summary": "Enable a schedule",
- "operationId": "schedules_enable",
- "parameters": [
- {
- "name": "workflow_permanent_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Permanent Id"
- }
- },
- {
- "name": "workflow_schedule_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Schedule Id"
- }
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "enable_workflow_schedule"
- }
- },
- "/v1/workflows/{workflow_permanent_id}/schedules/{workflow_schedule_id}/disable": {
- "post": {
- "tags": [
- "Schedules"
- ],
- "summary": "Disable a schedule",
- "operationId": "schedules_disable",
- "parameters": [
- {
- "name": "workflow_permanent_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Permanent Id"
- }
- },
- {
- "name": "workflow_schedule_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Workflow Schedule Id"
- }
- },
- {
- "name": "x-api-key",
- "in": "header",
- "required": false,
- "schema": {
- "type": "string",
- "nullable": true,
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
- "title": "X-Api-Key"
- },
- "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/WorkflowScheduleResponse"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- },
- "x-fern-sdk-group-name": "agent",
- "x-fern-sdk-method-name": "disable_workflow_schedule"
- }
}
},
"components": {
@@ -4512,8 +3963,8 @@
"properties": {
"file": {
"type": "string",
- "title": "File",
- "format": "binary"
+ "format": "binary",
+ "title": "File"
}
},
"type": "object",
@@ -10999,8 +10450,8 @@
},
"engine": {
"$ref": "#/components/schemas/RunEngine",
- "description": "\nThe engine that powers the agent task. The default value is `skyvern-1.0`, which is good for simple tasks like filling a form, or searching for information on Google. `skyvern-2.0` is the latest Skyvern agent that performs well with complex and multi-step tasks. The `openai-cua` engine uses OpenAI's CUA model. The `anthropic-cua` uses Anthropic's Claude Sonnet 3.7 model with the computer use tool.\n",
- "default": "skyvern-1.0"
+ "description": "\nThe engine that powers the agent task. The default value is `skyvern-2.0`, the latest Skyvern agent that performs pretty well with complex and multi-step tasks. `skyvern-1.0` is good for simple tasks like filling a form, or searching for information on Google. The `openai-cua` engine uses OpenAI's CUA model. The `anthropic-cua` uses Anthropic's Claude Sonnet 3.7 model with the computer use tool.\n",
+ "default": "skyvern-2.0"
},
"title": {
"type": "string",
@@ -14264,280 +13715,6 @@
"azure_vault"
],
"title": "CredentialType"
- },
- "DeleteScheduleResponse": {
- "properties": {
- "ok": {
- "type": "boolean",
- "title": "Ok"
- }
- },
- "type": "object",
- "required": [
- "ok"
- ],
- "title": "DeleteScheduleResponse"
- },
- "OrganizationScheduleItem": {
- "properties": {
- "workflow_schedule_id": {
- "type": "string",
- "title": "Workflow Schedule Id"
- },
- "organization_id": {
- "type": "string",
- "title": "Organization Id"
- },
- "workflow_permanent_id": {
- "type": "string",
- "title": "Workflow Permanent Id"
- },
- "workflow_title": {
- "type": "string",
- "title": "Workflow Title"
- },
- "cron_expression": {
- "type": "string",
- "title": "Cron Expression"
- },
- "timezone": {
- "type": "string",
- "title": "Timezone"
- },
- "enabled": {
- "type": "boolean",
- "title": "Enabled"
- },
- "parameters": {
- "additionalProperties": true,
- "type": "object",
- "nullable": true,
- "title": "Parameters"
- },
- "name": {
- "type": "string",
- "nullable": true,
- "title": "Name"
- },
- "description": {
- "type": "string",
- "nullable": true,
- "title": "Description"
- },
- "next_run": {
- "type": "string",
- "format": "date-time",
- "nullable": true,
- "title": "Next Run"
- },
- "created_at": {
- "type": "string",
- "format": "date-time",
- "title": "Created At"
- },
- "modified_at": {
- "type": "string",
- "format": "date-time",
- "title": "Modified At"
- }
- },
- "type": "object",
- "required": [
- "workflow_schedule_id",
- "organization_id",
- "workflow_permanent_id",
- "workflow_title",
- "cron_expression",
- "timezone",
- "enabled",
- "created_at",
- "modified_at"
- ],
- "title": "OrganizationScheduleItem",
- "description": "Compact schedule projection for the org-wide list endpoint."
- },
- "OrganizationScheduleListResponse": {
- "properties": {
- "schedules": {
- "items": {
- "$ref": "#/components/schemas/OrganizationScheduleItem"
- },
- "type": "array",
- "title": "Schedules"
- },
- "total_count": {
- "type": "integer",
- "title": "Total Count"
- },
- "page": {
- "type": "integer",
- "title": "Page"
- },
- "page_size": {
- "type": "integer",
- "title": "Page Size"
- }
- },
- "type": "object",
- "required": [
- "schedules",
- "total_count",
- "page",
- "page_size"
- ],
- "title": "OrganizationScheduleListResponse"
- },
- "WorkflowSchedule": {
- "properties": {
- "workflow_schedule_id": {
- "type": "string",
- "title": "Workflow Schedule Id"
- },
- "organization_id": {
- "type": "string",
- "title": "Organization Id"
- },
- "workflow_permanent_id": {
- "type": "string",
- "title": "Workflow Permanent Id"
- },
- "cron_expression": {
- "type": "string",
- "title": "Cron Expression"
- },
- "timezone": {
- "type": "string",
- "title": "Timezone"
- },
- "enabled": {
- "type": "boolean",
- "title": "Enabled"
- },
- "parameters": {
- "additionalProperties": true,
- "type": "object",
- "nullable": true,
- "title": "Parameters"
- },
- "temporal_schedule_id": {
- "type": "string",
- "nullable": true,
- "title": "Temporal Schedule Id"
- },
- "name": {
- "type": "string",
- "nullable": true,
- "title": "Name"
- },
- "description": {
- "type": "string",
- "nullable": true,
- "title": "Description"
- },
- "created_at": {
- "type": "string",
- "format": "date-time",
- "title": "Created At"
- },
- "modified_at": {
- "type": "string",
- "format": "date-time",
- "title": "Modified At"
- },
- "deleted_at": {
- "type": "string",
- "format": "date-time",
- "nullable": true,
- "title": "Deleted At"
- }
- },
- "type": "object",
- "required": [
- "workflow_schedule_id",
- "organization_id",
- "workflow_permanent_id",
- "cron_expression",
- "timezone",
- "enabled",
- "created_at",
- "modified_at"
- ],
- "title": "WorkflowSchedule"
- },
- "WorkflowScheduleListResponse": {
- "properties": {
- "schedules": {
- "items": {
- "$ref": "#/components/schemas/WorkflowSchedule"
- },
- "type": "array",
- "title": "Schedules"
- }
- },
- "type": "object",
- "required": [
- "schedules"
- ],
- "title": "WorkflowScheduleListResponse"
- },
- "WorkflowScheduleResponse": {
- "properties": {
- "schedule": {
- "$ref": "#/components/schemas/WorkflowSchedule"
- },
- "next_runs": {
- "items": {
- "type": "string",
- "format": "date-time"
- },
- "type": "array",
- "title": "Next Runs"
- }
- },
- "type": "object",
- "required": [
- "schedule"
- ],
- "title": "WorkflowScheduleResponse"
- },
- "WorkflowScheduleUpsertRequest": {
- "properties": {
- "cron_expression": {
- "type": "string",
- "title": "Cron Expression"
- },
- "timezone": {
- "type": "string",
- "title": "Timezone"
- },
- "enabled": {
- "type": "boolean",
- "title": "Enabled",
- "default": true
- },
- "parameters": {
- "additionalProperties": true,
- "type": "object",
- "nullable": true,
- "title": "Parameters"
- },
- "name": {
- "type": "string",
- "nullable": true,
- "title": "Name"
- },
- "description": {
- "type": "string",
- "nullable": true,
- "title": "Description"
- }
- },
- "type": "object",
- "required": [
- "cron_expression",
- "timezone"
- ],
- "title": "WorkflowScheduleUpsertRequest"
}
}
},
@@ -14552,4 +13729,4 @@
"url": "http://localhost:8000"
}
]
-}
+}
\ No newline at end of file
diff --git a/docs/running-automations/run-a-task.mdx b/docs/running-automations/run-a-task.mdx
index 4eceecadd..fe4fdf523 100644
--- a/docs/running-automations/run-a-task.mdx
+++ b/docs/running-automations/run-a-task.mdx
@@ -271,7 +271,7 @@ The response from polling (`get_run`) and webhooks have slightly different struc
"run_request": {
"prompt": "Get the title of the top post",
"url": "https://news.ycombinator.com/",
- "engine": "skyvern-1.0"
+ "engine": "skyvern-2.0"
}
}
```
diff --git a/docs/running-automations/task-parameters.mdx b/docs/running-automations/task-parameters.mdx
index 4d66e4b43..c67197b64 100644
--- a/docs/running-automations/task-parameters.mdx
+++ b/docs/running-automations/task-parameters.mdx
@@ -125,14 +125,14 @@ The AI engine that powers the task. These are not iterations—they're suited fo
| Engine | Description |
|--------|-------------|
-| `skyvern-1.0` | **Default.** Single objective, precise, faster, cheaper. Best for simple tasks like form filling or single-page extraction. |
-| `skyvern-2.0` | Multi-objective, flexible, handles complex multi-step tasks. 85.85% on WebVoyager benchmark. Slower and more expensive. |
+| `skyvern-2.0` | **Default.** Multi-objective, flexible, handles complex multi-step tasks. 85.85% on WebVoyager benchmark. Slower and more expensive. |
+| `skyvern-1.0` | Single objective, precise, faster, cheaper. Best for simple tasks like form filling or single-page extraction. |
| `openai-cua` | OpenAI's Computer Use Agent |
| `anthropic-cua` | Anthropic Claude Sonnet with computer use |
| `ui-tars` | UI-TARS model (Seed1.5-VL) via Doubao API |
<Tip>
-The default engine `skyvern-1.0` works well for clear, single-goal tasks with faster, cheaper execution. Switch to `skyvern-2.0` when the task requires flexibility or multiple steps.
+Use `skyvern-1.0` when you have a clear, single goal and want faster, cheaper execution. Use `skyvern-2.0` when the task requires flexibility or multiple steps.
</Tip>
<CodeGroup>
diff --git a/docs/snippets/core-concepts-content.mdx b/docs/snippets/core-concepts-content.mdx
index 1847e792c..5f0cf0e67 100644
--- a/docs/snippets/core-concepts-content.mdx
+++ b/docs/snippets/core-concepts-content.mdx
@@ -360,8 +360,8 @@ Skyvern supports multiple AI engines for task execution:
| Engine | Description |
|--------|-------------|
-| `skyvern-1.0` | Default Skyvern model — fast, precise, ideal for simple tasks |
-| `skyvern-2.0` | Latest Skyvern model — flexible, handles complex multi-step tasks |
+| `skyvern-2.0` | Latest Skyvern model (default, recommended) |
+| `skyvern-1.0` | Previous Skyvern model |
| `openai-cua` | OpenAI Computer Use Agent |
| `anthropic-cua` | Anthropic Computer Use Agent |
| `ui-tars` | UI-TARS model |
diff --git a/docs/ts-sdk-reference/complete-reference.mdx b/docs/ts-sdk-reference/complete-reference.mdx
index 9cc63312b..5831db259 100644
--- a/docs/ts-sdk-reference/complete-reference.mdx
+++ b/docs/ts-sdk-reference/complete-reference.mdx
@@ -97,7 +97,7 @@ const result = await skyvern.runTask({
body: {
prompt: string, // Required. Natural language instructions.
url?: string, // Starting page URL.
- engine?: RunEngine, // "skyvern_v1" (default), "skyvern_v2", "openai_cua", "anthropic_cua", "ui_tars"
+ engine?: RunEngine, // "skyvern_v2" (default), "skyvern_v1", "openai_cua", "anthropic_cua", "ui_tars"
max_steps?: number, // Cap AI steps to limit cost.
data_extraction_schema?: Record<string, unknown> | string, // JSON Schema for output.
browser_session_id?: string, // Run in existing session.
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index a22eabadc..0833f2649 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import abc
import ast
import asyncio
+import codecs
import copy
import csv
import json
@@ -3738,6 +3739,13 @@ class FileParserBlock(Block):
# Parameter 1 of Literal[...] cannot be of type "Any"
block_type: Literal[BlockType.FILE_URL_PARSER] = BlockType.FILE_URL_PARSER # type: ignore
+ # FileParserBlock CSV constants
+ _CSV_SNIFF_LINES = 5
+ _CSV_BINARY_PREFIX_BYTES = 4096
+ _CSV_UTF_BOMS = (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE, codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)
+ # Bounded cap for legitimate wide cells (JSON blobs, long descriptions); applied only while parsing.
+ _MAX_CSV_FIELD_SIZE_BYTES = 10 * 1024 * 1024
+
file_url: str
file_type: FileType
json_schema: dict[str, Any] | None = None
@@ -3840,12 +3848,46 @@ class FileParserBlock(Block):
# latin-1 always succeeds (1:1 byte mapping), so this is a safety fallback
return "latin-1"
+ def _sniff_csv_delimiter(self, file_path: str) -> tuple[str, str]:
+ """Return (delimiter, encoding). Samples full lines to avoid mid-row truncation."""
+ # Read small raw byte prefix to quickly detect empty binary files before attempting text decoding/sniffing
+ with open(file_path, "rb") as f:
+ raw_prefix = f.read(self._CSV_BINARY_PREFIX_BYTES)
+ # Reject files that contain no meaningful bytes
+ if not raw_prefix.strip():
+ raise csv.Error("File is empty")
+ # Reject likely binary content:
+ # - Presence of null bytes is a strong binary signal
+ # - Exception: UTF-16/UTF-32 text often starts with BOM and may contain null bytes
+ if b"\x00" in raw_prefix and not raw_prefix.startswith(self._CSV_UTF_BOMS):
+ raise csv.Error("File contains binary data")
+
+ # Detect best text encoding for file, then read only the first N full lines so csv.Sniffer sees complete rows
+ encoding = self._detect_file_encoding(file_path)
+ with open(file_path, encoding=encoding, errors="replace", newline="") as file:
+ lines: list[str] = []
+ for _ in range(self._CSV_SNIFF_LINES):
+ line = file.readline()
+ if not line:
+ break
+ lines.append(line)
+
+ # Build the sniffer sample from complete lines only
+ sample = "".join(lines)
+ # Guard against files that decode but still contain no meaningful text
+ if not sample.strip():
+ raise csv.Error("File is empty")
+
+ try:
+ delimiter = csv.Sniffer().sniff(sample).delimiter
+ except csv.Error:
+ delimiter = "\t" if file_path.lower().endswith(".tsv") else ","
+ return delimiter, encoding
+
def validate_file_type(self, file_url_used: str, file_path: str) -> None:
if self.file_type == FileType.CSV:
try:
- encoding = self._detect_file_encoding(file_path)
- with open(file_path, encoding=encoding, errors="replace") as file:
- csv.Sniffer().sniff(file.read(1024))
+ self._sniff_csv_delimiter(file_path)
except csv.Error as e:
raise InvalidFileType(file_url=file_url_used, file_type=self.file_type, error=str(e))
elif self.file_type == FileType.EXCEL:
@@ -3878,25 +3920,14 @@ class FileParserBlock(Block):
async def _parse_csv_file(self, file_path: str) -> list[dict[str, Any]]:
"""Parse CSV/TSV file and return list of dictionaries."""
- parsed_data = []
- encoding = self._detect_file_encoding(file_path)
- with open(file_path, encoding=encoding, errors="replace") as file:
- # Try to detect the delimiter (comma for CSV, tab for TSV)
- sample = file.read(1024)
- file.seek(0) # Reset file pointer
-
- # Use csv.Sniffer to detect the delimiter
- try:
- dialect = csv.Sniffer().sniff(sample)
- delimiter = dialect.delimiter
- except csv.Error:
- # Default to comma if detection fails
- delimiter = ","
-
- reader = csv.DictReader(file, delimiter=delimiter)
- for row in reader:
- parsed_data.append(row)
- return parsed_data
+ delimiter, encoding = self._sniff_csv_delimiter(file_path)
+ previous_limit = csv.field_size_limit(self._MAX_CSV_FIELD_SIZE_BYTES)
+ try:
+ with open(file_path, encoding=encoding, errors="replace", newline="") as file:
+ reader = csv.DictReader(file, delimiter=delimiter)
+ return list(reader)
+ finally:
+ csv.field_size_limit(previous_limit)
def _clean_dataframe_for_json(self, df: pd.DataFrame) -> list[dict[str, Any]]:
"""Clean DataFrame to ensure it can be serialized to JSON."""