Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): show the csv quote option only for csv formats (#121171)

Tom Owersebfe18de56dd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
index c6d6cb9a6c8f..78d2473336a9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
@@ -25,15 +25,20 @@ class _CombineFilesConfig(Protocol):
def enabled(self) -> bool: ...
+class _FileFormatConfig(Protocol):
+ @property
+ def selection(self) -> FileFormatChoice: ...
+ @property
+ def csv_allow_double_quotes(self) -> bool: ...
+
+
class FileOptionsConfig(Protocol):
"""The settings every object storage source's generated config carries."""
@property
def file_pattern(self) -> str | None: ...
@property
- def file_format(self) -> FileFormatChoice: ...
- @property
- def csv_allow_double_quotes(self) -> bool: ...
+ def file_format(self) -> _FileFormatConfig: ...
@property
def combine_files(self) -> _CombineFilesConfig | None: ...
@@ -41,9 +46,9 @@ class FileOptionsConfig(Protocol):
def discovery_options_from(config: FileOptionsConfig) -> DiscoveryOptions:
combine = config.combine_files
return DiscoveryOptions(
- file_format=config.file_format,
+ file_format=config.file_format.selection,
file_pattern=(config.file_pattern or "").strip() or None,
- allow_double_quotes=config.csv_allow_double_quotes,
+ allow_double_quotes=config.file_format.csv_allow_double_quotes,
combined_table_name=combine.table_name if combine is not None and combine.enabled else None,
)
@@ -87,6 +92,20 @@ def folder_fields(container: str) -> list[FieldType]:
)
+def _csv_quote_field() -> FieldType:
+ return SourceFieldSelectConfig(
+ name="csv_allow_double_quotes",
+ label="Do double quotes enclose CSV values?",
+ required=True,
+ defaultValue="true",
+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,
+ options=[
+ SourceFieldSelectConfigOption(label="Yes", value="true"),
+ SourceFieldSelectConfigOption(label="No, treat them as plain text", value="false"),
+ ],
+ )
+
+
def file_option_fields() -> list[FieldType]:
return cast(
list[FieldType],
@@ -97,25 +116,20 @@ def file_option_fields() -> list[FieldType]:
required=True,
defaultValue="infer",
options=[
- SourceFieldSelectConfigOption(label="Detect from file extension", value="infer"),
- SourceFieldSelectConfigOption(label="CSV with a header row", value="csv"),
- SourceFieldSelectConfigOption(label="CSV without a header row", value="csv_no_header"),
+ SourceFieldSelectConfigOption(
+ label="Detect from file extension", value="infer", fields=[_csv_quote_field()]
+ ),
+ SourceFieldSelectConfigOption(
+ label="CSV with a header row", value="csv", fields=[_csv_quote_field()]
+ ),
+ SourceFieldSelectConfigOption(
+ label="CSV without a header row", value="csv_no_header", fields=[_csv_quote_field()]
+ ),
SourceFieldSelectConfigOption(label="JSON Lines", value="jsonl"),
SourceFieldSelectConfigOption(label="Parquet", value="parquet"),
SourceFieldSelectConfigOption(label="Delta Lake", value="delta"),
],
),
- SourceFieldSelectConfig(
- name="csv_allow_double_quotes",
- label="Do double quotes enclose CSV values?",
- required=True,
- defaultValue="true",
- converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,
- options=[
- SourceFieldSelectConfigOption(label="Yes", value="true"),
- SourceFieldSelectConfigOption(label="No, treat them as plain text", value="false"),
- ],
- ),
SourceFieldSwitchGroupConfig(
name="combine_files",
label="Combine every file into one table?",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py
index 90d6dfb25d33..eb5ce3669ae2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py
@@ -8,6 +8,7 @@ from unittest.mock import patch
from azure.core.exceptions import HttpResponseError, ResourceNotFoundError
from fsspec.implementations.local import LocalFileSystem
+from products.warehouse_sources.backend.facade.source_config import SourceFieldSelectConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Source
from products.warehouse_sources.backend.temporal.data_imports.sources.azure_blob.source import AzureBlobSource
from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare_r2.source import CloudflareR2Source
@@ -169,6 +170,45 @@ def test_a_blocked_location_is_never_opened(provider: str) -> None:
opened.assert_not_called()
+CSV_FORMATS = ["infer", "csv", "csv_no_header"]
+NON_CSV_FORMATS = ["jsonl", "parquet", "delta"]
+
+
[email protected]("provider", list(PROVIDERS))
[email protected]("file_format", CSV_FORMATS + NON_CSV_FORMATS)
+def test_the_csv_quote_option_shows_only_under_csv_formats(provider: str, file_format: str) -> None:
+ source, _ = PROVIDERS[provider]
+ fields = source.get_source_config.fields
+
+ [select] = [f for f in fields if isinstance(f, SourceFieldSelectConfig) and f.name == "file_format"]
+ [option] = [o for o in select.options if o.value == file_format]
+
+ assert [f.name for f in option.fields or []] == (["csv_allow_double_quotes"] if file_format in CSV_FORMATS else [])
+ assert "csv_allow_double_quotes" not in {f.name for f in fields}
+
+
[email protected]("provider", list(PROVIDERS))
+ "file_format, expected_format, expected_allow_double_quotes",
+ [
+ *[({"selection": f, "csv_allow_double_quotes": "false"}, f, False) for f in CSV_FORMATS],
+ *[({"selection": f, "csv_allow_double_quotes": "true"}, f, True) for f in CSV_FORMATS],
+ *[({"selection": f}, f, True) for f in CSV_FORMATS + NON_CSV_FORMATS],
+ ("parquet", "parquet", True),
+ ({}, "infer", True),
+ ],
+)
+def test_discovery_options_read_the_csv_quote_option_from_the_file_format(
+ provider: str, file_format: Any, expected_format: str, expected_allow_double_quotes: bool
+) -> None:
+ source, values = PROVIDERS[provider]
+
+ options = source.discovery_options(source.parse_config({**values, "file_format": file_format}))
+
+ assert options.file_format == expected_format
+ assert options.allow_double_quotes is expected_allow_double_quotes
+
+
class _FakeContainerClient:
def __init__(self, blobs: list[Any] | None = None, error: Exception | None = None) -> None:
self.blobs = blobs or []
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
index 9fb8f23589dc..677cff8744dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
@@ -12,14 +12,19 @@ class AmazonS3CombineFilesConfig(config.Config):
enabled: bool = config.value(converter=config.str_to_bool, default=False)
+class AmazonS3FileFormatConfig(config.Config):
+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"
+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
+
+
@config.config
class AmazonS3SourceConfig(config.Config):
bucket: str
region: str
access_key_id: str
secret_access_key: str
+ file_format: AmazonS3FileFormatConfig
path: str | None = None
file_pattern: str | None = None
- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")
- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
combine_files: AmazonS3CombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py
index 4ddcc5c9491c..775d87ca556d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py
@@ -12,13 +12,18 @@ class AzureBlobCombineFilesConfig(config.Config):
enabled: bool = config.value(converter=config.str_to_bool, default=False)
+class AzureBlobFileFormatConfig(config.Config):
+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"
+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
+
+
@config.config
class AzureBlobSourceConfig(config.Config):
account_name: str
container: str
account_key: str
+ file_format: AzureBlobFileFormatConfig
path: str | None = None
file_pattern: str | None = None
- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")
- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
combine_files: AzureBlobCombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py
index c40a2b592296..9fba89debc62 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py
@@ -12,14 +12,19 @@ class CloudflareR2CombineFilesConfig(config.Config):
enabled: bool = config.value(converter=config.str_to_bool, default=False)
+class CloudflareR2FileFormatConfig(config.Config):
+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"
+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
+
+
@config.config
class CloudflareR2SourceConfig(config.Config):
account_id: str
bucket: str
access_key_id: str
secret_access_key: str
+ file_format: CloudflareR2FileFormatConfig
path: str | None = None
file_pattern: str | None = None
- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")
- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
combine_files: CloudflareR2CombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py
index 53217756b46c..03340408394e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py
@@ -12,13 +12,18 @@ class GoogleCloudStorageCombineFilesConfig(config.Config):
enabled: bool = config.value(converter=config.str_to_bool, default=False)
+class GoogleCloudStorageFileFormatConfig(config.Config):
+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"
+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
+
+
@config.config
class GoogleCloudStorageSourceConfig(config.Config):
bucket: str
access_key_id: str
secret_access_key: str
+ file_format: GoogleCloudStorageFileFormatConfig
path: str | None = None
file_pattern: str | None = None
- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")
- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)
combine_files: GoogleCloudStorageCombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index bf7399a27e8c..c31ebdc3d0ba 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -9346,6 +9346,7 @@ class TestExternalDataSource(APIBaseTest):
"region": "us-east-1",
"access_key_id": "AKIAEXAMPLE",
"secret_access_key": "secret",
+ "file_format": {"selection": "infer", "csv_allow_double_quotes": "true"},
}
with patch(
"products.warehouse_sources.backend.presentation.views.external_data_source.helpers.feature_enabled_or_false",