:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): show the csv quote option only for csv formats (#121171)
Tom Owersebfe18de56dd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.pyindex c6d6cb9a6c8f..78d2473336a9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py@@ -25,15 +25,20 @@ class _CombineFilesConfig(Protocol):def enabled(self) -> bool: ...+class _FileFormatConfig(Protocol):+ @property+ def selection(self) -> FileFormatChoice: ...+ @property+ def csv_allow_double_quotes(self) -> bool: ...++class FileOptionsConfig(Protocol):"""The settings every object storage source's generated config carries."""@propertydef file_pattern(self) -> str | None: ...@property- def file_format(self) -> FileFormatChoice: ...- @property- def csv_allow_double_quotes(self) -> bool: ...+ def file_format(self) -> _FileFormatConfig: ...@propertydef combine_files(self) -> _CombineFilesConfig | None: ...@@ -41,9 +46,9 @@ class FileOptionsConfig(Protocol):def discovery_options_from(config: FileOptionsConfig) -> DiscoveryOptions:combine = config.combine_filesreturn DiscoveryOptions(- file_format=config.file_format,+ file_format=config.file_format.selection,file_pattern=(config.file_pattern or "").strip() or None,- allow_double_quotes=config.csv_allow_double_quotes,+ allow_double_quotes=config.file_format.csv_allow_double_quotes,combined_table_name=combine.table_name if combine is not None and combine.enabled else None,)@@ -87,6 +92,20 @@ def folder_fields(container: str) -> list[FieldType]:)+def _csv_quote_field() -> FieldType:+ return SourceFieldSelectConfig(+ name="csv_allow_double_quotes",+ label="Do double quotes enclose CSV values?",+ required=True,+ defaultValue="true",+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,+ options=[+ SourceFieldSelectConfigOption(label="Yes", value="true"),+ SourceFieldSelectConfigOption(label="No, treat them as plain text", value="false"),+ ],+ )++def file_option_fields() -> list[FieldType]:return cast(list[FieldType],@@ -97,25 +116,20 @@ def file_option_fields() -> list[FieldType]:required=True,defaultValue="infer",options=[- SourceFieldSelectConfigOption(label="Detect from file extension", value="infer"),- SourceFieldSelectConfigOption(label="CSV with a header row", value="csv"),- SourceFieldSelectConfigOption(label="CSV without a header row", value="csv_no_header"),+ SourceFieldSelectConfigOption(+ label="Detect from file extension", value="infer", fields=[_csv_quote_field()]+ ),+ SourceFieldSelectConfigOption(+ label="CSV with a header row", value="csv", fields=[_csv_quote_field()]+ ),+ SourceFieldSelectConfigOption(+ label="CSV without a header row", value="csv_no_header", fields=[_csv_quote_field()]+ ),SourceFieldSelectConfigOption(label="JSON Lines", value="jsonl"),SourceFieldSelectConfigOption(label="Parquet", value="parquet"),SourceFieldSelectConfigOption(label="Delta Lake", value="delta"),],),- SourceFieldSelectConfig(- name="csv_allow_double_quotes",- label="Do double quotes enclose CSV values?",- required=True,- defaultValue="true",- converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,- options=[- SourceFieldSelectConfigOption(label="Yes", value="true"),- SourceFieldSelectConfigOption(label="No, treat them as plain text", value="false"),- ],- ),SourceFieldSwitchGroupConfig(name="combine_files",label="Combine every file into one table?",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.pyindex 90d6dfb25d33..eb5ce3669ae2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py@@ -8,6 +8,7 @@ from unittest.mock import patchfrom azure.core.exceptions import HttpResponseError, ResourceNotFoundErrorfrom fsspec.implementations.local import LocalFileSystem+from products.warehouse_sources.backend.facade.source_config import SourceFieldSelectConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.amazon_s3.source import AmazonS3Sourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.azure_blob.source import AzureBlobSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare_r2.source import CloudflareR2Source@@ -169,6 +170,45 @@ def test_a_blocked_location_is_never_opened(provider: str) -> None:opened.assert_not_called()+CSV_FORMATS = ["infer", "csv", "csv_no_header"]+NON_CSV_FORMATS = ["jsonl", "parquet", "delta"]++[email protected]("provider", list(PROVIDERS))[email protected]("file_format", CSV_FORMATS + NON_CSV_FORMATS)+def test_the_csv_quote_option_shows_only_under_csv_formats(provider: str, file_format: str) -> None:+ source, _ = PROVIDERS[provider]+ fields = source.get_source_config.fields++ [select] = [f for f in fields if isinstance(f, SourceFieldSelectConfig) and f.name == "file_format"]+ [option] = [o for o in select.options if o.value == file_format]++ assert [f.name for f in option.fields or []] == (["csv_allow_double_quotes"] if file_format in CSV_FORMATS else [])+ assert "csv_allow_double_quotes" not in {f.name for f in fields}++[email protected]("provider", list(PROVIDERS))+ "file_format, expected_format, expected_allow_double_quotes",+ [+ *[({"selection": f, "csv_allow_double_quotes": "false"}, f, False) for f in CSV_FORMATS],+ *[({"selection": f, "csv_allow_double_quotes": "true"}, f, True) for f in CSV_FORMATS],+ *[({"selection": f}, f, True) for f in CSV_FORMATS + NON_CSV_FORMATS],+ ("parquet", "parquet", True),+ ({}, "infer", True),+ ],+)+def test_discovery_options_read_the_csv_quote_option_from_the_file_format(+ provider: str, file_format: Any, expected_format: str, expected_allow_double_quotes: bool+) -> None:+ source, values = PROVIDERS[provider]++ options = source.discovery_options(source.parse_config({**values, "file_format": file_format}))++ assert options.file_format == expected_format+ assert options.allow_double_quotes is expected_allow_double_quotes++class _FakeContainerClient:def __init__(self, blobs: list[Any] | None = None, error: Exception | None = None) -> None:self.blobs = blobs or []diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.pyindex 9fb8f23589dc..677cff8744dd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py@@ -12,14 +12,19 @@ class AmazonS3CombineFilesConfig(config.Config):enabled: bool = config.value(converter=config.str_to_bool, default=False)+class AmazonS3FileFormatConfig(config.Config):+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)++@config.configclass AmazonS3SourceConfig(config.Config):bucket: strregion: straccess_key_id: strsecret_access_key: str+ file_format: AmazonS3FileFormatConfigpath: str | None = Nonefile_pattern: str | None = None- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)combine_files: AmazonS3CombineFilesConfig | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.pyindex 4ddcc5c9491c..775d87ca556d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py@@ -12,13 +12,18 @@ class AzureBlobCombineFilesConfig(config.Config):enabled: bool = config.value(converter=config.str_to_bool, default=False)+class AzureBlobFileFormatConfig(config.Config):+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)++@config.configclass AzureBlobSourceConfig(config.Config):account_name: strcontainer: straccount_key: str+ file_format: AzureBlobFileFormatConfigpath: str | None = Nonefile_pattern: str | None = None- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)combine_files: AzureBlobCombineFilesConfig | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.pyindex c40a2b592296..9fba89debc62 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py@@ -12,14 +12,19 @@ class CloudflareR2CombineFilesConfig(config.Config):enabled: bool = config.value(converter=config.str_to_bool, default=False)+class CloudflareR2FileFormatConfig(config.Config):+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)++@config.configclass CloudflareR2SourceConfig(config.Config):account_id: strbucket: straccess_key_id: strsecret_access_key: str+ file_format: CloudflareR2FileFormatConfigpath: str | None = Nonefile_pattern: str | None = None- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)combine_files: CloudflareR2CombineFilesConfig | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.pyindex 53217756b46c..03340408394e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py@@ -12,13 +12,18 @@ class GoogleCloudStorageCombineFilesConfig(config.Config):enabled: bool = config.value(converter=config.str_to_bool, default=False)+class GoogleCloudStorageFileFormatConfig(config.Config):+ selection: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = "infer"+ csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)++@config.configclass GoogleCloudStorageSourceConfig(config.Config):bucket: straccess_key_id: strsecret_access_key: str+ file_format: GoogleCloudStorageFileFormatConfigpath: str | None = Nonefile_pattern: str | None = None- file_format: Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"] = config.value(default="infer")- csv_allow_double_quotes: bool = config.value(default=config.str_to_bool("true"), converter=config.str_to_bool)combine_files: GoogleCloudStorageCombineFilesConfig | None = Nonediff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex bf7399a27e8c..c31ebdc3d0ba 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -9346,6 +9346,7 @@ class TestExternalDataSource(APIBaseTest):"region": "us-east-1","access_key_id": "AKIAEXAMPLE","secret_access_key": "secret",+ "file_format": {"selection": "infer", "csv_allow_double_quotes": "true"},}with patch("products.warehouse_sources.backend.presentation.views.external_data_source.helpers.feature_enabled_or_false",