:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): keep date columns when migrating self-managed json tables (#125618)
Tom Owersd1e6d1e0a59a
diff --git a/products/warehouse_sources/backend/file_upload_sources.py b/products/warehouse_sources/backend/file_upload_sources.pyindex 2286008c633f..ec3222e5a571 100644--- a/products/warehouse_sources/backend/file_upload_sources.py+++ b/products/warehouse_sources/backend/file_upload_sources.py@@ -60,7 +60,9 @@ from products.warehouse_sources.backend.presentation.views.external_data_source.)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import CsvOptionsfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (+ CsvColumnType,csv_column_types_from_columns,+ json_column_types_from_columns,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (TableDefinition,@@ -157,6 +159,14 @@ def pending_upload_job_inputs(file_format: UploadedFileFormat, schema_name: str)return {"upload_id": "", "filename": "", "file_format": file_format, "schema_name": schema_name}+def _stored_column_types(read: str, read_settings: UploadReadSettings) -> tuple[CsvColumnType, ...]:+ if read == "csv":+ return csv_column_types_from_columns(read_settings.columns)+ if read == "jsonl":+ return json_column_types_from_columns(read_settings.columns)+ return ()++def upload_schema_metadata(schema_name: str,filename: str,@@ -176,7 +186,7 @@ def upload_schema_metadata(csv=CsvOptions(has_header=True,allow_double_quotes=read_settings.allow_double_quotes is not False,- column_types=csv_column_types_from_columns(read_settings.columns) if read == "csv" else (),+ column_types=_stored_column_types(read, read_settings),),pattern=None,source_format=read,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.pyindex 2a58328db1ff..9ead4fe8a9b7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py@@ -12,6 +12,9 @@ from posthog.dataclasses import frozenCsvColumnKind = Literal["integer", "unsigned", "float", "decimal", "boolean", "date", "datetime"]CSV_COLUMN_KINDS: frozenset[str] = frozenset(get_args(CsvColumnKind))+# The CSV column kinds a JSON Lines column is cast to. JSON already types numbers and booleans.+JSON_CAST_KINDS: frozenset[str] = frozenset({"date", "datetime"})+_WRAPPER = re.compile(r"^(?:Nullable|LowCardinality)\((?P<inner>.*)\)$")_TIMEZONE_ARGUMENT = re.compile(r"'(?P<timezone>[^']+)'")_DECIMAL_ARGUMENTS = re.compile(r"^\s*(?P<first>\d+)\s*(?:,\s*(?P<second>\d+)\s*)?$")@@ -180,14 +183,25 @@ def stored_clickhouse_types(columns: Mapping[str, Any] | None) -> dict[str, str]def csv_column_types_from_columns(columns: Mapping[str, Any] | None) -> tuple[CsvColumnType, ...]:"""The casts for a warehouse table's stored columns, in their stored order."""+ return _column_types(columns, only_kinds=None)+++def _column_types(columns: Mapping[str, Any] | None, only_kinds: frozenset[str] | None) -> tuple[CsvColumnType, ...]:column_types: list[CsvColumnType] = []for name, clickhouse_type in stored_clickhouse_types(columns).items():+ if only_kinds is not None and _CLICKHOUSE_KINDS.get(_unwrap(clickhouse_type).base) not in only_kinds:+ continuecolumn_type = csv_column_type(name, clickhouse_type)if column_type is not None:column_types.append(column_type)return tuple(column_types)+def json_column_types_from_columns(columns: Mapping[str, Any] | None) -> tuple[CsvColumnType, ...]:+ """The casts for the date and time columns of a warehouse table. Other JSON columns keep their JSON type."""+ return _column_types(columns, only_kinds=JSON_CAST_KINDS)++def _column_type_from_metadata(name: str, raw: object) -> CsvColumnType | None:if isinstance(raw, str):return CsvColumnType(name=name, kind=cast(CsvColumnKind, raw)) if raw in CSV_COLUMN_KINDS else Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.pyindex ffc468836a4e..75a31846e6c3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py@@ -9,6 +9,7 @@ from typing import IO, Any, Literal, castimport pyarrow as paimport pyarrow.csv as pa_csv+import pyarrow.compute as pcimport pyarrow.parquet as pqfrom structlog.types import FilteringBoundLogger@@ -23,6 +24,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objFileFormat,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (+ JSON_CAST_KINDS,+ CsvColumnType,cast_csv_columns,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import is_compressed@@ -244,12 +247,46 @@ def _as_rows(value: Any) -> Iterator[dict[str, Any]]:yield {"value": value}+def _typed_jsonl(table: pa.Table, column_types: tuple[CsvColumnType, ...], nulled_values: dict[str, int]) -> pa.Table:+ # Only the date and time columns are cast. A JSON number or boolean already has its type.+ casts = tuple(column_type for column_type in column_types if column_type.kind in JSON_CAST_KINDS)+ for column_type in casts:+ if column_type.name not in table.column_names:+ continue+ index = table.schema.get_field_index(column_type.name)+ if not pa.types.is_string(table.schema.field(index).type):+ table = table.set_column(index, column_type.name, pc.cast(table.column(index), pa.string(), safe=False))+ result = cast_csv_columns(table, casts)+ for column, count in result.nulled_values.items():+ nulled_values[column] = nulled_values.get(column, 0) + count+ return result.table++def iter_jsonl_tables(stream: IO[bytes],file_path: str = "",max_rows: int = JSONL_CHUNK_ROWS,max_bytes: int = JSONL_CHUNK_BYTES,+ column_types: tuple[CsvColumnType, ...] = (),+ logger: FilteringBoundLogger | None = None,) -> Iterator[pa.Table]:+ """Stream a JSON Lines file as tables, casting the date and time columns in `column_types`.++ A JSON file holds a date as text. A table moved from a self-managed table keeps the date types+ ClickHouse read, so those columns are cast like CSV columns, and a value that is not a date is null.+ """+ nulled_values: dict[str, int] = {}+ for table in _iter_untyped_jsonl_tables(stream, file_path, max_rows, max_bytes):+ yield _typed_jsonl(table, column_types, nulled_values) if column_types else table+ if nulled_values and logger is not None:+ logger.warning(+ "Read JSON values that don't match their column type as null",+ file=file_path,+ nulled_values=nulled_values,+ )+++def _iter_untyped_jsonl_tables(stream: IO[bytes], file_path: str, max_rows: int, max_bytes: int) -> Iterator[pa.Table]:rows: list[dict[str, Any]] = []pending_bytes = 0line_number = 0@@ -362,7 +399,7 @@ def iter_file_tables(delimiter = csv_options.delimiter or ("\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ",")tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)else:- tables = iter_jsonl_tables(binary, stored.path)+ tables = iter_jsonl_tables(binary, stored.path, column_types=csv_options.column_types, logger=logger)rows_read = 0try:for table in tables:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.pyindex b7d76d62eb39..b8e6e16ff125 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objUnsupportedCsvColumnError,clickhouse_base_type,csv_column_types_from_columns,+ json_column_types_from_columns,stored_clickhouse_types,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (@@ -65,8 +66,6 @@ _FORMATS: dict[str, FileFormatChoice] = {}# Column types the import pipeline writes as JSON strings, which breaks queries that read their parts._NESTED_TYPES = frozenset({"Array", "Map", "Tuple", "Nested", "JSON", "Object", "Variant", "Dynamic"})-# Types ClickHouse reads from JSON strings, which the JSON import keeps as strings.-_JSON_TEXT_TYPES = frozenset({"Date", "Date32", "DateTime", "DateTime64"})# Columns the import adds to every row. A stored column of the same name would lose its values to them._FILE_COLUMNS = frozenset({FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN, FILE_ROW_NUMBER_COLUMN})@@ -195,24 +194,12 @@ def _refuse_padded_header_names(columns: Mapping[str, Any] | None) -> None:)-def _refuse_json_text_types(columns: Mapping[str, Any] | None) -> None:- # A JSON file holds dates and times as strings, and the JSON import keeps them as strings.- for name, clickhouse_type in stored_clickhouse_types(columns).items():- base = clickhouse_base_type(clickhouse_type)- if base in _JSON_TEXT_TYPES:- raise UnmappableTableError(- f"the column '{name}' is a {base}, and the JSON import would store it as text, not as a {base}"- )--def refuse_columns_an_import_changes(columns: Mapping[str, Any] | None, file_format: FileFormatChoice) -> None:"""Refuse a file whose import would change the columns a table read in place exposes."""_refuse_nested_columns(columns)_refuse_file_column_names(columns)if file_format == "csv":_refuse_padded_header_names(columns)- if file_format == "jsonl":- _refuse_json_text_types(columns)def table_definition_for(@@ -250,13 +237,16 @@ def table_definition_for(except ObjectStorageError as e:raise UnmappableTableError(str(e)) from etry:- column_types = csv_column_types_from_columns(columns) if file_format in ("csv", "csv_no_header") else ()+ if file_format == "jsonl":+ column_types = json_column_types_from_columns(columns)+ elif file_format in ("csv", "csv_no_header"):+ column_types = csv_column_types_from_columns(columns)+ else:+ column_types = ()except UnsupportedCsvColumnError as e:raise UnmappableTableError(str(e)) from eif file_format == "csv":_refuse_padded_header_names(columns)- if file_format == "jsonl":- _refuse_json_text_types(columns)return TableDefinition(name=name,kind="files",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex b51fdf63931f..173c3b382d49 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -2,13 +2,15 @@ import ioimport osimport csvimport gzip+import jsonimport tempfile-from datetime import UTC, datetime, timedelta+from datetime import UTC, date, datetime, timedeltafrom pathlib import Pathfrom types import SimpleNamespacefrom typing import Anyimport pytest+from unittest.mock import MagicMockfrom django.test import override_settings@@ -34,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objObjectStoreConfig,ObjectStoreProvider,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import CsvColumnTypefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import (ObjectStorageDeltaCursor,iter_delta_tables,@@ -574,6 +577,66 @@ class TestJsonl:{"value": "five"},]+ @parameterized.expand(+ [+ (+ "date",+ CsvColumnType(name="d", kind="date"),+ [{"d": "2024-03-01"}, {"d": None}, {"d": "2024-03-02"}],+ [date(2024, 3, 1), None, date(2024, 3, 2)],+ 0,+ ),+ (+ "datetime_in_utc",+ CsvColumnType(name="d", kind="datetime"),+ [{"d": "2024-03-01 10:30:00"}, {"d": "2024-03-01T10:30:00Z"}, {"d": None}],+ [datetime(2024, 3, 1, 10, 30, tzinfo=UTC)] * 2 + [None],+ 0,+ ),+ (+ "datetime_in_a_time_zone",+ CsvColumnType(name="d", kind="datetime", timezone="America/New_York"),+ [{"d": "2024-07-01 10:30:00"}],+ [datetime(2024, 7, 1, 14, 30, tzinfo=UTC)],+ 0,+ ),+ (+ "bad_values_become_null",+ CsvColumnType(name="d", kind="date"),+ [{"d": "2024-03-01"}, {"d": "not a date"}, {"d": "2024-13-45"}],+ [date(2024, 3, 1), None, None],+ 2,+ ),+ (+ "column_with_only_nulls",+ CsvColumnType(name="d", kind="date"),+ [{"d": None}, {"d": None}],+ [None, None],+ 0,+ ),+ ]+ )+ def test_stored_date_types_cast_json_text(+ self, _name: str, column_type: CsvColumnType, rows: list[dict], expected: list, nulled: int+ ) -> None:+ content = b"\n".join(json.dumps(row).encode() for row in rows) + b"\n"+ logger = MagicMock()++ [table] = iter_jsonl_tables(io.BytesIO(content), "x.jsonl", column_types=(column_type,), logger=logger)++ assert table.schema.field("d").type == column_type.arrow_type+ assert table.column("d").to_pylist() == expected+ assert logger.warning.called is bool(nulled)++ def test_columns_without_a_stored_type_stay_as_they_are(self) -> None:+ content = b'{"d": "2024-03-01", "n": 1}\n'++ [table] = iter_jsonl_tables(+ io.BytesIO(content), "x.jsonl", column_types=(CsvColumnType(name="n", kind="integer"),)+ )++ assert table.to_pylist() == [{"d": "2024-03-01", "n": 1}]+def test_a_too_deeply_nested_record_is_a_parse_error(self) -> None:with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):list(iter_jsonl_tables(io.BytesIO(b"[" * 100_000 + b"]" * 100_000 + b"\n"), "deep.jsonl"))diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.pyindex 164db21c0586..108ca33d6b37 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py@@ -59,6 +59,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.objUnmappableTableError,needs_preserved_column_names,parse_self_managed_url,+ refuse_columns_an_import_changes,source_format_for,table_definition_for,)@@ -539,11 +540,18 @@ class TestTableDefinition:table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)@parameterized.expand([("Date",), ("Nullable(Date32)",), ("DateTime",), ("Nullable(DateTime64(9))",)])- def test_json_dates_and_times_are_refused(self, column_type: str) -> None:+ def test_json_dates_and_times_are_cast_not_refused(self, column_type: str) -> None:location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.jsonl")+ columns = {"id": "Int64", "created": column_type}- with pytest.raises(UnmappableTableError, match="'created'"):- table_definition_for("orders", location, "JSONEachRow", {"id": "Int64", "created": column_type}, None)+ refuse_columns_an_import_changes(columns, "jsonl")+ definition = table_definition_for("orders", location, "JSONEachRow", columns, None)++ assert [column.name for column in definition.csv.column_types] == ["created"]++ def test_json_nested_columns_are_still_refused(self) -> None:+ with pytest.raises(UnmappableTableError, match="'tags'"):+ refuse_columns_an_import_changes({"tags": "Array(String)", "day": "Date"}, "jsonl")def test_csv_header_names_with_surrounding_spaces_are_refused(self) -> None:location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.pyindex 1a20cc0e9b47..46e9c8eb23dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py@@ -215,7 +215,7 @@ class FileUploadSource(SimpleSource[FileUploadInputs]):has_header=True,allow_double_quotes=csv_metadata.get("allow_double_quotes") is not False,column_types=csv_column_types_from_metadata(csv_metadata.get("column_types"))- if file_format == "csv"+ if file_format in ("csv", "jsonl")else (),)diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.pyindex 2b9252bb7c7f..e14aeb70ac6d 100644--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py@@ -748,10 +748,10 @@ class TestUploadedTableMigration(_MigrationTest):"'tags' is a Array",),(- "json_dates",+ "json_unknown_time_zone",DataWarehouseTableFormat.JSON,- {"day": {"clickhouse": "Nullable(Date)", "hogql": "DateDatabaseField"}},- "'day' is a Date",+ {"at": {"clickhouse": "DateTime('Nowhere/Land')", "hogql": "DateTimeDatabaseField"}},+ "time zone 'Nowhere/Land'",),("padded_csv_header",@@ -780,6 +780,28 @@ class TestUploadedTableMigration(_MigrationTest):assert reason in (outcome.status.reason or "")assert not ExternalDataSource.objects.filter(team=self.team).exists()+ @parameterized.expand(+ [+ ("date", "Nullable(Date)", "date"),+ ("datetime", "DateTime64(3)", "datetime"),+ ("datetime_with_time_zone", "DateTime('Europe/London')", {"kind": "datetime", "timezone": "Europe/London"}),+ ]+ )+ def test_an_uploaded_json_table_with_a_date_column_keeps_the_date_type(+ self, _name: str, clickhouse_type: str, stored: Any+ ) -> None:+ table = self._uploaded("events", "managed/team_{team}/events.jsonl", DataWarehouseTableFormat.JSON)+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(+ columns={"day": {"clickhouse": clickhouse_type, "hogql": "DateDatabaseField"}}+ )++ [outcome] = self._migrate()++ assert outcome.status.state == TableMigrationState.AWAITING_SYNC+ schema_metadata = _schema_for(table).schema_metadata+ assert schema_metadata is not None+ assert schema_metadata["csv"]["column_types"] == {"day": stored}+def test_teams_without_the_flag_keep_their_uploaded_tables(self) -> None:self.flag_missing.return_value = Trueself._uploaded("orders", "file_uploads/team_{team}/{upload}/orders.csv")