Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): keep date columns when migrating self-managed json tables (#125618)

Tom Owersd1e6d1e0a59a
diff --git a/products/warehouse_sources/backend/file_upload_sources.py b/products/warehouse_sources/backend/file_upload_sources.py
index 2286008c633f..ec3222e5a571 100644
--- a/products/warehouse_sources/backend/file_upload_sources.py
+++ b/products/warehouse_sources/backend/file_upload_sources.py
@@ -60,7 +60,9 @@ from products.warehouse_sources.backend.presentation.views.external_data_source.
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import CsvOptions
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (
+ CsvColumnType,
csv_column_types_from_columns,
+ json_column_types_from_columns,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (
TableDefinition,
@@ -157,6 +159,14 @@ def pending_upload_job_inputs(file_format: UploadedFileFormat, schema_name: str)
return {"upload_id": "", "filename": "", "file_format": file_format, "schema_name": schema_name}
+def _stored_column_types(read: str, read_settings: UploadReadSettings) -> tuple[CsvColumnType, ...]:
+ if read == "csv":
+ return csv_column_types_from_columns(read_settings.columns)
+ if read == "jsonl":
+ return json_column_types_from_columns(read_settings.columns)
+ return ()
+
+
def upload_schema_metadata(
schema_name: str,
filename: str,
@@ -176,7 +186,7 @@ def upload_schema_metadata(
csv=CsvOptions(
has_header=True,
allow_double_quotes=read_settings.allow_double_quotes is not False,
- column_types=csv_column_types_from_columns(read_settings.columns) if read == "csv" else (),
+ column_types=_stored_column_types(read, read_settings),
),
pattern=None,
source_format=read,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py
index 2a58328db1ff..9ead4fe8a9b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/csv_types.py
@@ -12,6 +12,9 @@ from posthog.dataclasses import frozen
CsvColumnKind = Literal["integer", "unsigned", "float", "decimal", "boolean", "date", "datetime"]
CSV_COLUMN_KINDS: frozenset[str] = frozenset(get_args(CsvColumnKind))
+# The CSV column kinds a JSON Lines column is cast to. JSON already types numbers and booleans.
+JSON_CAST_KINDS: frozenset[str] = frozenset({"date", "datetime"})
+
_WRAPPER = re.compile(r"^(?:Nullable|LowCardinality)\((?P<inner>.*)\)$")
_TIMEZONE_ARGUMENT = re.compile(r"'(?P<timezone>[^']+)'")
_DECIMAL_ARGUMENTS = re.compile(r"^\s*(?P<first>\d+)\s*(?:,\s*(?P<second>\d+)\s*)?$")
@@ -180,14 +183,25 @@ def stored_clickhouse_types(columns: Mapping[str, Any] | None) -> dict[str, str]
def csv_column_types_from_columns(columns: Mapping[str, Any] | None) -> tuple[CsvColumnType, ...]:
"""The casts for a warehouse table's stored columns, in their stored order."""
+ return _column_types(columns, only_kinds=None)
+
+
+def _column_types(columns: Mapping[str, Any] | None, only_kinds: frozenset[str] | None) -> tuple[CsvColumnType, ...]:
column_types: list[CsvColumnType] = []
for name, clickhouse_type in stored_clickhouse_types(columns).items():
+ if only_kinds is not None and _CLICKHOUSE_KINDS.get(_unwrap(clickhouse_type).base) not in only_kinds:
+ continue
column_type = csv_column_type(name, clickhouse_type)
if column_type is not None:
column_types.append(column_type)
return tuple(column_types)
+def json_column_types_from_columns(columns: Mapping[str, Any] | None) -> tuple[CsvColumnType, ...]:
+ """The casts for the date and time columns of a warehouse table. Other JSON columns keep their JSON type."""
+ return _column_types(columns, only_kinds=JSON_CAST_KINDS)
+
+
def _column_type_from_metadata(name: str, raw: object) -> CsvColumnType | None:
if isinstance(raw, str):
return CsvColumnType(name=name, kind=cast(CsvColumnKind, raw)) if raw in CSV_COLUMN_KINDS else None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
index ffc468836a4e..75a31846e6c3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
@@ -9,6 +9,7 @@ from typing import IO, Any, Literal, cast
import pyarrow as pa
import pyarrow.csv as pa_csv
+import pyarrow.compute as pc
import pyarrow.parquet as pq
from structlog.types import FilteringBoundLogger
@@ -23,6 +24,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
FileFormat,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import (
+ JSON_CAST_KINDS,
+ CsvColumnType,
cast_csv_columns,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import is_compressed
@@ -244,12 +247,46 @@ def _as_rows(value: Any) -> Iterator[dict[str, Any]]:
yield {"value": value}
+def _typed_jsonl(table: pa.Table, column_types: tuple[CsvColumnType, ...], nulled_values: dict[str, int]) -> pa.Table:
+ # Only the date and time columns are cast. A JSON number or boolean already has its type.
+ casts = tuple(column_type for column_type in column_types if column_type.kind in JSON_CAST_KINDS)
+ for column_type in casts:
+ if column_type.name not in table.column_names:
+ continue
+ index = table.schema.get_field_index(column_type.name)
+ if not pa.types.is_string(table.schema.field(index).type):
+ table = table.set_column(index, column_type.name, pc.cast(table.column(index), pa.string(), safe=False))
+ result = cast_csv_columns(table, casts)
+ for column, count in result.nulled_values.items():
+ nulled_values[column] = nulled_values.get(column, 0) + count
+ return result.table
+
+
def iter_jsonl_tables(
stream: IO[bytes],
file_path: str = "",
max_rows: int = JSONL_CHUNK_ROWS,
max_bytes: int = JSONL_CHUNK_BYTES,
+ column_types: tuple[CsvColumnType, ...] = (),
+ logger: FilteringBoundLogger | None = None,
) -> Iterator[pa.Table]:
+ """Stream a JSON Lines file as tables, casting the date and time columns in `column_types`.
+
+ A JSON file holds a date as text. A table moved from a self-managed table keeps the date types
+ ClickHouse read, so those columns are cast like CSV columns, and a value that is not a date is null.
+ """
+ nulled_values: dict[str, int] = {}
+ for table in _iter_untyped_jsonl_tables(stream, file_path, max_rows, max_bytes):
+ yield _typed_jsonl(table, column_types, nulled_values) if column_types else table
+ if nulled_values and logger is not None:
+ logger.warning(
+ "Read JSON values that don't match their column type as null",
+ file=file_path,
+ nulled_values=nulled_values,
+ )
+
+
+def _iter_untyped_jsonl_tables(stream: IO[bytes], file_path: str, max_rows: int, max_bytes: int) -> Iterator[pa.Table]:
rows: list[dict[str, Any]] = []
pending_bytes = 0
line_number = 0
@@ -362,7 +399,7 @@ def iter_file_tables(
delimiter = csv_options.delimiter or ("\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ",")
tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)
else:
- tables = iter_jsonl_tables(binary, stored.path)
+ tables = iter_jsonl_tables(binary, stored.path, column_types=csv_options.column_types, logger=logger)
rows_read = 0
try:
for table in tables:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
index b7d76d62eb39..b8e6e16ff125 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/self_managed.py
@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
UnsupportedCsvColumnError,
clickhouse_base_type,
csv_column_types_from_columns,
+ json_column_types_from_columns,
stored_clickhouse_types,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.discovery import (
@@ -65,8 +66,6 @@ _FORMATS: dict[str, FileFormatChoice] = {
}
# Column types the import pipeline writes as JSON strings, which breaks queries that read their parts.
_NESTED_TYPES = frozenset({"Array", "Map", "Tuple", "Nested", "JSON", "Object", "Variant", "Dynamic"})
-# Types ClickHouse reads from JSON strings, which the JSON import keeps as strings.
-_JSON_TEXT_TYPES = frozenset({"Date", "Date32", "DateTime", "DateTime64"})
# Columns the import adds to every row. A stored column of the same name would lose its values to them.
_FILE_COLUMNS = frozenset({FILE_PATH_COLUMN, FILE_MODIFIED_AT_COLUMN, FILE_ROW_NUMBER_COLUMN})
@@ -195,24 +194,12 @@ def _refuse_padded_header_names(columns: Mapping[str, Any] | None) -> None:
)
-def _refuse_json_text_types(columns: Mapping[str, Any] | None) -> None:
- # A JSON file holds dates and times as strings, and the JSON import keeps them as strings.
- for name, clickhouse_type in stored_clickhouse_types(columns).items():
- base = clickhouse_base_type(clickhouse_type)
- if base in _JSON_TEXT_TYPES:
- raise UnmappableTableError(
- f"the column '{name}' is a {base}, and the JSON import would store it as text, not as a {base}"
- )
-
-
def refuse_columns_an_import_changes(columns: Mapping[str, Any] | None, file_format: FileFormatChoice) -> None:
"""Refuse a file whose import would change the columns a table read in place exposes."""
_refuse_nested_columns(columns)
_refuse_file_column_names(columns)
if file_format == "csv":
_refuse_padded_header_names(columns)
- if file_format == "jsonl":
- _refuse_json_text_types(columns)
def table_definition_for(
@@ -250,13 +237,16 @@ def table_definition_for(
except ObjectStorageError as e:
raise UnmappableTableError(str(e)) from e
try:
- column_types = csv_column_types_from_columns(columns) if file_format in ("csv", "csv_no_header") else ()
+ if file_format == "jsonl":
+ column_types = json_column_types_from_columns(columns)
+ elif file_format in ("csv", "csv_no_header"):
+ column_types = csv_column_types_from_columns(columns)
+ else:
+ column_types = ()
except UnsupportedCsvColumnError as e:
raise UnmappableTableError(str(e)) from e
if file_format == "csv":
_refuse_padded_header_names(columns)
- if file_format == "jsonl":
- _refuse_json_text_types(columns)
return TableDefinition(
name=name,
kind="files",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index b51fdf63931f..173c3b382d49 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -2,13 +2,15 @@ import io
import os
import csv
import gzip
+import json
import tempfile
-from datetime import UTC, datetime, timedelta
+from datetime import UTC, date, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
+from unittest.mock import MagicMock
from django.test import override_settings
@@ -34,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
ObjectStoreConfig,
ObjectStoreProvider,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.csv_types import CsvColumnType
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.delta import (
ObjectStorageDeltaCursor,
iter_delta_tables,
@@ -574,6 +577,66 @@ class TestJsonl:
{"value": "five"},
]
+ @parameterized.expand(
+ [
+ (
+ "date",
+ CsvColumnType(name="d", kind="date"),
+ [{"d": "2024-03-01"}, {"d": None}, {"d": "2024-03-02"}],
+ [date(2024, 3, 1), None, date(2024, 3, 2)],
+ 0,
+ ),
+ (
+ "datetime_in_utc",
+ CsvColumnType(name="d", kind="datetime"),
+ [{"d": "2024-03-01 10:30:00"}, {"d": "2024-03-01T10:30:00Z"}, {"d": None}],
+ [datetime(2024, 3, 1, 10, 30, tzinfo=UTC)] * 2 + [None],
+ 0,
+ ),
+ (
+ "datetime_in_a_time_zone",
+ CsvColumnType(name="d", kind="datetime", timezone="America/New_York"),
+ [{"d": "2024-07-01 10:30:00"}],
+ [datetime(2024, 7, 1, 14, 30, tzinfo=UTC)],
+ 0,
+ ),
+ (
+ "bad_values_become_null",
+ CsvColumnType(name="d", kind="date"),
+ [{"d": "2024-03-01"}, {"d": "not a date"}, {"d": "2024-13-45"}],
+ [date(2024, 3, 1), None, None],
+ 2,
+ ),
+ (
+ "column_with_only_nulls",
+ CsvColumnType(name="d", kind="date"),
+ [{"d": None}, {"d": None}],
+ [None, None],
+ 0,
+ ),
+ ]
+ )
+ def test_stored_date_types_cast_json_text(
+ self, _name: str, column_type: CsvColumnType, rows: list[dict], expected: list, nulled: int
+ ) -> None:
+ content = b"\n".join(json.dumps(row).encode() for row in rows) + b"\n"
+ logger = MagicMock()
+
+ [table] = iter_jsonl_tables(io.BytesIO(content), "x.jsonl", column_types=(column_type,), logger=logger)
+
+ assert table.schema.field("d").type == column_type.arrow_type
+ assert table.column("d").to_pylist() == expected
+ assert logger.warning.called is bool(nulled)
+
+ def test_columns_without_a_stored_type_stay_as_they_are(self) -> None:
+ content = b'{"d": "2024-03-01", "n": 1}\n'
+
+ [table] = iter_jsonl_tables(
+ io.BytesIO(content), "x.jsonl", column_types=(CsvColumnType(name="n", kind="integer"),)
+ )
+
+ assert table.to_pylist() == [{"d": "2024-03-01", "n": 1}]
+
def test_a_too_deeply_nested_record_is_a_parse_error(self) -> None:
with pytest.raises(ObjectStorageError, match=FILE_PARSE_ERROR):
list(iter_jsonl_tables(io.BytesIO(b"[" * 100_000 + b"]" * 100_000 + b"\n"), "deep.jsonl"))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
index 164db21c0586..108ca33d6b37 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
@@ -59,6 +59,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
UnmappableTableError,
needs_preserved_column_names,
parse_self_managed_url,
+ refuse_columns_an_import_changes,
source_format_for,
table_definition_for,
)
@@ -539,11 +540,18 @@ class TestTableDefinition:
table_definition_for("orders", parse_self_managed_url(url), table_format, columns, None)
@parameterized.expand([("Date",), ("Nullable(Date32)",), ("DateTime",), ("Nullable(DateTime64(9))",)])
- def test_json_dates_and_times_are_refused(self, column_type: str) -> None:
+ def test_json_dates_and_times_are_cast_not_refused(self, column_type: str) -> None:
location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.jsonl")
+ columns = {"id": "Int64", "created": column_type}
- with pytest.raises(UnmappableTableError, match="'created'"):
- table_definition_for("orders", location, "JSONEachRow", {"id": "Int64", "created": column_type}, None)
+ refuse_columns_an_import_changes(columns, "jsonl")
+ definition = table_definition_for("orders", location, "JSONEachRow", columns, None)
+
+ assert [column.name for column in definition.csv.column_types] == ["created"]
+
+ def test_json_nested_columns_are_still_refused(self) -> None:
+ with pytest.raises(UnmappableTableError, match="'tags'"):
+ refuse_columns_an_import_changes({"tags": "Array(String)", "day": "Date"}, "jsonl")
def test_csv_header_names_with_surrounding_spaces_are_refused(self) -> None:
location = parse_self_managed_url("https://storage.googleapis.com/lake/orders/*.csv")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py
index 1a20cc0e9b47..46e9c8eb23dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py
@@ -215,7 +215,7 @@ class FileUploadSource(SimpleSource[FileUploadInputs]):
has_header=True,
allow_double_quotes=csv_metadata.get("allow_double_quotes") is not False,
column_types=csv_column_types_from_metadata(csv_metadata.get("column_types"))
- if file_format == "csv"
+ if file_format in ("csv", "jsonl")
else (),
)
diff --git a/products/warehouse_sources/backend/tests/test_self_managed_migration.py b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
index 2b9252bb7c7f..e14aeb70ac6d 100644
--- a/products/warehouse_sources/backend/tests/test_self_managed_migration.py
+++ b/products/warehouse_sources/backend/tests/test_self_managed_migration.py
@@ -748,10 +748,10 @@ class TestUploadedTableMigration(_MigrationTest):
"'tags' is a Array",
),
(
- "json_dates",
+ "json_unknown_time_zone",
DataWarehouseTableFormat.JSON,
- {"day": {"clickhouse": "Nullable(Date)", "hogql": "DateDatabaseField"}},
- "'day' is a Date",
+ {"at": {"clickhouse": "DateTime('Nowhere/Land')", "hogql": "DateTimeDatabaseField"}},
+ "time zone 'Nowhere/Land'",
),
(
"padded_csv_header",
@@ -780,6 +780,28 @@ class TestUploadedTableMigration(_MigrationTest):
assert reason in (outcome.status.reason or "")
assert not ExternalDataSource.objects.filter(team=self.team).exists()
+ @parameterized.expand(
+ [
+ ("date", "Nullable(Date)", "date"),
+ ("datetime", "DateTime64(3)", "datetime"),
+ ("datetime_with_time_zone", "DateTime('Europe/London')", {"kind": "datetime", "timezone": "Europe/London"}),
+ ]
+ )
+ def test_an_uploaded_json_table_with_a_date_column_keeps_the_date_type(
+ self, _name: str, clickhouse_type: str, stored: Any
+ ) -> None:
+ table = self._uploaded("events", "managed/team_{team}/events.jsonl", DataWarehouseTableFormat.JSON)
+ DataWarehouseTable.raw_objects.filter(pk=table.pk).update(
+ columns={"day": {"clickhouse": clickhouse_type, "hogql": "DateDatabaseField"}}
+ )
+
+ [outcome] = self._migrate()
+
+ assert outcome.status.state == TableMigrationState.AWAITING_SYNC
+ schema_metadata = _schema_for(table).schema_metadata
+ assert schema_metadata is not None
+ assert schema_metadata["csv"]["column_types"] == {"day": stored}
+
def test_teams_without_the_flag_keep_their_uploaded_tables(self) -> None:
self.flag_missing.return_value = True
self._uploaded("orders", "file_uploads/team_{team}/{upload}/orders.csv")