:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): read csv and json lines files from fsspec streams (#129966)
Tom Owersbb636c25dd35
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.pyindex 14c28a1a48aa..92f38471ac08 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py@@ -403,7 +403,12 @@ def iter_file_tables(else:tables = iter_parquet_tables(stream, stored.path)else:- binary = cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if is_compressed(stored.path) else stream+ # fsspec files take no size in `readline`, so the bounded line reads need a standard buffered reader.+ binary = (+ cast(IO[bytes], gzip.GzipFile(fileobj=stream))+ if is_compressed(stored.path)+ else cast(IO[bytes], io.BufferedReader(_PrefixedStream(b"", stream)))+ )if file_format == "csv":delimiter = csv_options.delimiter or ("\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ",")tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex e7d1b0786e12..4c82a2708584 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -7,7 +7,7 @@ import tempfilefrom datetime import UTC, date, datetime, timedeltafrom pathlib import Pathfrom types import SimpleNamespace-from typing import Any+from typing import IO, Any, Literal, castimport pytestfrom unittest.mock import MagicMock, call, patch@@ -105,6 +105,22 @@ def stored(path: str) -> StoredObject:return StoredObject(path=path, size=10, modified_at=MODIFIED_AT)+class _FsspecLikeFile:+ """Like an fsspec file, `readline` takes no size."""++ def __init__(self, content: bytes) -> None:+ self._buffer = io.BytesIO(content)++ def readable(self) -> bool:+ return True++ def read(self, size: int = -1) -> bytes:+ return self._buffer.read(size)++ def readline(self) -> bytes:+ return self._buffer.readline()++def local_store(root: Path) -> ObjectStore:return ObjectStore(fs=LocalFileSystem(), root=str(root.resolve()), delta_uri_prefix="", delta_storage_options={})@@ -1123,6 +1139,21 @@ class TestFileSelection:assert len(tables) == 2assert [number for table in tables for number in table["_file_row_number"].to_pylist()] == list(range(rows))+ @parameterized.expand(+ [+ ("csv", "orders.csv", b"id,name\n1,a\n2,b\n", "csv"),+ ("jsonl", "orders.jsonl", b'{"id":1,"name":"a"}\n{"id":2,"name":"b"}\n', "jsonl"),+ ]+ )+ def test_reads_a_stream_whose_readline_takes_no_size(+ self, _name: str, path: str, content: bytes, file_format: Literal["csv", "jsonl"]+ ) -> None:+ stream = cast(IO[bytes], _FsspecLikeFile(content))++ tables = list(iter_file_tables(stream, stored(path), file_format, CsvOptions()))++ assert [str(value) for table in tables for value in table["id"].to_pylist()] == ["1", "2"]+def test_row_numbers_continue_across_the_batches_of_a_gzipped_parquet_file(self) -> None:rows = PARQUET_BATCH_ROWS + 3buffer = io.BytesIO()