Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): read csv and json lines files from fsspec streams (#129966)

Tom Owersbb636c25dd35
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
index 14c28a1a48aa..92f38471ac08 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/formats.py
@@ -403,7 +403,12 @@ def iter_file_tables(
else:
tables = iter_parquet_tables(stream, stored.path)
else:
- binary = cast(IO[bytes], gzip.GzipFile(fileobj=stream)) if is_compressed(stored.path) else stream
+ # fsspec files take no size in `readline`, so the bounded line reads need a standard buffered reader.
+ binary = (
+ cast(IO[bytes], gzip.GzipFile(fileobj=stream))
+ if is_compressed(stored.path)
+ else cast(IO[bytes], io.BufferedReader(_PrefixedStream(b"", stream)))
+ )
if file_format == "csv":
delimiter = csv_options.delimiter or ("\t" if stored.path.lower().endswith(TSV_SUFFIXES) else ",")
tables = iter_csv_tables(binary, csv_options, delimiter, stored.path, logger)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index e7d1b0786e12..4c82a2708584 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -7,7 +7,7 @@ import tempfile
from datetime import UTC, date, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
-from typing import Any
+from typing import IO, Any, Literal, cast
import pytest
from unittest.mock import MagicMock, call, patch
@@ -105,6 +105,22 @@ def stored(path: str) -> StoredObject:
return StoredObject(path=path, size=10, modified_at=MODIFIED_AT)
+class _FsspecLikeFile:
+ """Like an fsspec file, `readline` takes no size."""
+
+ def __init__(self, content: bytes) -> None:
+ self._buffer = io.BytesIO(content)
+
+ def readable(self) -> bool:
+ return True
+
+ def read(self, size: int = -1) -> bytes:
+ return self._buffer.read(size)
+
+ def readline(self) -> bytes:
+ return self._buffer.readline()
+
+
def local_store(root: Path) -> ObjectStore:
return ObjectStore(fs=LocalFileSystem(), root=str(root.resolve()), delta_uri_prefix="", delta_storage_options={})
@@ -1123,6 +1139,21 @@ class TestFileSelection:
assert len(tables) == 2
assert [number for table in tables for number in table["_file_row_number"].to_pylist()] == list(range(rows))
+ @parameterized.expand(
+ [
+ ("csv", "orders.csv", b"id,name\n1,a\n2,b\n", "csv"),
+ ("jsonl", "orders.jsonl", b'{"id":1,"name":"a"}\n{"id":2,"name":"b"}\n', "jsonl"),
+ ]
+ )
+ def test_reads_a_stream_whose_readline_takes_no_size(
+ self, _name: str, path: str, content: bytes, file_format: Literal["csv", "jsonl"]
+ ) -> None:
+ stream = cast(IO[bytes], _FsspecLikeFile(content))
+
+ tables = list(iter_file_tables(stream, stored(path), file_format, CsvOptions()))
+
+ assert [str(value) for table in tables for value in table["id"].to_pylist()] == ["1", "2"]
+
def test_row_numbers_continue_across_the_batches_of_a_gzipped_parquet_file(self) -> None:
rows = PARQUET_BATCH_ROWS + 3
buffer = io.BytesIO()