Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): stop flexible checksum validation on ranged s3 reads (#117551)

Tom Owersb4244da462ab
diff --git a/products/data_warehouse/backend/s3.py b/products/data_warehouse/backend/s3.py
index b1de72fbf04e..b707419c2ccb 100644
--- a/products/data_warehouse/backend/s3.py
+++ b/products/data_warehouse/backend/s3.py
@@ -14,6 +14,17 @@ import botocore.exceptions
from products.data_warehouse.backend.s3_proxy import boto_proxy_config_kwargs
+# A full-object checksum we write on PutObject (botocore's default since it added flexible
+# checksums) comes back on a ranged GetObject response too, but only covers the whole object, not
+# the range fsspec actually read. botocore's default response validation checks the range bytes
+# against it anyway and raises FlexibleChecksumError on every parquet footer read, which is always
+# ranged. "when_required" matches the request side, so GetObject (which never requires it) stops
+# computing and checking a checksum it can't validate correctly for a partial read.
+_CHECKSUM_CONFIG_KWARGS: dict[str, object] = {
+ "request_checksum_calculation": "when_required",
+ "response_checksum_validation": "when_required",
+}
+
def get_s3_client(*, endpoint_url: Optional[str] = None, skip_instance_cache: bool = False):
# Defaults for localhost dev and test suites
@@ -22,6 +33,7 @@ def get_s3_client(*, endpoint_url: Optional[str] = None, skip_instance_cache: bo
key=settings.DATAWAREHOUSE_LOCAL_ACCESS_KEY,
secret=settings.DATAWAREHOUSE_LOCAL_ACCESS_SECRET,
endpoint_url=settings.OBJECT_STORAGE_ENDPOINT,
+ config_kwargs=_CHECKSUM_CONFIG_KWARGS,
# skip_instance_cache ensures a fresh S3FileSystem instance is created each time,
# avoiding "Event loop is closed" errors when the event loop changes between async
# operations (e.g., between test modules with module-scoped event loops).
@@ -40,7 +52,7 @@ def get_s3_client(*, endpoint_url: Optional[str] = None, skip_instance_cache: bo
# (and the open sockets/file handles it holds) isn't worth the connection-reuse it'd otherwise buy.
extra = {"endpoint_url": endpoint_url} if endpoint_url is not None else {}
return s3fs.S3FileSystem(
- config_kwargs=boto_proxy_config_kwargs(endpoint_url=endpoint_url),
+ config_kwargs={**_CHECKSUM_CONFIG_KWARGS, **boto_proxy_config_kwargs(endpoint_url=endpoint_url)},
skip_instance_cache=skip_instance_cache,
**extra,
)
@@ -75,7 +87,7 @@ async def _shared_async_s3_client(endpoint_url: Optional[str]) -> s3fs.S3FileSys
s3 = s3fs.S3FileSystem(
asynchronous=True,
skip_instance_cache=True,
- config_kwargs=boto_proxy_config_kwargs(endpoint_url=endpoint_url),
+ config_kwargs={**_CHECKSUM_CONFIG_KWARGS, **boto_proxy_config_kwargs(endpoint_url=endpoint_url)},
**extra,
)
await s3.set_session()
@@ -99,6 +111,7 @@ async def aget_s3_client(*, fresh_instance: bool = False, endpoint_url: Optional
key=settings.DATAWAREHOUSE_LOCAL_ACCESS_KEY,
secret=settings.DATAWAREHOUSE_LOCAL_ACCESS_SECRET,
endpoint_url=settings.OBJECT_STORAGE_ENDPOINT,
+ config_kwargs=_CHECKSUM_CONFIG_KWARGS,
skip_instance_cache=True,
asynchronous=True,
)
@@ -108,7 +121,7 @@ async def aget_s3_client(*, fresh_instance: bool = False, endpoint_url: Optional
s3 = s3fs.S3FileSystem(
asynchronous=True,
skip_instance_cache=True,
- config_kwargs=boto_proxy_config_kwargs(endpoint_url=endpoint_url),
+ config_kwargs={**_CHECKSUM_CONFIG_KWARGS, **boto_proxy_config_kwargs(endpoint_url=endpoint_url)},
**extra,
)
await s3.set_session()
diff --git a/products/data_warehouse/backend/test/test_s3.py b/products/data_warehouse/backend/test/test_s3.py
index 93e04afa9c9c..9e1c19117607 100644
--- a/products/data_warehouse/backend/test/test_s3.py
+++ b/products/data_warehouse/backend/test/test_s3.py
@@ -84,3 +84,19 @@ class TestSharedAsyncClientPerLoop:
assert len(s3_module._LOOP_S3_CLIENTS) == 0
fresh._s3creator.__aexit__.assert_awaited_once()
+
+
+class TestChecksumConfig:
+ # botocore's default flexible-checksum validation checks a GetObject response against the
+ # full-object checksum we wrote on upload, even when fsspec only requested a byte range (as a
+ # parquet footer read always does) — a correct read then raises FlexibleChecksumError because
+ # the range bytes don't match a checksum computed over the whole object.
+ @patch(f"{_S3}.boto_proxy_config_kwargs", return_value={})
+ @override_settings(USE_LOCAL_SETUP=False)
+ def test_get_s3_client_disables_flexible_checksum_validation(self, _proxy: MagicMock) -> None:
+ with patch(f"{_S3}.s3fs.S3FileSystem", _filesystem_factory()) as filesystem:
+ s3_module.get_s3_client()
+
+ config_kwargs = filesystem.call_args.kwargs["config_kwargs"]
+ assert config_kwargs["response_checksum_validation"] == "when_required"
+ assert config_kwargs["request_checksum_calculation"] == "when_required"