:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): stop flexible checksum validation on ranged s3 reads (#117551)
Tom Owersb4244da462ab
diff --git a/products/data_warehouse/backend/s3.py b/products/data_warehouse/backend/s3.pyindex b1de72fbf04e..b707419c2ccb 100644--- a/products/data_warehouse/backend/s3.py+++ b/products/data_warehouse/backend/s3.py@@ -14,6 +14,17 @@ import botocore.exceptionsfrom products.data_warehouse.backend.s3_proxy import boto_proxy_config_kwargs+# A full-object checksum we write on PutObject (botocore's default since it added flexible+# checksums) comes back on a ranged GetObject response too, but only covers the whole object, not+# the range fsspec actually read. botocore's default response validation checks the range bytes+# against it anyway and raises FlexibleChecksumError on every parquet footer read, which is always+# ranged. "when_required" matches the request side, so GetObject (which never requires it) stops+# computing and checking a checksum it can't validate correctly for a partial read.+_CHECKSUM_CONFIG_KWARGS: dict[str, object] = {+ "request_checksum_calculation": "when_required",+ "response_checksum_validation": "when_required",+}+def get_s3_client(*, endpoint_url: Optional[str] = None, skip_instance_cache: bool = False):# Defaults for localhost dev and test suites@@ -22,6 +33,7 @@ def get_s3_client(*, endpoint_url: Optional[str] = None, skip_instance_cache: bokey=settings.DATAWAREHOUSE_LOCAL_ACCESS_KEY,secret=settings.DATAWAREHOUSE_LOCAL_ACCESS_SECRET,endpoint_url=settings.OBJECT_STORAGE_ENDPOINT,+ config_kwargs=_CHECKSUM_CONFIG_KWARGS,# skip_instance_cache ensures a fresh S3FileSystem instance is created each time,# avoiding "Event loop is closed" errors when the event loop changes between async# operations (e.g., between test modules with module-scoped event loops).@@ -40,7 +52,7 @@ def get_s3_client(*, endpoint_url: Optional[str] = None, skip_instance_cache: bo# (and the open sockets/file handles it holds) isn't worth the connection-reuse it'd otherwise buy.extra = {"endpoint_url": endpoint_url} if endpoint_url is not None else {}return s3fs.S3FileSystem(- config_kwargs=boto_proxy_config_kwargs(endpoint_url=endpoint_url),+ config_kwargs={**_CHECKSUM_CONFIG_KWARGS, **boto_proxy_config_kwargs(endpoint_url=endpoint_url)},skip_instance_cache=skip_instance_cache,**extra,)@@ -75,7 +87,7 @@ async def _shared_async_s3_client(endpoint_url: Optional[str]) -> s3fs.S3FileSyss3 = s3fs.S3FileSystem(asynchronous=True,skip_instance_cache=True,- config_kwargs=boto_proxy_config_kwargs(endpoint_url=endpoint_url),+ config_kwargs={**_CHECKSUM_CONFIG_KWARGS, **boto_proxy_config_kwargs(endpoint_url=endpoint_url)},**extra,)await s3.set_session()@@ -99,6 +111,7 @@ async def aget_s3_client(*, fresh_instance: bool = False, endpoint_url: Optionalkey=settings.DATAWAREHOUSE_LOCAL_ACCESS_KEY,secret=settings.DATAWAREHOUSE_LOCAL_ACCESS_SECRET,endpoint_url=settings.OBJECT_STORAGE_ENDPOINT,+ config_kwargs=_CHECKSUM_CONFIG_KWARGS,skip_instance_cache=True,asynchronous=True,)@@ -108,7 +121,7 @@ async def aget_s3_client(*, fresh_instance: bool = False, endpoint_url: Optionals3 = s3fs.S3FileSystem(asynchronous=True,skip_instance_cache=True,- config_kwargs=boto_proxy_config_kwargs(endpoint_url=endpoint_url),+ config_kwargs={**_CHECKSUM_CONFIG_KWARGS, **boto_proxy_config_kwargs(endpoint_url=endpoint_url)},**extra,)await s3.set_session()diff --git a/products/data_warehouse/backend/test/test_s3.py b/products/data_warehouse/backend/test/test_s3.pyindex 93e04afa9c9c..9e1c19117607 100644--- a/products/data_warehouse/backend/test/test_s3.py+++ b/products/data_warehouse/backend/test/test_s3.py@@ -84,3 +84,19 @@ class TestSharedAsyncClientPerLoop:assert len(s3_module._LOOP_S3_CLIENTS) == 0fresh._s3creator.__aexit__.assert_awaited_once()+++class TestChecksumConfig:+ # botocore's default flexible-checksum validation checks a GetObject response against the+ # full-object checksum we wrote on upload, even when fsspec only requested a byte range (as a+ # parquet footer read always does) — a correct read then raises FlexibleChecksumError because+ # the range bytes don't match a checksum computed over the whole object.+ @patch(f"{_S3}.boto_proxy_config_kwargs", return_value={})+ @override_settings(USE_LOCAL_SETUP=False)+ def test_get_s3_client_disables_flexible_checksum_validation(self, _proxy: MagicMock) -> None:+ with patch(f"{_S3}.s3fs.S3FileSystem", _filesystem_factory()) as filesystem:+ s3_module.get_s3_client()++ config_kwargs = filesystem.call_args.kwargs["config_kwargs"]+ assert config_kwargs["response_checksum_validation"] == "when_required"+ assert config_kwargs["request_checksum_calculation"] == "when_required"