:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): read hive partition folders that start with an underscore (#121174)
Tom Owersd89ea8abf71b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.pyindex a6090e29726b..da97a7ef87e6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py@@ -48,6 +48,7 @@ EXTENSION_FORMATS: dict[str, FileFormat] = {}_FILE_FORMAT_CHOICES: frozenset[str] = frozenset(get_args(FileFormatChoice))+_HIVE_PARTITION_FOLDER = re.compile(r"[^.=][^=]*=.*")@frozen@@ -154,9 +155,19 @@ def _table_format(path: str, choice: FileFormatChoice) -> FileFormatChoice | Nonreturn choice-def _is_hidden(segments: Iterable[str]) -> bool:+def _is_hidden_name(segment: str) -> bool:# `_SUCCESS`, `_committed_*`, `.crc` and `_temporary/` are Spark and Hadoop bookkeeping, not data.- return any(segment.startswith(("_", ".")) for segment in segments)+ return segment.startswith(("_", "."))+++def _is_hidden_directory(segment: str) -> bool:+ # A hive partition folder such as `_ph_partition_key=2026-w36` holds data, even when its key starts with `_`.+ return _is_hidden_name(segment) and _HIVE_PARTITION_FOLDER.fullmatch(segment) is None+++def _is_hidden(path: str) -> bool:+ *directories, file_name = path.split("/")+ return _is_hidden_name(file_name) or any(_is_hidden_directory(directory) for directory in directories)def _delta_roots(objects: Iterable[StoredObject]) -> list[str]:@@ -165,7 +176,7 @@ def _delta_roots(objects: Iterable[StoredObject]) -> list[str]:segments = stored.path.split("/")if DELTA_LOG_DIRECTORY in segments[:-1]:root_segments = segments[: segments.index(DELTA_LOG_DIRECTORY)]- if not _is_hidden(root_segments):+ if not any(_is_hidden_directory(segment) for segment in root_segments):roots.add("/".join(root_segments))return sorted(roots)@@ -252,7 +263,7 @@ def group_tables(data_files: list[tuple[StoredObject, FileFormatChoice]] = []for stored in ordered:segments = stored.path.split("/")- if not segments[-1] or _is_hidden(segments) or _is_under(stored.path, delta_roots):+ if not segments[-1] or _is_hidden(stored.path) or _is_under(stored.path, delta_roots):continueif pattern is not None and not fnmatch.fnmatchcase(stored.path, pattern):continuediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex a14165f83b4e..b9afff82a43e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -179,6 +179,68 @@ class TestGroupTables:DiscoveryOptions(),[("out", "files", "folder", "out", "csv", ["out/part-0.csv"])],),+ (+ "a_hive_partition_folder_whose_key_starts_with_an_underscore_holds_data",+ ["orders/_ph_partition_key=2024-w10/part.parquet", "orders/_ph_partition_key=2024-w11/part.parquet"],+ DiscoveryOptions(),+ [+ (+ "orders",+ "files",+ "folder",+ "orders",+ "parquet",+ [+ "orders/_ph_partition_key=2024-w10/part.parquet",+ "orders/_ph_partition_key=2024-w11/part.parquet",+ ],+ ),+ ],+ ),+ (+ "nested_underscore_hive_partitions_fold_into_the_table_above",+ ["orders/_a=1/_b=2/part.parquet", "orders/_a=1/_b=3/part.parquet"],+ DiscoveryOptions(),+ [+ (+ "orders",+ "files",+ "folder",+ "orders",+ "parquet",+ ["orders/_a=1/_b=2/part.parquet", "orders/_a=1/_b=3/part.parquet"],+ ),+ ],+ ),+ (+ "bookkeeping_stays_skipped_inside_an_underscore_partition",+ [+ "orders/_ph_partition_key=2024-w10/_SUCCESS",+ "orders/_ph_partition_key=2024-w10/_committed_123",+ "orders/_ph_partition_key=2024-w10/.part.parquet.crc",+ "orders/_ph_partition_key=2024-w10/_temporary/x.parquet",+ "orders/_temporary/_a=1/x.parquet",+ "orders/.hidden=1/x.parquet",+ "orders/_ph_partition_key=2024-w10/part.parquet",+ ],+ DiscoveryOptions(),+ [+ (+ "orders",+ "files",+ "folder",+ "orders",+ "parquet",+ ["orders/_ph_partition_key=2024-w10/part.parquet"],+ ),+ ],+ ),+ (+ "a_delta_table_under_an_underscore_partition_folder_is_found",+ ["_region=eu/sales/_delta_log/0.json", "_temporary/sales/_delta_log/0.json"],+ DiscoveryOptions(),+ [("region_eu_sales", "delta", None, "_region=eu/sales", "delta", [])],+ ),("pattern_matches_across_folders",["a/x.csv", "a/y.json", "b/z.csv"],