Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): read hive partition folders that start with an underscore (#121174)

Tom Owersd89ea8abf71b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
index a6090e29726b..da97a7ef87e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
@@ -48,6 +48,7 @@ EXTENSION_FORMATS: dict[str, FileFormat] = {
}
_FILE_FORMAT_CHOICES: frozenset[str] = frozenset(get_args(FileFormatChoice))
+_HIVE_PARTITION_FOLDER = re.compile(r"[^.=][^=]*=.*")
@frozen
@@ -154,9 +155,19 @@ def _table_format(path: str, choice: FileFormatChoice) -> FileFormatChoice | Non
return choice
-def _is_hidden(segments: Iterable[str]) -> bool:
+def _is_hidden_name(segment: str) -> bool:
# `_SUCCESS`, `_committed_*`, `.crc` and `_temporary/` are Spark and Hadoop bookkeeping, not data.
- return any(segment.startswith(("_", ".")) for segment in segments)
+ return segment.startswith(("_", "."))
+
+
+def _is_hidden_directory(segment: str) -> bool:
+ # A hive partition folder such as `_ph_partition_key=2026-w36` holds data, even when its key starts with `_`.
+ return _is_hidden_name(segment) and _HIVE_PARTITION_FOLDER.fullmatch(segment) is None
+
+
+def _is_hidden(path: str) -> bool:
+ *directories, file_name = path.split("/")
+ return _is_hidden_name(file_name) or any(_is_hidden_directory(directory) for directory in directories)
def _delta_roots(objects: Iterable[StoredObject]) -> list[str]:
@@ -165,7 +176,7 @@ def _delta_roots(objects: Iterable[StoredObject]) -> list[str]:
segments = stored.path.split("/")
if DELTA_LOG_DIRECTORY in segments[:-1]:
root_segments = segments[: segments.index(DELTA_LOG_DIRECTORY)]
- if not _is_hidden(root_segments):
+ if not any(_is_hidden_directory(segment) for segment in root_segments):
roots.add("/".join(root_segments))
return sorted(roots)
@@ -252,7 +263,7 @@ def group_tables(
data_files: list[tuple[StoredObject, FileFormatChoice]] = []
for stored in ordered:
segments = stored.path.split("/")
- if not segments[-1] or _is_hidden(segments) or _is_under(stored.path, delta_roots):
+ if not segments[-1] or _is_hidden(stored.path) or _is_under(stored.path, delta_roots):
continue
if pattern is not None and not fnmatch.fnmatchcase(stored.path, pattern):
continue
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index a14165f83b4e..b9afff82a43e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -179,6 +179,68 @@ class TestGroupTables:
DiscoveryOptions(),
[("out", "files", "folder", "out", "csv", ["out/part-0.csv"])],
),
+ (
+ "a_hive_partition_folder_whose_key_starts_with_an_underscore_holds_data",
+ ["orders/_ph_partition_key=2024-w10/part.parquet", "orders/_ph_partition_key=2024-w11/part.parquet"],
+ DiscoveryOptions(),
+ [
+ (
+ "orders",
+ "files",
+ "folder",
+ "orders",
+ "parquet",
+ [
+ "orders/_ph_partition_key=2024-w10/part.parquet",
+ "orders/_ph_partition_key=2024-w11/part.parquet",
+ ],
+ ),
+ ],
+ ),
+ (
+ "nested_underscore_hive_partitions_fold_into_the_table_above",
+ ["orders/_a=1/_b=2/part.parquet", "orders/_a=1/_b=3/part.parquet"],
+ DiscoveryOptions(),
+ [
+ (
+ "orders",
+ "files",
+ "folder",
+ "orders",
+ "parquet",
+ ["orders/_a=1/_b=2/part.parquet", "orders/_a=1/_b=3/part.parquet"],
+ ),
+ ],
+ ),
+ (
+ "bookkeeping_stays_skipped_inside_an_underscore_partition",
+ [
+ "orders/_ph_partition_key=2024-w10/_SUCCESS",
+ "orders/_ph_partition_key=2024-w10/_committed_123",
+ "orders/_ph_partition_key=2024-w10/.part.parquet.crc",
+ "orders/_ph_partition_key=2024-w10/_temporary/x.parquet",
+ "orders/_temporary/_a=1/x.parquet",
+ "orders/.hidden=1/x.parquet",
+ "orders/_ph_partition_key=2024-w10/part.parquet",
+ ],
+ DiscoveryOptions(),
+ [
+ (
+ "orders",
+ "files",
+ "folder",
+ "orders",
+ "parquet",
+ ["orders/_ph_partition_key=2024-w10/part.parquet"],
+ ),
+ ],
+ ),
+ (
+ "a_delta_table_under_an_underscore_partition_folder_is_found",
+ ["_region=eu/sales/_delta_log/0.json", "_temporary/sales/_delta_log/0.json"],
+ DiscoveryOptions(),
+ [("region_eu_sales", "delta", None, "_region=eu/sales", "delta", [])],
+ ),
(
"pattern_matches_across_folders",
["a/x.csv", "a/y.json", "b/z.csv"],