Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): take one glob url for object storage sources (#121203)

Tom Owers2567625ae709
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
index 48cb193b10ac..a5ef6fb5112b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/source.py
@@ -11,17 +11,20 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
DiscoveryOptions,
ObjectStoreConfig,
ObjectStoreProvider,
- normalize_prefix,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.fields import (
FEATURE_FLAG,
KEYWORDS,
discovery_options_from,
file_option_fields,
- folder_fields,
secret_field,
source_caption,
text_field,
+ url_field,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
+ S3_EXAMPLE_URL,
+ parse_s3_url,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
@@ -52,17 +55,15 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):
@property
def connection_host_fields(self) -> list[str]:
- # The bucket and region pick where the stored key is sent, so changing either must re-require it.
- return ["bucket", "region"]
+ # The URL picks the bucket and region the stored key is sent to, so changing it must re-require the key.
+ return ["url_pattern"]
def object_store_config(self, config: AmazonS3SourceConfig) -> ObjectStoreConfig:
- return ObjectStoreConfig(
- provider=ObjectStoreProvider.S3,
- bucket=config.bucket.strip(),
- region=config.region.strip(),
+ return ObjectStoreConfig.from_url(
+ ObjectStoreProvider.S3,
+ parse_s3_url(config.url_pattern),
access_key_id=config.access_key_id.strip(),
secret_access_key=config.secret_access_key.strip(),
- prefix=normalize_prefix(config.path),
)
def discovery_options(self, config: AmazonS3SourceConfig) -> DiscoveryOptions:
@@ -82,9 +83,7 @@ class AmazonS3Source(ObjectStorageSource[AmazonS3SourceConfig]):
fields=cast(
list[FieldType],
[
- text_field("bucket", "Bucket", "my-company-exports"),
- text_field("region", "Region", "us-east-1"),
- *folder_fields("bucket"),
+ url_field(S3_EXAMPLE_URL),
text_field("access_key_id", "Access key ID", "AKIA..."),
secret_field("secret_access_key", "Secret access key"),
*file_option_fields(),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
index 1f6ea9c96fb7..3c42e62ca32f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_s3/tests/test_amazon_s3_source.py
@@ -54,17 +54,20 @@ class _LocalAmazonS3Source(AmazonS3Source):
def open_store(self, store_config: StoreConfig) -> ObjectStore:
assert isinstance(store_config, ObjectStoreConfig)
+ assert (store_config.bucket, store_config.region) == ("customer-data", "us-east-1")
folder = self.root / store_config.bucket / store_config.prefix
return ObjectStore(fs=self.fs, root=str(folder).rstrip("/"), delta_uri_prefix="", delta_storage_options={})
+def url(key: str) -> str:
+ return f"https://customer-data.s3.us-east-1.amazonaws.com/{key}"
+
+
def make_config(**overrides: Any) -> AmazonS3SourceConfig:
values: dict[str, Any] = {
- "bucket": "customer-data",
- "region": "us-east-1",
+ "url_pattern": url("exports/"),
"access_key_id": "AKIAEXAMPLE",
"secret_access_key": "secret",
- "path": "exports",
**overrides,
}
return AmazonS3SourceConfig.from_dict(values)
@@ -102,6 +105,65 @@ def test_sync_reads_the_table_discovery_offered(tmp_path: Path, with_metadata: b
assert [(row["id"], row["_file_name"]) for row in rows] == [("1", "orders/a.csv"), ("2", "orders/b.csv")]
+ "key, expected",
+ [
+ # Hive partition folders under the URL's folder fold into one table named after that folder.
+ ("exports/orders/**/*.parquet", {"orders": ("files", "", "**/*.parquet")}),
+ ("exports/*.csv", {"a": ("files", "a.csv", "*.csv")}),
+ ("exports/**.csv", {"a": ("files", "a.csv", "**.csv"), "daily": ("files", "daily", "**.csv")}),
+ ("exports/orders", {"orders": ("files", "orders", None)}),
+ ("exports/a.csv", {"a": ("files", "a.csv", None)}),
+ ("exports/sales/", {"sales": ("delta", "", None)}),
+ ("exports/sales", {"sales": ("delta", "sales", None)}),
+ (
+ "exports/",
+ {
+ "a": ("files", "a.csv", None),
+ "daily": ("files", "daily", None),
+ "orders": ("files", "orders", None),
+ "sales": ("delta", "sales", None),
+ },
+ ),
+ ],
+)
+def test_the_url_selects_the_tables(tmp_path: Path, key: str, expected: dict[str, tuple[str, str, str | None]]) -> None:
+ for relative in (
+ "exports/a.csv",
+ "exports/daily/b.csv",
+ "exports/orders/_part_key=2024-w10/part-0.parquet",
+ "exports/orders/_part_key=2024-w11/part-0.parquet",
+ "exports/orders/_part_key=2024-w11/_SUCCESS",
+ "exports/sales/_delta_log/00000000000000000000.json",
+ "exports/sales/part-0.parquet",
+ ):
+ write(tmp_path, relative, b"")
+
+ schemas = _LocalAmazonS3Source(tmp_path).get_schemas(make_config(url_pattern=url(key)), team_id=1)
+
+ assert {
+ schema.name: (
+ schema.schema_metadata["kind"],
+ schema.schema_metadata["path"],
+ schema.schema_metadata["pattern"],
+ )
+ for schema in schemas
+ if schema.schema_metadata is not None
+ } == expected
+
+
+def test_sync_reads_only_the_files_the_url_matches(tmp_path: Path) -> None:
+ write(tmp_path, "exports/orders/a-1.csv", b"id\n1\n")
+ write(tmp_path, "exports/orders/a-2.csv", b"id\n2\n")
+ source = _LocalAmazonS3Source(tmp_path)
+ config = make_config(url_pattern=url("exports/**-1.csv"))
+ [schema] = source.get_schemas(config, team_id=1)
+
+ rows = read_rows(source, config, schema.name, schema.schema_metadata)
+
+ assert [(row["id"], row["_file_name"]) for row in rows] == [("1", "orders/a-1.csv")]
+
+
def test_sync_keeps_the_settings_the_table_was_discovered_with(tmp_path: Path) -> None:
write(tmp_path, "exports/raw/a.txt", b"1,2\n")
source = _LocalAmazonS3Source(tmp_path)
@@ -117,7 +179,8 @@ def test_sync_keeps_the_settings_the_table_was_discovered_with(tmp_path: Path) -
[
({"exports/a.csv": b"id\n1\n"}, {}, None, None),
({"exports/notes.md": b"# hi"}, {}, None, NO_IMPORTABLE_FILES_ERROR),
- ({}, {"bucket": "Not A Bucket"}, None, INVALID_CONFIG_ERROR),
+ ({}, {"url_pattern": "https://s3.us-east-1.amazonaws.com/Not_A_Bucket/exports/"}, None, INVALID_CONFIG_ERROR),
+ ({}, {"url_pattern": "https://customer-data.s3.amazonaws.com/exports/"}, None, INVALID_CONFIG_ERROR),
({}, {}, PermissionError("Access Denied"), ACCESS_DENIED_ERROR),
({}, {}, FileNotFoundError("NoSuchBucket"), NO_SUCH_BUCKET_ERROR),
],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py
index 5668334c2b78..81a35048a36c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_blob/source.py
@@ -10,17 +10,19 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.bas
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
AzureBlobConfig,
DiscoveryOptions,
- normalize_prefix,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.fields import (
FEATURE_FLAG,
KEYWORDS,
discovery_options_from,
file_option_fields,
- folder_fields,
secret_field,
source_caption,
- text_field,
+ url_field,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
+ AZURE_EXAMPLE_URL,
+ parse_azure_url,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
@@ -51,16 +53,11 @@ class AzureBlobSource(ObjectStorageSource[AzureBlobSourceConfig]):
@property
def connection_host_fields(self) -> list[str]:
- # The account and container pick where the stored key is sent, so changing either must re-require it.
- return ["account_name", "container"]
+ # The URL picks the account and container the stored key is sent to, so changing it must re-require the key.
+ return ["url_pattern"]
def object_store_config(self, config: AzureBlobSourceConfig) -> AzureBlobConfig:
- return AzureBlobConfig(
- account_name=config.account_name.strip().lower(),
- account_key=config.account_key.strip(),
- container=config.container.strip(),
- prefix=normalize_prefix(config.path),
- )
+ return AzureBlobConfig.from_url(parse_azure_url(config.url_pattern), account_key=config.account_key.strip())
def discovery_options(self, config: AzureBlobSourceConfig) -> DiscoveryOptions:
return discovery_options_from(config)
@@ -79,9 +76,7 @@ class AzureBlobSource(ObjectStorageSource[AzureBlobSourceConfig]):
fields=cast(
list[FieldType],
[
- text_field("account_name", "Storage account name", "mycompanystorage"),
- text_field("container", "Container", "exports"),
- *folder_fields("container"),
+ url_field(AZURE_EXAMPLE_URL),
secret_field(
"account_key",
"Account key",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py
index 9012b0e4cb85..3c9f2cf9c353 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare_r2/source.py
@@ -8,21 +8,23 @@ from products.warehouse_sources.backend.facade.source_config import (
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
- AUTO_REGION,
DiscoveryOptions,
ObjectStoreConfig,
ObjectStoreProvider,
- normalize_prefix,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.fields import (
FEATURE_FLAG,
KEYWORDS,
discovery_options_from,
file_option_fields,
- folder_fields,
secret_field,
source_caption,
text_field,
+ url_field,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
+ R2_EXAMPLE_URL,
+ parse_r2_url,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
@@ -48,18 +50,15 @@ class CloudflareR2Source(ObjectStorageSource[CloudflareR2SourceConfig]):
@property
def connection_host_fields(self) -> list[str]:
- # The account and bucket pick where the stored key is sent, so changing either must re-require it.
- return ["account_id", "bucket"]
+ # The URL picks the account and bucket the stored key is sent to, so changing it must re-require the key.
+ return ["url_pattern"]
def object_store_config(self, config: CloudflareR2SourceConfig) -> ObjectStoreConfig:
- return ObjectStoreConfig(
- provider=ObjectStoreProvider.R2,
- bucket=config.bucket.strip(),
- region=AUTO_REGION,
+ return ObjectStoreConfig.from_url(
+ ObjectStoreProvider.R2,
+ parse_r2_url(config.url_pattern),
access_key_id=config.access_key_id.strip(),
secret_access_key=config.secret_access_key.strip(),
- prefix=normalize_prefix(config.path),
- account_id=config.account_id.strip().lower(),
)
def discovery_options(self, config: CloudflareR2SourceConfig) -> DiscoveryOptions:
@@ -79,14 +78,7 @@ class CloudflareR2Source(ObjectStorageSource[CloudflareR2SourceConfig]):
fields=cast(
list[FieldType],
[
- text_field(
- "account_id",
- "Account ID",
- "",
- caption="The 32-character ID on the R2 overview page of the Cloudflare dashboard.",
- ),
- text_field("bucket", "Bucket", "my-company-exports"),
- *folder_fields("bucket"),
+ url_field(R2_EXAMPLE_URL),
text_field(
"access_key_id",
"Access key ID",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
index 98eb90ecb9c0..27ccd852b797 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/config.py
@@ -9,6 +9,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
INVALID_CONFIG_ERROR,
ObjectStorageError,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
+ StorageUrl,
+ select_keys,
+)
FileFormatChoice = Literal["infer", "csv", "csv_no_header", "jsonl", "parquet", "delta"]
# The format one file is read with. A CSV file's header choice lives in `CsvOptions`.
@@ -50,9 +54,15 @@ def _valid_bucket(provider: ObjectStoreProvider, bucket: str) -> bool:
return all(len(part) <= _MAX_BUCKET_NAME_PART_LENGTH for part in bucket.split("."))
-def _validate_prefix(prefix: str) -> None:
- if any(segment in (".", "..") for segment in prefix.split("/")):
- raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: the folder can't contain '.' or '..' segments.")
+def _validate_keys(prefix: str, listing_prefix: str) -> None:
+ if any(segment in (".", "..") for segment in (*prefix.split("/"), listing_prefix)):
+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: the URL path can't contain '.' or '..' folders.")
+ if "/" in listing_prefix:
+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: the listing prefix must be one folder or file name.")
+
+
+def _display_location(container: str, prefix: str, listing_prefix: str) -> str:
+ return "/".join(part for part in (container, prefix, listing_prefix) if part)
@frozen
@@ -71,13 +81,30 @@ class ObjectStoreConfig:
prefix: str = ""
# The Cloudflare account that owns an R2 bucket. Other providers leave it unset.
account_id: str | None = None
+ listing_prefix: str = ""
+ # A ClickHouse glob over paths relative to `prefix`.
+ pattern: str | None = None
+
+ @classmethod
+ def from_url(
+ cls, provider: ObjectStoreProvider, url: StorageUrl, access_key_id: str, secret_access_key: str
+ ) -> "ObjectStoreConfig":
+ keys = select_keys(url.key)
+ return cls(
+ provider=provider,
+ bucket=url.bucket,
+ region=url.region or AUTO_REGION,
+ access_key_id=access_key_id,
+ secret_access_key=secret_access_key,
+ prefix=keys.prefix,
+ account_id=url.account,
+ listing_prefix=keys.listing_prefix,
+ pattern=keys.pattern,
+ )
def __post_init__(self) -> None:
if not _valid_bucket(self.provider, self.bucket):
- raise ObjectStorageError(
- f"{INVALID_CONFIG_ERROR}: '{self.bucket}' isn't a valid bucket name. Enter only the bucket "
- "name, without a URL or a folder."
- )
+ raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{self.bucket}' in the URL isn't a valid bucket name.")
if self.provider == ObjectStoreProvider.S3:
if not _REGION_NAME.fullmatch(self.region):
raise ObjectStorageError(
@@ -88,8 +115,7 @@ class ObjectStoreConfig:
if self.provider == ObjectStoreProvider.R2:
if self.account_id is None or not _R2_ACCOUNT_ID.fullmatch(self.account_id):
raise ObjectStorageError(
- f"{INVALID_CONFIG_ERROR}: the account ID isn't valid. Enter the 32-character Cloudflare "
- "account ID, without a URL."
+ f"{INVALID_CONFIG_ERROR}: the URL must start with the 32-character Cloudflare account ID."
)
elif self.account_id is not None:
raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: this storage provider has no account ID setting.")
@@ -97,7 +123,7 @@ class ObjectStoreConfig:
# never be used to read a customer-chosen bucket.
if not self.access_key_id.strip() or not self.secret_access_key.strip():
raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: enter both the access key ID and the secret access key.")
- _validate_prefix(self.prefix)
+ _validate_keys(self.prefix, self.listing_prefix)
@property
def endpoint_url(self) -> str | None:
@@ -122,7 +148,7 @@ class ObjectStoreConfig:
@property
def display_location(self) -> str:
- return f"{self.bucket}/{self.prefix}" if self.prefix else self.bucket
+ return _display_location(self.bucket, self.prefix, self.listing_prefix)
@frozen
@@ -133,23 +159,36 @@ class AzureBlobConfig:
account_key: str = dataclasses.field(repr=False)
container: str
prefix: str = ""
+ listing_prefix: str = ""
+ # A ClickHouse glob over paths relative to `prefix`.
+ pattern: str | None = None
+
+ @classmethod
+ def from_url(cls, url: StorageUrl, account_key: str) -> "AzureBlobConfig":
+ keys = select_keys(url.key)
+ return cls(
+ account_name=url.account or "",
+ account_key=account_key,
+ container=url.bucket,
+ prefix=keys.prefix,
+ listing_prefix=keys.listing_prefix,
+ pattern=keys.pattern,
+ )
def __post_init__(self) -> None:
if not _AZURE_ACCOUNT_NAME.fullmatch(self.account_name):
raise ObjectStorageError(
- f"{INVALID_CONFIG_ERROR}: '{self.account_name}' isn't a valid storage account name. Enter only "
- "the account name, without a URL."
+ f"{INVALID_CONFIG_ERROR}: '{self.account_name}' in the URL isn't a valid storage account name."
)
if not _AZURE_CONTAINER_NAME.fullmatch(self.container) or "--" in self.container:
raise ObjectStorageError(
- f"{INVALID_CONFIG_ERROR}: '{self.container}' isn't a valid container name. Enter only the "
- "container name, without a URL or a folder."
+ f"{INVALID_CONFIG_ERROR}: '{self.container}' in the URL isn't a valid container name."
)
if not _AZURE_ACCOUNT_KEY.fullmatch(self.account_key):
raise ObjectStorageError(
f"{INVALID_CONFIG_ERROR}: the account key isn't valid. Copy one of the storage account's access keys."
)
- _validate_prefix(self.prefix)
+ _validate_keys(self.prefix, self.listing_prefix)
@property
def location_url(self) -> str:
@@ -162,7 +201,7 @@ class AzureBlobConfig:
@property
def display_location(self) -> str:
- return f"{self.container}/{self.prefix}" if self.prefix else self.container
+ return _display_location(self.container, self.prefix, self.listing_prefix)
@property
def connection_string(self) -> str:
@@ -188,7 +227,6 @@ class DiscoveryOptions:
"""The settings that decide which files exist as tables, and how each one is read."""
file_format: FileFormatChoice = "infer"
- file_pattern: str | None = None
allow_double_quotes: bool = True
combined_table_name: str | None = None
@@ -196,8 +234,3 @@ class DiscoveryOptions:
# The API checks the shape of a payload, not that a `Literal` field holds one of its values.
if self.file_format not in get_args(FileFormatChoice):
raise ObjectStorageError(f"{INVALID_CONFIG_ERROR}: '{self.file_format}' isn't a supported file format.")
-
-
-def normalize_prefix(path: str | None) -> str:
- """Drop empty segments only. A '.' or '..' segment stays, so `ObjectStoreConfig` rejects it."""
- return "/".join(segment for segment in (path or "").strip().split("/") if segment)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
index da97a7ef87e6..5c0c2e103659 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/discovery.py
@@ -1,5 +1,4 @@
import re
-import fnmatch
import posixpath
import dataclasses
from collections.abc import Iterable, Mapping
@@ -24,6 +23,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
ObjectStore,
StoredObject,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (
+ ClickHouseGlob,
+ compile_clickhouse_glob,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.settings import (
MAX_TABLE_OBJECTS,
MAX_TABLES,
@@ -55,13 +58,15 @@ _HIVE_PARTITION_FOLDER = re.compile(r"[^.=][^=]*=.*")
class TableDefinition:
name: str
kind: TableKind
- # Relative to the store root. For a `file` scope it is the file itself.
+ # Relative to the store root. For a `file` scope it is the file itself. For an `all` scope it is the
+ # folder or file the source lists, or "" for the whole store root.
path: str
scope: FileScope | None
# "delta" for a Delta table. For files: the format the files are read with, or "infer" when one
# combined table mixes formats and each file is read by its extension.
file_format: FileFormatChoice
csv: CsvOptions
+ # A ClickHouse glob over paths relative to the store root, from the source's files URL.
pattern: str | None
# The source's configured format at discovery, replayed at sync so the same files group together.
source_format: FileFormatChoice
@@ -190,6 +195,26 @@ def _is_under(path: str, roots: frozenset[str]) -> bool:
return any("/".join(segments[:depth]) in roots for depth in range(1, len(segments) + 1))
+def _roots_with_matching_file(objects: Iterable[StoredObject], roots: frozenset[str], glob: ClickHouseGlob) -> set[str]:
+ """The roots that hold at least one file the glob matches, found in one pass over the listing."""
+ matched: set[str] = set()
+ if not roots:
+ return matched
+ for stored in objects:
+ if not glob.matches(stored.path):
+ continue
+ if "" in roots:
+ matched.add("")
+ segments = stored.path.split("/")[:-1]
+ for depth in range(1, len(segments) + 1):
+ folder = "/".join(segments[:depth])
+ if folder in roots:
+ matched.add(folder)
+ if matched == roots:
+ break
+ return matched
+
+
def _table_folder(path: str) -> str | None:
"""The folder a file's table is named after, or None for a file directly under the root.
@@ -215,6 +240,8 @@ def group_tables(
objects: Iterable[StoredObject],
options: DiscoveryOptions,
root_name: str,
+ pattern: str | None = None,
+ listing_prefix: str = "",
logger: FilteringBoundLogger | None = None,
max_tables: int | None = MAX_TABLES,
) -> list[TableDefinition]:
@@ -224,6 +251,7 @@ def group_tables(
- Other files group by their folder, with hive partition folders folded into the table above.
- A file directly under the root is a table of its own.
- `combined_table_name` puts every file into one table.
+ - `pattern` keeps only the data files it matches, and only the Delta tables it matches a file of.
The same listing and options always give the same tables, which is what lets a sync find its
table again from `schema_metadata`.
@@ -231,7 +259,7 @@ def group_tables(
ordered = sorted(objects, key=lambda stored: stored.path)
delta_roots = frozenset(_delta_roots(ordered))
choice = options.file_format
- pattern = options.file_pattern or None
+ glob = compile_clickhouse_glob(pattern) if pattern else None
combined_name = (options.combined_table_name or "").strip() or None
def definition(
@@ -256,7 +284,8 @@ def group_tables(
candidates: list[TableDefinition] = []
if choice in ("infer", "delta"):
- for root in sorted(delta_roots):
+ offered_roots = sorted(delta_roots if glob is None else _roots_with_matching_file(ordered, delta_roots, glob))
+ for root in offered_roots:
candidates.append(definition(root.replace("/", "_") or root_name, "delta", root, None, "delta", []))
if choice != "delta":
@@ -265,7 +294,7 @@ def group_tables(
segments = stored.path.split("/")
if not segments[-1] or _is_hidden(stored.path) or _is_under(stored.path, delta_roots):
continue
- if pattern is not None and not fnmatch.fnmatchcase(stored.path, pattern):
+ if glob is not None and not glob.matches(stored.path):
continue
file_format = _table_format(stored.path, choice)
if file_format is not None:
@@ -276,7 +305,9 @@ def group_tables(
formats = {file_format for _, file_format in data_files}
combined_format = formats.pop() if len(formats) == 1 else "infer"
candidates.append(
- definition(combined_name, "files", "", "all", combined_format, [f for f, _ in data_files])
+ definition(
+ combined_name, "files", listing_prefix, "all", combined_format, [f for f, _ in data_files]
+ )
)
else:
folders: dict[tuple[str, FileFormatChoice], list[StoredObject]] = {}
@@ -329,20 +360,26 @@ def resolve_table_files(
"""
if table.kind != "files":
return table
- listing_prefix = "" if table.scope == "all" else table.path
- objects = store.list_objects(listing_prefix, max_objects + 1)
+ objects = store.list_objects(table.path, max_objects + 1)
if len(objects) > max_objects:
raise ObjectStorageError(
- f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{listing_prefix or '/'}'. Point the source at a "
- "narrower folder, or set a file pattern that matches fewer files."
+ f"{TOO_MANY_FILES_ERROR} ({max_objects:,} files) in '{table.path or '/'}'. Point the URL at a "
+ "narrower folder, or use a pattern that matches fewer files."
)
options = DiscoveryOptions(
file_format=table.source_format,
- file_pattern=table.pattern,
allow_double_quotes=table.csv.allow_double_quotes,
combined_table_name=table.name if table.scope == "all" else None,
)
- for candidate in group_tables(objects, options, root_name=table.name, logger=logger, max_tables=None):
+ for candidate in group_tables(
+ objects,
+ options,
+ root_name=table.name,
+ pattern=table.pattern,
+ listing_prefix=table.path if table.scope == "all" else "",
+ logger=logger,
+ max_tables=None,
+ ):
# A combined table keeps its identity when a new file changes which formats it mixes.
if candidate.same_location(table) or (table.scope == "all" and candidate.scope == "all"):
return dataclasses.replace(candidate, name=table.name)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
index 78d2473336a9..3a5c4a265148 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/fields.py
@@ -35,8 +35,6 @@ class _FileFormatConfig(Protocol):
class FileOptionsConfig(Protocol):
"""The settings every object storage source's generated config carries."""
- @property
- def file_pattern(self) -> str | None: ...
@property
def file_format(self) -> _FileFormatConfig: ...
@property
@@ -47,7 +45,6 @@ def discovery_options_from(config: FileOptionsConfig) -> DiscoveryOptions:
combine = config.combine_files
return DiscoveryOptions(
file_format=config.file_format.selection,
- file_pattern=(config.file_pattern or "").strip() or None,
allow_double_quotes=config.file_format.csv_allow_double_quotes,
combined_table_name=combine.table_name if combine is not None and combine.enabled else None,
)
@@ -56,39 +53,24 @@ def discovery_options_from(config: FileOptionsConfig) -> DiscoveryOptions:
def source_caption(store: str) -> str:
return (
f"Import CSV, JSON Lines and Parquet files, and Delta Lake tables, from {store}. PostHog creates one "
- "table per folder, one per file at the top of the folder you choose, and one per Delta table. A full "
+ "table per folder, one per file directly in the folder the URL points to, and one per Delta table. A full "
"refresh reads every file again, so the table matches the files. An incremental or append-only sync "
"reads only new and changed files, and keeps the rows of files you delete."
)
-def folder_fields(container: str) -> list[FieldType]:
- """The folder and file pattern fields. `container` is the provider's word for a bucket."""
- return cast(
- list[FieldType],
- [
- SourceFieldInputConfig(
- name="path",
- label="Folder (optional)",
- type=SourceFieldInputConfigType.TEXT,
- required=False,
- placeholder="exports/",
- caption=f"Leave it empty to import from the whole {container}.",
- secret=False,
- ),
- SourceFieldInputConfig(
- name="file_pattern",
- label="File pattern (optional)",
- type=SourceFieldInputConfigType.TEXT,
- required=False,
- placeholder="*.parquet",
- caption=(
- "A glob matched against each file's path inside the folder, where * also matches "
- "across folders. Leave it empty to import every file."
- ),
- secret=False,
- ),
- ],
+def url_field(placeholder: str) -> FieldType:
+ return SourceFieldInputConfig(
+ name="url_pattern",
+ label="Files URL",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder=placeholder,
+ caption=(
+ "Use * to match any part of a file or folder name, and ** to match across folders. A URL "
+ "without a wildcard imports the folder or file it points to."
+ ),
+ secret=False,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/globs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/globs.py
new file mode 100644
index 000000000000..5e6e587bd74e
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/globs.py
@@ -0,0 +1,233 @@
+import re
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ INVALID_CONFIG_ERROR,
+ ObjectStorageError,
+)
+
+GLOB_CHARACTERS = frozenset("*?{")
+# Characters ClickHouse does not document as glob syntax, but that other glob dialects use. A path
+# that holds them could mean either thing, so it is refused instead of read one way.
+_AMBIGUOUS_CHARACTERS = frozenset("[]}")
+_NUMERIC_RANGE = re.compile(r"(\d+)\.\.(\d+)")
+# The matcher tracks one state per open wildcard, so its work grows with how many a pattern holds.
+# Object keys are at most 1,024 bytes, and real patterns need a handful of wildcards.
+MAX_PATTERN_LENGTH = 1024
+MAX_WILDCARDS = 64
+
+
+@frozen
+class _Literal:
+ character: str
+
+
+@frozen
+class _Question:
+ pass
+
+
+@frozen
+class _Star:
+ cross_folders: bool
+
+
+@frozen
+class _Alternatives:
+ options: tuple[str, ...]
+
+
+@frozen
+class _NumericRange:
+ low: int
+ high: int
+ width: int | None
+ max_width: int
+
+
+_Token = _Literal | _Question | _Star | _Alternatives | _NumericRange
+
+
+@frozen
+class _AlternativeState:
+ token: int
+ alternative: int
+ offset: int
+
+
+@frozen
+class _NumericState:
+ token: int
+ digits: str
+
+
+_State = int | _AlternativeState | _NumericState
+
+
+@frozen
+class ClickHouseGlob:
+ """A path glob with the ClickHouse `s3()` semantics, matched against a whole path.
+
+ `*` matches any characters except `/`, `**` matches any characters including `/`, `?` matches
+ one character except `/`, `{a,b}` matches one of the literal alternatives, and `{N..M}` matches
+ a whole number from N to M.
+ """
+
+ pattern: str
+ tokens: tuple[_Token, ...]
+
+ def _epsilon_closure(self, states: set[_State]) -> set[_State]:
+ pending = [state for state in states if isinstance(state, int)]
+ # Empty alternatives can reach the same token through several paths.
+ visited = set(pending)
+ while pending:
+ index = pending.pop()
+ if index == len(self.tokens):
+ continue
+ token = self.tokens[index]
+ can_skip = False
+ if isinstance(token, _Star):
+ can_skip = True
+ elif isinstance(token, _Alternatives):
+ states.remove(index)
+ for alternative, option in enumerate(token.options):
+ if option:
+ states.add(_AlternativeState(token=index, alternative=alternative, offset=0))
+ else:
+ can_skip = True
+ elif isinstance(token, _NumericRange):
+ states.remove(index)
+ states.add(_NumericState(token=index, digits=""))
+ if can_skip and index + 1 not in visited:
+ visited.add(index + 1)
+ states.add(index + 1)
+ pending.append(index + 1)
+ return states
+
+ def matches(self, path: str) -> bool:
+ states = self._epsilon_closure({0})
+ for character in path:
+ following: set[_State] = set()
+ for state in states:
+ if isinstance(state, int):
+ if state == len(self.tokens):
+ continue
+ token = self.tokens[state]
+ if isinstance(token, _Literal):
+ if character == token.character:
+ following.add(state + 1)
+ elif isinstance(token, _Question):
+ if character != "/":
+ following.add(state + 1)
+ elif isinstance(token, _Star):
+ if token.cross_folders or character != "/":
+ following.add(state)
+ elif isinstance(state, _AlternativeState):
+ token = self.tokens[state.token]
+ assert isinstance(token, _Alternatives)
+ option = token.options[state.alternative]
+ if character == option[state.offset]:
+ if state.offset + 1 == len(option):
+ following.add(state.token + 1)
+ else:
+ following.add(
+ _AlternativeState(
+ token=state.token, alternative=state.alternative, offset=state.offset + 1
+ )
+ )
+ else:
+ token = self.tokens[state.token]
+ assert isinstance(token, _NumericRange)
+ if not character.isdecimal():
+ continue
+ digits = state.digits + character
+ value = int(digits)
+ if token.low <= value <= token.high and (token.width is None or len(digits) == token.width):
+ following.add(state.token + 1)
+ if len(digits) < token.max_width and (token.width is not None or value != 0):
+ following.add(_NumericState(token=state.token, digits=digits))
+ if not following:
+ return False
+ states = self._epsilon_closure(following)
+ return len(self.tokens) in states
+
+
+def has_glob(path: str) -> bool:
+ return any(character in GLOB_CHARACTERS for character in path)
+
+
+def glob_key_prefix(pattern: str) -> str:
+ """Return the literal key prefix before the first glob character."""
+ for index, character in enumerate(pattern):
+ if character in GLOB_CHARACTERS:
+ return pattern[:index]
+ return pattern
+
+
+def glob_listing_prefix(pattern: str) -> str:
+ """The folder that holds every path the glob can match: the segments before the first glob."""
+ literal: list[str] = []
+ segments = pattern.split("/")
+ for index, segment in enumerate(segments):
+ if has_glob(segment):
+ return "/".join(literal)
+ if index < len(segments) - 1:
+ literal.append(segment)
+ # No glob at all: the pattern names one file, and a listing of that path finds it.
+ return pattern
+
+
+def _invalid(pattern: str, detail: str) -> ObjectStorageError:
+ return ObjectStorageError(f"{INVALID_CONFIG_ERROR}: the file pattern '{pattern}' {detail}.")
+
+
+def compile_clickhouse_glob(pattern: str) -> ClickHouseGlob:
+ if len(pattern) > MAX_PATTERN_LENGTH:
+ raise _invalid(pattern[:64] + "...", f"is longer than {MAX_PATTERN_LENGTH:,} characters")
+ tokens: list[_Token] = []
+ wildcards = 0
+ index = 0
+ while index < len(pattern):
+ character = pattern[index]
+ if character == "*":
+ # A run of stars matches what its widest star matches, so it is one token.
+ end = index
+ while end < len(pattern) and pattern[end] == "*":
+ end += 1
+ tokens.append(_Star(cross_folders=end - index > 1))
+ wildcards += 1
+ index = end
+ continue
+ elif character == "?":
+ tokens.append(_Question())
+ wildcards += 1
+ elif character == "{":
+ end = pattern.find("}", index)
+ if end == -1:
+ raise _invalid(pattern, "has a '{' with no closing '}'")
+ body = pattern[index + 1 : end]
+ if not body or any(nested in GLOB_CHARACTERS or nested in _AMBIGUOUS_CHARACTERS for nested in body):
+ raise _invalid(pattern, "has a '{...}' group PostHog can't read")
+ numeric = _NUMERIC_RANGE.fullmatch(body)
+ if numeric is not None:
+ start, stop = numeric.group(1), numeric.group(2)
+ low, high = sorted((int(start), int(stop)))
+ padded = (len(start) > 1 and start.startswith("0")) or (len(stop) > 1 and stop.startswith("0"))
+ width = max(len(start), len(stop)) if padded else None
+ tokens.append(_NumericRange(low=low, high=high, width=width, max_width=width or len(str(high))))
+ wildcards += 1
+ else:
+ options = tuple(body.split(","))
+ tokens.append(_Alternatives(options=options))
+ wildcards += len(options)
+ index = end + 1
+ continue
+ elif character in _AMBIGUOUS_CHARACTERS:
+ raise _invalid(pattern, f"has a '{character}' that PostHog can't read as a file pattern")
+ else:
+ tokens.append(_Literal(character=character))
+ index += 1
+ if wildcards > MAX_WILDCARDS:
+ raise _invalid(pattern, f"has more than {MAX_WILDCARDS} wildcards and alternatives")
+ return ClickHouseGlob(pattern=pattern, tokens=tuple(tokens))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py
new file mode 100644
index 000000000000..f065a066e5cf
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/locations.py
@@ -0,0 +1,169 @@
+"""Parse the files URL of an object storage source into a bucket and the key glob after it.
+
+Each provider accepts only its own hosts. The endpoint the stored credentials go to is rebuilt from
+the parsed bucket, region and account, never taken from the URL's host as written.
+"""
+
+import re
+from urllib.parse import unquote
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ INVALID_CONFIG_ERROR,
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (
+ compile_clickhouse_glob,
+ glob_listing_prefix,
+ has_glob,
+)
+
+S3_EXAMPLE_URL = "https://my-company-exports.s3.us-east-1.amazonaws.com/orders/**/*.parquet"
+GCS_EXAMPLE_URL = "https://storage.googleapis.com/my-company-exports/orders/**/*.parquet"
+R2_EXAMPLE_URL = "https://<account-id>.r2.cloudflarestorage.com/my-company-exports/orders/**/*.parquet"
+AZURE_EXAMPLE_URL = "https://mycompanystorage.blob.core.windows.net/exports/orders/**/*.parquet"
+
+_URL = re.compile(r"(?P<scheme>[A-Za-z][A-Za-z0-9+.-]*)://(?P<host>[^/]*)(?P<path>.*)", re.DOTALL)
+# No user name, port, or other character that could make a client read the authority differently.
+_HOST = re.compile(r"[a-z0-9._-]+")
+_REGION = r"[a-z]{2}(?:-[a-z0-9]+)+-\d+"
+_S3_VIRTUAL_HOSTED = re.compile(rf"(?P<bucket>[a-z0-9.-]+)\.s3\.(?P<region>{_REGION})\.amazonaws\.com")
+_S3_PATH_STYLE = re.compile(rf"s3\.(?P<region>{_REGION})\.amazonaws\.com")
+_S3_WITHOUT_REGION = re.compile(r"(?:[a-z0-9.-]+\.)?s3\.amazonaws\.com")
+_GCS_HOST = "storage.googleapis.com"
+_R2_HOST = re.compile(r"(?P<account>[0-9a-f]{32})\.r2\.cloudflarestorage\.com")
+_AZURE_HOST = re.compile(r"(?P<account>[a-z0-9]{3,24})\.blob\.core\.windows\.net")
+_CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f]")
+
+
+@frozen
+class StorageUrl:
+ # The bucket, or the container for Azure.
+ bucket: str
+ # The decoded object key after the bucket. It can hold ClickHouse glob syntax.
+ key: str
+ region: str | None = None
+ # The Cloudflare account ID for R2, or the storage account name for Azure.
+ account: str | None = None
+
+
+@frozen
+class KeySelection:
+ """The keys under a bucket that a files URL selects."""
+
+ # The folder the store is rooted at. Table names and table paths are relative to it.
+ prefix: str
+ # The one folder or file under `prefix` that discovery lists, or "" to list all of `prefix`.
+ listing_prefix: str
+ # A ClickHouse glob over paths relative to `prefix`, or None to keep every listed file.
+ pattern: str | None
+
+
+@frozen
+class _UrlParts:
+ scheme: str
+ host: str
+ # The raw path after the host, still percent-encoded.
+ path: str
+
+
+def _invalid(detail: str) -> ObjectStorageError:
+ return ObjectStorageError(f"{INVALID_CONFIG_ERROR}: {detail}")
+
+
+def _split(url: str, schemes: tuple[str, ...], example: str) -> _UrlParts:
+ """The scheme, host and raw path of a URL, after refusing what could change where it points."""
+ match = _URL.fullmatch(url.strip())
+ if match is None:
+ raise _invalid(f"enter the full URL of the files, such as {example}")
+ scheme = match.group("scheme").lower()
+ if scheme not in schemes:
+ raise _invalid(f"the URL must start with {' or '.join(f'{allowed}://' for allowed in schemes)}.")
+ host = match.group("host").lower()
+ if not _HOST.fullmatch(host):
+ raise _invalid("the URL host can't contain a user name, a port, or special characters.")
+ path = match.group("path")
+ if "#" in path or "\\" in path:
+ raise _invalid("the URL can't contain '#' or '\\'.")
+ return _UrlParts(scheme=scheme, host=host, path=path)
+
+
+def _key(raw_key: str) -> str:
+ key = unquote(raw_key)
+ if _CONTROL_CHARACTERS.search(key):
+ raise _invalid("the URL path can't contain control characters.")
+ segments = key.split("/")
+ if any(segment in (".", "..") for segment in (*segments, *raw_key.split("/"))):
+ raise _invalid("the URL path can't contain '.' or '..' folders.")
+ if "" in segments[:-1]:
+ raise _invalid("the URL path can't contain an empty folder ('//').")
+ return key
+
+
+def _path_style(path: str, example: str, region: str | None = None, account: str | None = None) -> StorageUrl:
+ """Split `/<bucket>/<key>` into the bucket and the decoded key."""
+ bucket, _, raw_key = path.removeprefix("/").partition("/")
+ if not bucket:
+ raise _invalid(f"the URL doesn't name a bucket. Use a URL such as {example}")
+ if "%" in bucket or has_glob(bucket):
+ raise _invalid("the bucket name in the URL must be exact. Put wildcards in the path after the bucket.")
+ return StorageUrl(bucket=bucket, key=_key(raw_key), region=region, account=account)
+
+
+def parse_s3_url(url: str) -> StorageUrl:
+ parts = _split(url, ("https",), S3_EXAMPLE_URL)
+ host = parts.host
+ if (virtual_hosted := _S3_VIRTUAL_HOSTED.fullmatch(host)) is not None:
+ return StorageUrl(
+ bucket=virtual_hosted.group("bucket"),
+ key=_key(parts.path.removeprefix("/")),
+ region=virtual_hosted.group("region"),
+ )
+ if (path_style := _S3_PATH_STYLE.fullmatch(host)) is not None:
+ return _path_style(parts.path, S3_EXAMPLE_URL, region=path_style.group("region"))
+ if _S3_WITHOUT_REGION.fullmatch(host):
+ raise _invalid(f"the URL doesn't name the bucket's region. Use the regional URL, such as {S3_EXAMPLE_URL}")
+ raise _invalid(f"'{host}' isn't an Amazon S3 URL. Use a URL such as {S3_EXAMPLE_URL}")
+
+
+def parse_gcs_url(url: str) -> StorageUrl:
+ parts = _split(url, ("https", "gs"), GCS_EXAMPLE_URL)
+ if parts.scheme == "gs":
+ return _path_style(f"/{parts.host}{parts.path}", GCS_EXAMPLE_URL)
+ if parts.host != _GCS_HOST:
+ raise _invalid(f"'{parts.host}' isn't a Google Cloud Storage URL. Use a URL such as {GCS_EXAMPLE_URL}")
+ return _path_style(parts.path, GCS_EXAMPLE_URL)
+
+
+def parse_r2_url(url: str) -> StorageUrl:
+ parts = _split(url, ("https",), R2_EXAMPLE_URL)
+ if (r2 := _R2_HOST.fullmatch(parts.host)) is None:
+ raise _invalid(f"'{parts.host}' isn't a Cloudflare R2 URL. Use a URL such as {R2_EXAMPLE_URL}")
+ return _path_style(parts.path, R2_EXAMPLE_URL, account=r2.group("account"))
+
+
+def parse_azure_url(url: str) -> StorageUrl:
+ parts = _split(url, ("https",), AZURE_EXAMPLE_URL)
+ if (azure := _AZURE_HOST.fullmatch(parts.host)) is None:
+ raise _invalid(f"'{parts.host}' isn't an Azure Blob Storage URL. Use a URL such as {AZURE_EXAMPLE_URL}")
+ return _path_style(parts.path, AZURE_EXAMPLE_URL, account=azure.group("account"))
+
+
+def select_keys(key: str) -> KeySelection:
+ """Root the store at the literal folder before the first glob, and keep the glob for the rest.
+
+ A key with no glob selects the folder it names, or the one file it names.
+ """
+ if has_glob(key):
+ folder = glob_listing_prefix(key)
+ pattern = key[len(folder) + 1 :] if folder else key
+ if pattern.endswith("/"):
+ # A glob that ends at a folder reads every file under the folders it matches.
+ pattern = f"{pattern}**"
+ compile_clickhouse_glob(pattern)
+ return KeySelection(prefix=folder, listing_prefix="", pattern=pattern)
+ if not key or key.endswith("/"):
+ return KeySelection(prefix=key.rstrip("/"), listing_prefix="", pattern=None)
+ folder, _, name = key.rpartition("/")
+ return KeySelection(prefix=folder, listing_prefix=name, pattern=None)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index d89d330eb23a..684ac2d114d8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -94,7 +94,7 @@ def validate_store_location(config: StoreConfig) -> None:
class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDeltaCursor], Generic[ConfigType]):
- """Imports the files in a bucket folder: CSV, JSON Lines and Parquet files, and Delta tables.
+ """Imports the files a bucket URL selects: CSV, JSON Lines and Parquet files, and Delta tables.
A full refresh reads each table in full, so the table matches the bucket at the time of the sync.
The incremental and append sync types read only the files modified since the last sync. They
@@ -124,8 +124,10 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
"The storage provider rejected these credentials. Check the access key and secret, and that "
"the key can list and read the bucket, then reconnect the source."
),
- NO_SUCH_BUCKET_ERROR: "The bucket doesn't exist. Check the bucket name, then reconnect the source.",
- WRONG_REGION_ERROR: "The bucket is in a different region. Fix the region, then reconnect the source.",
+ NO_SUCH_BUCKET_ERROR: "The bucket doesn't exist. Check the bucket in the URL, then reconnect the source.",
+ WRONG_REGION_ERROR: (
+ "The bucket is in a different region. Fix the region in the URL, then reconnect the source."
+ ),
NO_TABLE_FILES_ERROR: (
"The files behind this table are no longer in the bucket. Refresh the source's tables, or "
"restore the files."
@@ -140,13 +142,20 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
) -> list[TableDefinition]:
store_config = self.object_store_config(config)
try:
- objects = store.list_objects("", MAX_DISCOVERY_OBJECTS)
+ objects = store.list_objects(store_config.listing_prefix, MAX_DISCOVERY_OBJECTS)
except Exception as e:
_raise_translated(e, store_config.display_location)
if len(objects) >= MAX_DISCOVERY_OBJECTS and logger is not None:
logger.warning("Reached the object storage listing limit; later objects are ignored")
root_name = store_config.prefix.rsplit("/", 1)[-1] if store_config.prefix else store_config.root_name
- return group_tables(objects, self.discovery_options(config), root_name=root_name, logger=logger)
+ return group_tables(
+ objects,
+ self.discovery_options(config),
+ root_name=root_name,
+ pattern=store_config.pattern,
+ listing_prefix=store_config.listing_prefix,
+ logger=logger,
+ )
def validate_credentials(
self,
@@ -172,7 +181,7 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
if not tables:
return False, (
f"{NO_IMPORTABLE_FILES_ERROR} in '{store_config.display_location}'. PostHog can read the bucket but "
- "found nothing to import there. Check the folder, the file pattern, and the file format."
+ "found nothing to import there. Check the URL and the file format."
)
return True, None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_globs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_globs.py
new file mode 100644
index 000000000000..df1443bfa7c3
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_globs.py
@@ -0,0 +1,128 @@
+import time
+
+import pytest
+
+from parameterized import parameterized
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (
+ compile_clickhouse_glob,
+ glob_key_prefix,
+ glob_listing_prefix,
+)
+
+
+class TestClickHouseGlob:
+ @parameterized.expand(
+ [
+ ("star_stays_in_its_folder", "data/*.csv", "data/a.csv", True),
+ ("star_matches_empty", "data/*.csv", "data/.csv", True),
+ ("star_does_not_cross_folders", "data/*.csv", "data/2024/a.csv", False),
+ ("double_star_crosses_folders", "data/**.csv", "data/2024/01/a.csv", True),
+ ("double_star_folder", "data/**", "data/x/y", True),
+ ("question_mark_is_one_character", "part-?.csv", "part-1.csv", True),
+ ("question_mark_is_not_two", "part-?.csv", "part-10.csv", False),
+ ("question_mark_does_not_match_slash", "a?b", "a/b", False),
+ ("alternatives", "{orders,refunds}.csv", "refunds.csv", True),
+ ("alternatives_are_literal", "{a.b,c}.csv", "aXb.csv", False),
+ ("range", "part-{1..12}.csv", "part-12.csv", True),
+ ("range_upper_bound", "part-{1..12}.csv", "part-13.csv", False),
+ ("range_reversed_bounds", "part-{12..1}.csv", "part-5.csv", True),
+ ("range_without_padding_refuses_leading_zero", "part-{1..12}.csv", "part-05.csv", False),
+ ("padded_range", "part-{01..12}.csv", "part-05.csv", True),
+ ("padded_range_refuses_unpadded", "part-{01..12}.csv", "part-5.csv", False),
+ ("literal_regex_characters", "a+b(1).csv", "a+b(1).csv", True),
+ ("literal_dot_is_not_any_character", "a.csv", "aXcsv", False),
+ ("literal_path_is_exact", "data/a.csv", "data/a.csv.bak", False),
+ ("hidden_files_match_like_clickhouse", "data/*", "data/_SUCCESS", True),
+ ("range_leaves_digits_for_star", "part-{1..12}*.csv", "part-123.csv", True),
+ ("range_does_not_take_extra_digits", "part-{1..12}.csv", "part-123.csv", False),
+ ("empty_alternative", "{a,}x.csv", "x.csv", True),
+ ("padded_range_in_middle", "d={01..31}/h-*.csv", "d=07/h-1.csv", True),
+ ("padded_range_in_middle_refuses_unpadded", "d={01..31}/h-*.csv", "d=7/h-1.csv", False),
+ ("zero_range", "{0..0}", "0", True),
+ ("zero_range_refuses_extra_zero", "{0..0}", "00", False),
+ ("double_star_after_range", "{1..12}**.csv", "123/nested/a.csv", True),
+ ("alternatives_before_star", "{a,ab}*.csv", "abc.csv", True),
+ ("double_star_matches_empty", "**", "", True),
+ ("star_does_not_match_slash", "*", "/", False),
+ ("question_mark_does_not_match_only_slash", "?", "/", False),
+ ("triple_star_crosses_folders", "***", "a/b", True),
+ ("four_stars_match_empty", "****", "", True),
+ ("adjacent_empty_alternatives", "*{a,}{b,}x", "x", True),
+ ("range_leaves_digits_for_literal", "{1..12}3", "123", True),
+ ("padded_reversed_range", "{12..01}", "07", True),
+ ("padded_range_uses_widest_bound", "{0..0012}", "0007", True),
+ ]
+ )
+ def test_matches(self, _name: str, pattern: str, path: str, expected: bool) -> None:
+ assert compile_clickhouse_glob(pattern).matches(path) is expected
+
+ @parameterized.expand(
+ [
+ ("unclosed_brace", "a{b,c.csv"),
+ ("empty_braces", "a{}.csv"),
+ ("nested_glob_in_braces", "{a*,b}.csv"),
+ ("square_brackets", "part-[0-9].csv"),
+ ("stray_closing_brace", "a}.csv"),
+ ("nested_question_mark", "{a?,b}.csv"),
+ ("nested_brace", "{{a,b},c}.csv"),
+ ("bracket_in_braces", "{a[,b}.csv"),
+ ("closing_bracket_in_braces", "{a],b}.csv"),
+ ("stray_closing_bracket", "a].csv"),
+ ("too_long", "a" * 1025),
+ ("too_many_wildcards", "*a" * 65),
+ ("too_many_alternatives", "{" + ",".join(str(n) for n in range(65)) + "}"),
+ ]
+ )
+ def test_refuses_patterns_it_cant_read(self, _name: str, pattern: str) -> None:
+ with pytest.raises(ObjectStorageError):
+ compile_clickhouse_glob(pattern)
+
+ @parameterized.expand(
+ [
+ ("glob_in_file_name", "events/2024/*.csv", "events/2024"),
+ ("glob_in_folder", "events/*/a.csv", "events"),
+ ("glob_at_root", "*.csv", ""),
+ ("range_folder", "y={2023..2024}/a.parquet", ""),
+ ("literal_file", "events/a.csv", "events/a.csv"),
+ ]
+ )
+ def test_listing_prefix(self, _name: str, pattern: str, expected: str) -> None:
+ assert glob_listing_prefix(pattern) == expected
+
+ @parameterized.expand(
+ [
+ ("glob_in_file_name", "exports/orders-*.csv", "exports/orders-"),
+ ("glob_at_root", "*.csv", ""),
+ ("range_folder", "y={2023..2024}/a.parquet", "y="),
+ ("literal_file", "events/a.csv", "events/a.csv"),
+ ("question_mark", "a?b", "a"),
+ ]
+ )
+ def test_key_prefix(self, _name: str, pattern: str, expected: str) -> None:
+ assert glob_key_prefix(pattern) == expected
+
+ @parameterized.expand(
+ [
+ ("twenty_stars", "*a" * 20 + "*b.csv"),
+ ("eight_stars", "*a*a*a*a*a*a*a*a*b.csv"),
+ ]
+ )
+ def test_adversarial_match_time(self, _name: str, pattern: str) -> None:
+ glob = compile_clickhouse_glob(pattern)
+ path = "a" * 200 + ".csv"
+
+ started = time.perf_counter()
+ for _ in range(1000):
+ assert glob.matches(path) is False
+ assert time.perf_counter() - started < 2
+ assert glob.matches("a" * 200 + "b.csv") is True
+
+ def test_a_run_of_stars_is_one_wildcard(self) -> None:
+ glob = compile_clickhouse_glob("*" * 1000 + ".csv")
+
+ assert len(glob.tokens) == 5
+ assert glob.matches("a/b/c.csv") is True
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py
new file mode 100644
index 000000000000..66e5baf0a3e4
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_locations.py
@@ -0,0 +1,237 @@
+from collections.abc import Callable
+
+import pytest
+
+from django.test import override_settings
+
+from parameterized import parameterized
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
+ AzureBlobConfig,
+ ObjectStoreConfig,
+ ObjectStoreProvider,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.errors import (
+ BLOCKED_LOCATION_ERROR,
+ INVALID_CONFIG_ERROR,
+ ObjectStorageError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
+ KeySelection,
+ StorageUrl,
+ parse_azure_url,
+ parse_gcs_url,
+ parse_r2_url,
+ parse_s3_url,
+ select_keys,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
+ validate_store_location,
+)
+
+R2_ACCOUNT = "0123456789abcdef0123456789abcdef"
+Parser = Callable[[str], StorageUrl]
+
+
+class TestParseUrl:
+ @parameterized.expand(
+ [
+ (
+ "s3_virtual_hosted",
+ parse_s3_url,
+ "https://acme-exports.s3.eu-central-1.amazonaws.com/raw/team_7/orders/**/*.parquet",
+ StorageUrl(bucket="acme-exports", key="raw/team_7/orders/**/*.parquet", region="eu-central-1"),
+ ),
+ (
+ "s3_virtual_hosted_dotted_bucket",
+ parse_s3_url,
+ "https://acme.exports.s3.us-west-2.amazonaws.com/a.csv",
+ StorageUrl(bucket="acme.exports", key="a.csv", region="us-west-2"),
+ ),
+ (
+ "s3_path_style",
+ parse_s3_url,
+ "https://s3.ap-southeast-2.amazonaws.com/acme-exports/daily/*.csv",
+ StorageUrl(bucket="acme-exports", key="daily/*.csv", region="ap-southeast-2"),
+ ),
+ (
+ "s3_bucket_root",
+ parse_s3_url,
+ "https://acme-exports.s3.us-east-1.amazonaws.com",
+ StorageUrl(bucket="acme-exports", key="", region="us-east-1"),
+ ),
+ (
+ "s3_question_mark_is_a_glob",
+ parse_s3_url,
+ "https://acme-exports.s3.us-east-1.amazonaws.com/part-?.csv",
+ StorageUrl(bucket="acme-exports", key="part-?.csv", region="us-east-1"),
+ ),
+ (
+ "s3_percent_encoded_key",
+ parse_s3_url,
+ "HTTPS://Acme-Exports.S3.us-east-1.amazonaws.com/my%20files/a.csv",
+ StorageUrl(bucket="acme-exports", key="my files/a.csv", region="us-east-1"),
+ ),
+ (
+ "gcs_https",
+ parse_gcs_url,
+ "https://storage.googleapis.com/acme_exports/orders/{2024..2026}/*.jsonl",
+ StorageUrl(bucket="acme_exports", key="orders/{2024..2026}/*.jsonl"),
+ ),
+ (
+ "gcs_gs_scheme",
+ parse_gcs_url,
+ "gs://acme_exports/orders/",
+ StorageUrl(bucket="acme_exports", key="orders/"),
+ ),
+ (
+ "r2",
+ parse_r2_url,
+ f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/acme-exports/orders/*.csv",
+ StorageUrl(bucket="acme-exports", key="orders/*.csv", account=R2_ACCOUNT),
+ ),
+ (
+ "azure",
+ parse_azure_url,
+ "https://acmestore.blob.core.windows.net/exports/orders/**",
+ StorageUrl(bucket="exports", key="orders/**", account="acmestore"),
+ ),
+ ]
+ )
+ def test_accepts(self, _name: str, parse: Parser, url: str, expected: StorageUrl) -> None:
+ assert parse(url) == expected
+
+ @parameterized.expand(
+ [
+ ("s3_without_region_virtual_hosted", parse_s3_url, "https://acme.s3.amazonaws.com/a.csv", "region"),
+ ("s3_without_region_path_style", parse_s3_url, "https://s3.amazonaws.com/acme/a.csv", "region"),
+ ("s3_other_host", parse_s3_url, "https://acme.example.com/a.csv", "isn't an Amazon S3 URL"),
+ (
+ "s3_host_suffix",
+ parse_s3_url,
+ "https://acme.s3.us-east-1.amazonaws.com.example.com/a.csv",
+ "isn't an Amazon S3 URL",
+ ),
+ ("s3_dashed_region", parse_s3_url, "https://acme.s3-us-east-1.amazonaws.com/a.csv", "isn't an Amazon S3"),
+ ("s3_http", parse_s3_url, "http://acme.s3.us-east-1.amazonaws.com/a.csv", "https://"),
+ ("s3_user_info", parse_s3_url, "https://[email protected]/a.csv", "user name"),
+ ("s3_port", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com:444/a.csv", "port"),
+ ("s3_backslash", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com\\@example.com/a", "user name"),
+ ("s3_fragment", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a.csv#x", "'#'"),
+ ("s3_parent_folder", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a/../b.csv", "'..'"),
+ ("s3_encoded_parent_folder", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a/%2E%2E/b", "'..'"),
+ ("s3_current_folder", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/./b.csv", "'..'"),
+ ("s3_empty_folder", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a//b.csv", "'//'"),
+ ("s3_control_character", parse_s3_url, "https://acme.s3.us-east-1.amazonaws.com/a%0A.csv", "control"),
+ ("s3_glob_in_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/acme-*/a.csv", "exact"),
+ ("s3_encoded_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/acme%2Fx/a.csv", "exact"),
+ ("s3_no_bucket", parse_s3_url, "https://s3.us-east-1.amazonaws.com/", "doesn't name a bucket"),
+ ("s3_not_a_url", parse_s3_url, "acme-exports/a.csv", "full URL"),
+ ("gcs_other_host", parse_gcs_url, "https://storage.cloud.google.com/acme/a.csv", "isn't a Google Cloud"),
+ ("gcs_virtual_hosted", parse_gcs_url, "https://acme.storage.googleapis.com/a.csv", "isn't a Google Cloud"),
+ ("gcs_s3_scheme", parse_gcs_url, "s3://acme/a.csv", "gs://"),
+ ("r2_jurisdiction", parse_r2_url, f"https://{R2_ACCOUNT}.eu.r2.cloudflarestorage.com/a/b.csv", "R2 URL"),
+ ("r2_short_account", parse_r2_url, "https://abc.r2.cloudflarestorage.com/a/b.csv", "R2 URL"),
+ ("r2_other_host", parse_r2_url, f"https://{R2_ACCOUNT}.example.com/a/b.csv", "R2 URL"),
+ ("azure_other_cloud", parse_azure_url, "https://acme.blob.core.chinacloudapi.cn/x/a.csv", "Azure Blob"),
+ ("azure_dfs_host", parse_azure_url, "https://acme.dfs.core.windows.net/x/a.csv", "Azure Blob"),
+ ]
+ )
+ def test_refuses(self, _name: str, parse: Parser, url: str, detail: str) -> None:
+ with pytest.raises(ObjectStorageError) as error:
+ parse(url)
+
+ message = str(error.value)
+ assert message.startswith(INVALID_CONFIG_ERROR)
+ assert detail in message
+
+ def test_a_url_without_a_region_gives_an_example(self) -> None:
+ with pytest.raises(ObjectStorageError, match=r"such as https://[a-z-]+\.s3\.us-east-1\.amazonaws\.com/"):
+ parse_s3_url("https://acme-exports.s3.amazonaws.com/a.csv")
+
+
+class TestSelectKeys:
+ @parameterized.expand(
+ [
+ ("bucket_root", "", KeySelection(prefix="", listing_prefix="", pattern=None)),
+ ("folder", "raw/orders/", KeySelection(prefix="raw/orders", listing_prefix="", pattern=None)),
+ ("folder_or_file", "raw/orders", KeySelection(prefix="raw", listing_prefix="orders", pattern=None)),
+ ("file_at_the_root", "orders.csv", KeySelection(prefix="", listing_prefix="orders.csv", pattern=None)),
+ (
+ "glob_after_folders",
+ "raw/team_7/orders/**/*.parquet",
+ KeySelection(prefix="raw/team_7/orders", listing_prefix="", pattern="**/*.parquet"),
+ ),
+ (
+ "glob_inside_a_folder_name",
+ "raw/orders-*/a.csv",
+ KeySelection(prefix="raw", listing_prefix="", pattern="orders-*/a.csv"),
+ ),
+ ("glob_at_the_root", "*.csv", KeySelection(prefix="", listing_prefix="", pattern="*.csv")),
+ (
+ "glob_that_ends_at_a_folder",
+ "raw/*/",
+ KeySelection(prefix="raw", listing_prefix="", pattern="*/**"),
+ ),
+ ]
+ )
+ def test_select_keys(self, _name: str, key: str, expected: KeySelection) -> None:
+ assert select_keys(key) == expected
+
+ def test_a_glob_posthog_cant_read_is_an_invalid_config(self) -> None:
+ with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):
+ select_keys("raw/part-{1..2.csv")
+
+
+class TestUrlLocationChecks:
+ @parameterized.expand(
+ [
+ ("virtual_hosted", "https://posthog-owned.s3.us-east-1.amazonaws.com/a.csv"),
+ ("path_style", "https://s3.us-east-1.amazonaws.com/posthog-owned/a.csv"),
+ ]
+ )
+ def test_a_posthog_owned_bucket_is_blocked_in_either_url_form(self, _name: str, url: str) -> None:
+ config = ObjectStoreConfig.from_url(
+ ObjectStoreProvider.S3, parse_s3_url(url), access_key_id="AKIAEXAMPLE", secret_access_key="secret"
+ )
+
+ with override_settings(DATAWAREHOUSE_BUCKET="posthog-owned"):
+ with pytest.raises(ObjectStorageError, match=BLOCKED_LOCATION_ERROR):
+ validate_store_location(config)
+
+ @parameterized.expand(
+ [
+ (
+ "s3",
+ ObjectStoreConfig.from_url(
+ ObjectStoreProvider.S3,
+ parse_s3_url("https://s3.eu-west-1.amazonaws.com/acme-exports/raw/orders/*.csv"),
+ access_key_id="AKIAEXAMPLE",
+ secret_access_key="secret",
+ ),
+ "https://acme-exports.s3.eu-west-1.amazonaws.com/raw/orders/",
+ ),
+ (
+ "r2",
+ ObjectStoreConfig.from_url(
+ ObjectStoreProvider.R2,
+ parse_r2_url(f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/acme-exports/raw/a.csv"),
+ access_key_id="AKIAEXAMPLE",
+ secret_access_key="secret",
+ ),
+ f"https://{R2_ACCOUNT}.r2.cloudflarestorage.com/acme-exports/raw/",
+ ),
+ (
+ "azure",
+ AzureBlobConfig.from_url(
+ parse_azure_url("https://acmestore.blob.core.windows.net/exports/raw/**"),
+ account_key="c2VjcmV0LWtleQ==",
+ ),
+ "https://acmestore.blob.core.windows.net/exports/raw/",
+ ),
+ ]
+ )
+ def test_the_checked_location_is_rebuilt_from_the_parsed_parts(
+ self, _name: str, config: ObjectStoreConfig | AzureBlobConfig, expected: str
+ ) -> None:
+ assert config.location_url == expected
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index b9afff82a43e..f6bb421cf9fc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -241,15 +241,6 @@ class TestGroupTables:
DiscoveryOptions(),
[("region_eu_sales", "delta", None, "_region=eu/sales", "delta", [])],
),
- (
- "pattern_matches_across_folders",
- ["a/x.csv", "a/y.json", "b/z.csv"],
- DiscoveryOptions(file_pattern="*.csv"),
- [
- ("a", "files", "folder", "a", "csv", ["a/x.csv"]),
- ("b", "files", "folder", "b", "csv", ["b/z.csv"]),
- ],
- ),
(
"a_folder_with_two_formats_names_a_table_per_format",
["data/a.csv", "data/b.parquet"],
@@ -281,9 +272,96 @@ class TestGroupTables:
assert summarize(tables) == expected
+ @parameterized.expand(
+ [
+ (
+ "star_stays_in_its_folder",
+ ["a.csv", "daily/b.csv"],
+ "*.csv",
+ "",
+ DiscoveryOptions(),
+ [("a", "files", "file", "a.csv", "csv", ["a.csv"])],
+ ),
+ (
+ "double_star_crosses_folders",
+ ["a.csv", "daily/b.csv", "daily/c.json"],
+ "**.csv",
+ "",
+ DiscoveryOptions(),
+ [
+ ("a", "files", "file", "a.csv", "csv", ["a.csv"]),
+ ("daily", "files", "folder", "daily", "csv", ["daily/b.csv"]),
+ ],
+ ),
+ (
+ "underscore_hive_partitions_under_the_root_make_one_table",
+ [
+ "_part_key=2024-w10/part-0.parquet",
+ "_part_key=2024-w11/part-0.parquet",
+ "_part_key=2024-w11/_SUCCESS",
+ "notes.txt",
+ ],
+ "**/*.parquet",
+ "",
+ DiscoveryOptions(),
+ [
+ (
+ "orders",
+ "files",
+ "folder",
+ "",
+ "parquet",
+ ["_part_key=2024-w10/part-0.parquet", "_part_key=2024-w11/part-0.parquet"],
+ )
+ ],
+ ),
+ (
+ "a_delta_table_needs_the_pattern_to_match_one_of_its_files",
+ ["sales/_delta_log/0.json", "sales/p.parquet", "refunds/_delta_log/0.json", "a.csv"],
+ "**.parquet",
+ "",
+ DiscoveryOptions(),
+ [("sales", "delta", None, "sales", "delta", [])],
+ ),
+ ]
+ )
+ def test_url_selection(
+ self,
+ _name: str,
+ paths: list[str],
+ pattern: str | None,
+ listing_prefix: str,
+ options: DiscoveryOptions,
+ expected: list[tuple[str, str, str | None, str, str, list[str]]],
+ ) -> None:
+ tables = group_tables(
+ [stored(path) for path in paths],
+ options,
+ root_name="orders",
+ pattern=pattern,
+ listing_prefix=listing_prefix,
+ )
+
+ assert summarize(tables) == expected
+
+ def test_a_combined_table_lists_only_its_listing_prefix_at_sync(self, tmp_path: Path) -> None:
+ for path in ("daily/a.csv", "daily/2024/b.csv", "daily-archive/c.csv", "other.csv"):
+ (tmp_path / path).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / path).write_text("id\n1\n")
+ [table] = group_tables(
+ [stored("daily/a.csv")],
+ DiscoveryOptions(combined_table_name="daily"),
+ root_name="r",
+ listing_prefix="daily",
+ )
+
+ resolved = resolve_table_files(local_store(tmp_path), table)
+
+ assert [f.path for f in resolved.files] == ["daily/2024/b.csv", "daily/a.csv"]
+
def test_metadata_round_trip_rebuilds_the_same_table(self) -> None:
- options = DiscoveryOptions(file_format="csv_no_header", file_pattern="*.txt", allow_double_quotes=False)
- [table] = group_tables([stored("exports/a.txt")], options, root_name="root")
+ options = DiscoveryOptions(file_format="csv_no_header", allow_double_quotes=False)
+ [table] = group_tables([stored("exports/a.txt")], options, root_name="root", pattern="**.txt")
rebuilt = TableDefinition.from_schema_metadata(table.name, table.schema_metadata())
@@ -291,7 +369,7 @@ class TestGroupTables:
assert rebuilt.same_location(table)
assert (rebuilt.csv, rebuilt.pattern, rebuilt.source_format) == (
CsvOptions(has_header=False, allow_double_quotes=False),
- "*.txt",
+ "**.txt",
"csv_no_header",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py
index eb5ce3669ae2..f01015d41b1b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_providers.py
@@ -36,15 +36,18 @@ AZURE_KEY = "c2VjcmV0LWtleQ=="
S3_KEYS = {"access_key_id": "AKIAEXAMPLE", "secret_access_key": "secret"}
PROVIDERS: dict[str, tuple[ObjectStorageSource[Any], dict[str, Any]]] = {
- "s3": (AmazonS3Source(), {"bucket": "customer-data", "region": "eu-west-1", "path": "exports/", **S3_KEYS}),
- "gcs": (GoogleCloudStorageSource(), {"bucket": "customer_data", "path": "exports/", **S3_KEYS}),
+ "s3": (
+ AmazonS3Source(),
+ {"url_pattern": "https://customer-data.s3.eu-west-1.amazonaws.com/exports/", **S3_KEYS},
+ ),
+ "gcs": (GoogleCloudStorageSource(), {"url_pattern": "gs://customer_data/exports/", **S3_KEYS}),
"r2": (
CloudflareR2Source(),
- {"account_id": R2_ACCOUNT_ID, "bucket": "customer-data", "path": "exports/", **S3_KEYS},
+ {"url_pattern": f"https://{R2_ACCOUNT_ID}.r2.cloudflarestorage.com/customer-data/exports/", **S3_KEYS},
),
"azure": (
AzureBlobSource(),
- {"account_name": "customerstore", "container": "customer-data", "path": "exports/", "account_key": AZURE_KEY},
+ {"url_pattern": "https://customerstore.blob.core.windows.net/customer-data/exports/", "account_key": AZURE_KEY},
),
}
@@ -104,19 +107,20 @@ def test_each_provider_reads_from_its_own_endpoint(
@pytest.mark.parametrize(
"provider, overrides",
[
- ("gcs", {"bucket": "storage.googleapis.com/other"}),
- ("gcs", {"bucket": "[email protected]"}),
- ("r2", {"account_id": "evil.example.com/"}),
- ("r2", {"account_id": f"{R2_ACCOUNT_ID}.evil.example.com"}),
- ("r2", {"account_id": R2_ACCOUNT_ID[:-1]}),
- ("r2", {"bucket": "data_with_underscore"}),
- ("azure", {"account_name": "store.evil.example.com"}),
- ("azure", {"account_name": "store/x"}),
- ("azure", {"container": "data/../other"}),
- ("azure", {"container": "$root"}),
+ ("s3", {"url_pattern": "https://customer-data.s3.us-east-1.amazonaws.com.evil.example.com/a.csv"}),
+ ("s3", {"url_pattern": "https://evil.example.com/customer-data/a.csv"}),
+ ("s3", {"url_pattern": "https://s3.us-east-1.amazonaws.com/Not_A_Bucket/a.csv"}),
+ ("gcs", {"url_pattern": "https://storage.googleapis.com.evil.example.com/data/a.csv"}),
+ ("gcs", {"url_pattern": "https://[email protected]/data/a.csv"}),
+ ("gcs", {"url_pattern": "https://storage.googleapis.com:8443/data/a.csv"}),
+ ("r2", {"url_pattern": f"https://{R2_ACCOUNT_ID}.evil.example.com/data/a.csv"}),
+ ("r2", {"url_pattern": f"https://{R2_ACCOUNT_ID[:-1]}.r2.cloudflarestorage.com/data/a.csv"}),
+ ("r2", {"url_pattern": f"https://{R2_ACCOUNT_ID}.r2.cloudflarestorage.com/data_with_underscore/a.csv"}),
+ ("azure", {"url_pattern": "https://store.evil.example.com/data/a.csv"}),
+ ("azure", {"url_pattern": "https://customerstore.blob.core.windows.net/$root/a.csv"}),
+ ("azure", {"url_pattern": "https://customerstore.blob.core.windows.net/data/a/../b/x.csv"}),
("azure", {"account_key": f"{AZURE_KEY};BlobEndpoint=https://evil.example.com"}),
("azure", {"account_key": " "}),
- ("azure", {"path": "a/../b"}),
],
)
def test_provider_settings_that_could_retarget_the_request_are_rejected(
@@ -143,11 +147,12 @@ def test_provider_settings_that_could_retarget_the_request_are_rejected(
],
)
def test_gcs_bucket_name_length(bucket: str, valid: bool) -> None:
+ url_pattern = f"https://storage.googleapis.com/{bucket}/exports/"
if valid:
- assert _store_config("gcs", bucket=bucket).bucket == bucket
+ assert _store_config("gcs", url_pattern=url_pattern).bucket == bucket
else:
with pytest.raises(ObjectStorageError, match=INVALID_CONFIG_ERROR):
- _store_config("gcs", bucket=bucket)
+ _store_config("gcs", url_pattern=url_pattern)
@pytest.mark.parametrize("provider", list(PROVIDERS))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
index 677cff8744dd..24ba1c50e3ad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazons3.py
@@ -20,11 +20,8 @@ class AmazonS3FileFormatConfig(config.Config):
@config.config
class AmazonS3SourceConfig(config.Config):
- bucket: str
- region: str
+ url_pattern: str
access_key_id: str
secret_access_key: str
file_format: AmazonS3FileFormatConfig
- path: str | None = None
- file_pattern: str | None = None
combine_files: AmazonS3CombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py
index 775d87ca556d..3921c8cc2da9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureblob.py
@@ -20,10 +20,7 @@ class AzureBlobFileFormatConfig(config.Config):
@config.config
class AzureBlobSourceConfig(config.Config):
- account_name: str
- container: str
+ url_pattern: str
account_key: str
file_format: AzureBlobFileFormatConfig
- path: str | None = None
- file_pattern: str | None = None
combine_files: AzureBlobCombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py
index 9fba89debc62..c5877f25b57a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/cloudflarer2.py
@@ -20,11 +20,8 @@ class CloudflareR2FileFormatConfig(config.Config):
@config.config
class CloudflareR2SourceConfig(config.Config):
- account_id: str
- bucket: str
+ url_pattern: str
access_key_id: str
secret_access_key: str
file_format: CloudflareR2FileFormatConfig
- path: str | None = None
- file_pattern: str | None = None
combine_files: CloudflareR2CombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py
index 03340408394e..0f2acffdab22 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googlecloudstorage.py
@@ -20,10 +20,8 @@ class GoogleCloudStorageFileFormatConfig(config.Config):
@config.config
class GoogleCloudStorageSourceConfig(config.Config):
- bucket: str
+ url_pattern: str
access_key_id: str
secret_access_key: str
file_format: GoogleCloudStorageFileFormatConfig
- path: str | None = None
- file_pattern: str | None = None
combine_files: GoogleCloudStorageCombineFilesConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py
index eab891602901..db5cf46edca5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_cloud_storage/source.py
@@ -8,21 +8,23 @@ from products.warehouse_sources.backend.facade.source_config import (
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.config import (
- AUTO_REGION,
DiscoveryOptions,
ObjectStoreConfig,
ObjectStoreProvider,
- normalize_prefix,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.fields import (
FEATURE_FLAG,
KEYWORDS,
discovery_options_from,
file_option_fields,
- folder_fields,
secret_field,
source_caption,
text_field,
+ url_field,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.locations import (
+ GCS_EXAMPLE_URL,
+ parse_gcs_url,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.source_base import (
ObjectStorageSource,
@@ -52,17 +54,15 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfi
@property
def connection_host_fields(self) -> list[str]:
- # The bucket picks where the stored key is sent, so changing it must re-require the key.
- return ["bucket"]
+ # The URL picks the bucket the stored key is sent to, so changing it must re-require the key.
+ return ["url_pattern"]
def object_store_config(self, config: GoogleCloudStorageSourceConfig) -> ObjectStoreConfig:
- return ObjectStoreConfig(
- provider=ObjectStoreProvider.GCS,
- bucket=config.bucket.strip(),
- region=AUTO_REGION,
+ return ObjectStoreConfig.from_url(
+ ObjectStoreProvider.GCS,
+ parse_gcs_url(config.url_pattern),
access_key_id=config.access_key_id.strip(),
secret_access_key=config.secret_access_key.strip(),
- prefix=normalize_prefix(config.path),
)
def discovery_options(self, config: GoogleCloudStorageSourceConfig) -> DiscoveryOptions:
@@ -82,8 +82,7 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfi
fields=cast(
list[FieldType],
[
- text_field("bucket", "Bucket", "my-company-exports"),
- *folder_fields("bucket"),
+ url_field(GCS_EXAMPLE_URL),
text_field(
"access_key_id",
"HMAC access key",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
index 5c50e2c7a273..c566b672ea9f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
@@ -88,6 +88,7 @@ OK_ANSWERS: tuple[Answer, ...] = (
# connects without urllib3.
NOT_CHECKABLE: dict[str, str] = {
"Ably": "config",
+ "AmazonS3": "config",
"AppStoreConnect": "config",
"AppleSearchAds": "config",
"AzureBlob": "config",
@@ -116,6 +117,7 @@ NOT_CHECKABLE: dict[str, str] = {
"Github": "config",
"GoogleAds": "driver",
"GoogleAnalytics": "database",
+ "GoogleCloudStorage": "config",
"GooglePlayConsole": "config",
"GoogleSearchConsole": "database",
"GoogleSheets": "config",
@@ -384,7 +386,6 @@ OK_NEEDS_VENDOR_BODY: set[str] = {
"Adapty",
"AdobeAnalytics",
"Ahrefs",
- "AmazonS3",
"Anvil",
"Apollo",
"Arcade",
@@ -414,7 +415,6 @@ OK_NEEDS_VENDOR_BODY: set[str] = {
"Fieldpulse",
"GainsightCs",
"Getdx",
- "GoogleCloudStorage",
"GrowthBook",
"Gusto",
"Harness",
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index c31ebdc3d0ba..ddbed84fc07e 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -9342,8 +9342,7 @@ class TestExternalDataSource(APIBaseTest):
)
def test_source_behind_a_team_feature_flag(self, _name: str, path: str, flag_enabled: bool) -> None:
payload = {
- "bucket": "Not A Bucket",
- "region": "us-east-1",
+ "url_pattern": "https://files.example.com/orders/*.csv",
"access_key_id": "AKIAEXAMPLE",
"secret_access_key": "secret",
"file_format": {"selection": "infer", "csv_allow_double_quotes": "true"},
@@ -9358,7 +9357,7 @@ class TestExternalDataSource(APIBaseTest):
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
- # With the flag the request reaches credential validation, which rejects the bucket name
+ # With the flag the request reaches credential validation, which rejects the URL host
# before any network call.
expected = "The storage settings are invalid" if flag_enabled else "isn't available to connect yet"
assert expected in response.json()["message"]