Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

28.1 KB
Raw Blame
"""Centralized DuckLake object-storage credential handling.

This module provides a single entry point for all DuckLake storage credentials,
eliminating ad-hoc helpers spread across Temporal workflows and data warehouse utilities.

Usage:
    from products.managed_warehouse.backend.storage import DuckLakeStorageConfig, configure_connection

    # Get storage config from runtime environment
    storage_config = DuckLakeStorageConfig.from_runtime()

    # Configure a DuckDB connection with S3 credentials
    configure_connection(conn, storage_config)

    # Get options for deltalake library
    options = storage_config.to_deltalake_options()
"""

from __future__ import annotations

import re
import dataclasses
from typing import TYPE_CHECKING
from urllib.parse import urlparse

import psycopg

from posthog.dataclasses import frozen

from products.managed_warehouse.backend.common import (
    _get_org_id_for_team,
    default_bucket_region,
    escape as ducklake_escape,
    get_config,
    get_org_config,
    is_dev_mode,
)

if TYPE_CHECKING:
    import duckdb

    from products.managed_warehouse.backend.models import DuckgresServer


def _get_django_settings():
    """Lazy import Django settings to allow CLI tools to work without Django."""
    try:
        from django.conf import settings

        return settings
    except Exception:
        return None


@frozen
class AwsCredentials:
    access_key: str
    secret_key: str = dataclasses.field(repr=False)
    session_token: str | None = dataclasses.field(default=None, repr=False)


def _get_boto3_credentials() -> AwsCredentials:
    """Fetch AWS credentials via boto3.

    This is a workaround for DuckDB's CREDENTIAL_CHAIN not supporting
    IRSA (Web Identity Token) authentication. boto3 properly supports IRSA,
    so we use it to fetch credentials and pass them explicitly to DuckDB.

    See: https://github.com/duckdb/duckdb-aws/issues/31

    Returns:
        AwsCredentials; session_token may be None for static credentials.
    """
    import boto3

    session = boto3.Session()
    credentials = session.get_credentials()
    if credentials is None:
        raise RuntimeError("No AWS credentials available via boto3")
    frozen = credentials.get_frozen_credentials()
    if frozen.access_key is None or frozen.secret_key is None:
        raise RuntimeError("AWS credentials missing access_key or secret_key")
    return AwsCredentials(access_key=frozen.access_key, secret_key=frozen.secret_key, session_token=frozen.token)


def _get_cross_account_credentials(role_arn: str, external_id: str | None = None) -> AwsCredentials:
    """Assume a cross-account IAM role and return temporary credentials.

    Args:
        role_arn: The ARN of the role to assume in the destination account
        external_id: Optional external ID for the role assumption (recommended for security)

    Returns:
        AwsCredentials with a session token from the assumed role.
    """
    import boto3

    sts = boto3.client("sts")
    assume_role_kwargs: dict[str, str | int] = {
        "RoleArn": role_arn,
        "RoleSessionName": "ducklake-cross-account",
        "DurationSeconds": 3600,
    }
    if external_id:
        assume_role_kwargs["ExternalId"] = external_id

    response = sts.assume_role(**assume_role_kwargs)
    creds = response["Credentials"]
    return AwsCredentials(
        access_key=creds["AccessKeyId"], secret_key=creds["SecretAccessKey"], session_token=creds["SessionToken"]
    )


def normalize_endpoint(endpoint: str) -> tuple[str, bool]:
    """Normalize object storage endpoint URL.

    Extracts the host from a URL and determines SSL usage.

    Args:
        endpoint: The endpoint URL (e.g., "http://localhost:19000" or "localhost:19000")

    Returns:
        Tuple of (normalized_host, use_ssl)
    """
    value = endpoint.strip()
    if not value:
        return "", True

    if "://" in value:
        parsed = urlparse(value)
        normalized = parsed.netloc or parsed.path
        use_ssl = parsed.scheme.lower() == "https"
    else:
        use_ssl = value.lower().startswith("https")
        normalized = value.rstrip("/")

    return normalized.rstrip("/") or "", use_ssl


@dataclasses.dataclass(frozen=True)
class DuckLakeStorageConfig:
    """Configuration for DuckLake object storage access.

    This dataclass encapsulates all credentials and settings needed to access
    DuckLake storage, providing consistent serialization for DuckDB secrets,
    deltalake options, and boto3/s3fs clients.

    Use DuckLakeStorageConfig.from_runtime() to create an instance from the
    current runtime environment.
    """

    access_key: str
    secret_key: str = dataclasses.field(repr=False)
    region: str
    endpoint: str
    use_ssl: bool
    url_style: str
    is_local: bool

    @classmethod
    def from_runtime(
        cls,
        *,
        use_local_setup: bool | None = None,
        team_id: int | None = None,
        organization_id: str | None = None,
    ) -> DuckLakeStorageConfig:
        """Create storage config from current runtime environment.

        This factory method encapsulates the USE_LOCAL_SETUP branching logic,
        loading credentials from environment variables for local dev or falling
        back to IRSA credential chain for production.

        Args:
            use_local_setup: Override for USE_LOCAL_SETUP setting. If None,
                reads from Django settings or defaults to True for CLI tools.
            team_id: Optional team ID to look up team-specific configuration.
            organization_id: Optional organization ID. When provided, skips
                the Team→Organization DB lookup.

        Returns:
            DuckLakeStorageConfig instance with appropriate credentials.
        """
        if organization_id is not None:
            config = get_org_config(organization_id)
        elif team_id is not None:
            if is_dev_mode():
                config = get_config()
            else:
                config = get_org_config(_get_org_id_for_team(team_id))
        else:
            config = get_config()
        settings = _get_django_settings()

        if use_local_setup is None:
            use_local_setup = getattr(settings, "USE_LOCAL_SETUP", True) if settings else True

        access_key = config.get("DUCKLAKE_S3_ACCESS_KEY", "")
        secret_key = config.get("DUCKLAKE_S3_SECRET_KEY", "")
        region = config.get("DUCKLAKE_BUCKET_REGION") or default_bucket_region()

        raw_endpoint = getattr(settings, "OBJECT_STORAGE_ENDPOINT", "") if settings else ""

        if use_local_setup:
            normalized_endpoint, use_ssl = normalize_endpoint(raw_endpoint or "")
            return cls(
                access_key=access_key,
                secret_key=secret_key,
                region=region,
                endpoint=normalized_endpoint,
                use_ssl=use_ssl,
                url_style="path",
                is_local=True,
            )
        else:
            return cls(
                access_key="",
                secret_key="",
                region=region,
                endpoint="",
                use_ssl=True,
                url_style="path",
                is_local=False,
            )

    def to_duckdb_secret_sql(self) -> str:
        """Generate DuckDB CREATE SECRET SQL statement.

        Returns:
            SQL statement to create the S3 secret.
        """
        secret_name = "ducklake_s3"
        if not self.is_local:
            # Workaround: DuckDB's CREDENTIAL_CHAIN doesn't support IRSA (Web Identity Token).
            # Fetch credentials via boto3 which properly supports IRSA.
            # See: https://github.com/duckdb/duckdb-aws/issues/31
            creds = _get_boto3_credentials()
            secret_parts = [
                "TYPE S3",
                f"KEY_ID '{ducklake_escape(creds.access_key)}'",
                f"SECRET '{ducklake_escape(creds.secret_key)}'",
            ]
            if creds.session_token:
                secret_parts.append(f"SESSION_TOKEN '{ducklake_escape(creds.session_token)}'")
            if self.region:
                secret_parts.append(f"REGION '{ducklake_escape(self.region)}'")
            return f"CREATE OR REPLACE SECRET {secret_name} ({', '.join(secret_parts)})"

        secret_parts = ["TYPE S3"]
        if self.access_key:
            secret_parts.append(f"KEY_ID '{ducklake_escape(self.access_key)}'")
        if self.secret_key:
            secret_parts.append(f"SECRET '{ducklake_escape(self.secret_key)}'")
        if self.region:
            secret_parts.append(f"REGION '{ducklake_escape(self.region)}'")
        if self.endpoint:
            secret_parts.append(f"ENDPOINT '{ducklake_escape(self.endpoint)}'")
        secret_parts.append(f"USE_SSL {'true' if self.use_ssl else 'false'}")
        secret_parts.append(f"URL_STYLE '{ducklake_escape(self.url_style)}'")

        return f"CREATE OR REPLACE SECRET {secret_name} ({', '.join(secret_parts)})"

    def to_duckdb_scoped_secret_sql(
        self,
        secret_name: str,
        scope: str,
        *,
        credentials: AwsCredentials,
        region: str | None = None,
    ) -> str:
        """Generate a scoped DuckDB CREATE SECRET statement.

        DuckDB scoped secrets allow different credentials for different S3 paths.
        The secret with the most specific matching scope is used for each operation.

        Args:
            secret_name: Unique name for this secret
            scope: S3 bucket scope (e.g., 's3://bucket-name')
            credentials: AWS credentials for the scoped secret
            region: AWS region (defaults to self.region)

        Returns:
            SQL statement to create the scoped S3 secret
        """
        effective_region = region or self.region
        if not effective_region:
            raise ValueError("Region is required for scoped S3 secrets")

        secret_parts = [
            "TYPE S3",
            f"KEY_ID '{ducklake_escape(credentials.access_key)}'",
            f"SECRET '{ducklake_escape(credentials.secret_key)}'",
        ]
        if credentials.session_token:
            secret_parts.append(f"SESSION_TOKEN '{ducklake_escape(credentials.session_token)}'")
        secret_parts.append(f"REGION '{ducklake_escape(effective_region)}'")
        secret_parts.append(f"SCOPE '{ducklake_escape(scope)}'")

        return f"CREATE OR REPLACE SECRET {secret_name} ({', '.join(secret_parts)})"

    def to_deltalake_options(self) -> dict[str, str]:
        """Generate storage options for deltalake library.

        Returns:
            Dict of storage options to pass to deltalake.DeltaTable.
        """
        options: dict[str, str] = {
            "AWS_S3_ALLOW_UNSAFE_RENAME": "true",
        }

        if not self.is_local:
            if self.region:
                options["AWS_DEFAULT_REGION"] = self.region
            return options

        if self.access_key:
            options["aws_access_key_id"] = self.access_key
        if self.secret_key:
            options["aws_secret_access_key"] = self.secret_key
        if self.region:
            options["region_name"] = self.region
            options["AWS_DEFAULT_REGION"] = self.region
        if self.endpoint:
            settings = _get_django_settings()
            raw_endpoint = getattr(settings, "OBJECT_STORAGE_ENDPOINT", "") if settings else ""
            options["endpoint_url"] = raw_endpoint or self.endpoint
            options["AWS_ALLOW_HTTP"] = "true"

        return {key: value for key, value in options.items() if value}


def configure_connection(
    conn: duckdb.DuckDBPyConnection,
    storage_config: DuckLakeStorageConfig | None = None,
    *,
    install_extensions: bool = True,
) -> None:
    """Configure DuckDB connection for DuckLake storage access.

    Installs and loads necessary extensions (httpfs, delta) and creates the
    S3 secret for accessing object storage.

    Args:
        conn: DuckDB connection to configure.
        storage_config: Storage config to use. If None, creates one from runtime.
        install_extensions: Whether to install httpfs and delta extensions.
    """
    if storage_config is None:
        storage_config = DuckLakeStorageConfig.from_runtime()

    if install_extensions:
        conn.execute("INSTALL ducklake")
        conn.execute("INSTALL httpfs")
        conn.execute("INSTALL delta")

    conn.execute("LOAD ducklake")
    # Default inlining off. DuckLake's per-catalog `data_inlining_row_limit`
    # is cached in-memory at ATTACH time and never refreshed, so a SET on
    # `ducklake_metadata` doesn't reach long-lived workers. Setting the
    # session-level fallback here lets every connection default to no
    # inlining regardless of when the underlying worker first ATTACHed the
    # catalog (see ducklake_catalog.cpp DataInliningRowLimit lookup order).
    conn.execute("SET ducklake_default_data_inlining_row_limit = 0")
    conn.execute("LOAD httpfs")
    conn.execute("LOAD delta")
    conn.execute(storage_config.to_duckdb_secret_sql())


@dataclasses.dataclass(frozen=True)
class CrossAccountDestination:
    """Configuration for a cross-account S3 destination.

    Used with configure_cross_account_connection() to set up DuckDB credentials
    for writing to S3 buckets in different AWS accounts.

    Attributes:
        role_arn: The ARN of the IAM role to assume in the destination account
        bucket_name: The name of the destination S3 bucket
        external_id: Optional external ID for role assumption (recommended for security)
        region: Optional AWS region for the destination bucket
    """

    role_arn: str
    bucket_name: str
    external_id: str | None = None
    region: str | None = None


def configure_cross_account_connection(
    conn: duckdb.DuckDBPyConnection,
    source_storage_config: DuckLakeStorageConfig | None = None,
    destinations: list[CrossAccountDestination] | None = None,
    *,
    install_extensions: bool = True,
) -> None:
    """Configure DuckDB connection for cross-account S3 access.

    Sets up scoped secrets so DuckDB automatically uses the correct credentials
    based on the S3 path being accessed. Source bucket uses IRSA credentials,
    while destination buckets use credentials from assumed cross-account roles.

    Args:
        conn: DuckDB connection to configure
        source_storage_config: Storage config for the source bucket (uses IRSA credentials).
            If None, creates one from runtime.
        destinations: List of cross-account destinations to configure. Each destination
            will have its role assumed and a scoped secret created for its bucket.
        install_extensions: Whether to install DuckDB extensions
    """
    if source_storage_config is None:
        source_storage_config = DuckLakeStorageConfig.from_runtime()

    if install_extensions:
        conn.execute("INSTALL ducklake")
        conn.execute("INSTALL httpfs")
        conn.execute("INSTALL delta")

    conn.execute("LOAD ducklake")
    # See configure_connection for why this session-level fallback is set.
    conn.execute("SET ducklake_default_data_inlining_row_limit = 0")
    conn.execute("LOAD httpfs")
    conn.execute("LOAD delta")

    # Set up source credentials (from IRSA or local config)
    conn.execute(source_storage_config.to_duckdb_secret_sql())

    # Set up destination credentials (via cross-account role assumption)
    if destinations:
        for i, dest in enumerate(destinations):
            creds = _get_cross_account_credentials(
                dest.role_arn,
                external_id=dest.external_id,
            )
            secret_sql = source_storage_config.to_duckdb_scoped_secret_sql(
                secret_name=f"ducklake_s3_dest_{i}",
                scope=f"s3://{dest.bucket_name}",
                credentials=creds,
                region=dest.region,
            )
            conn.execute(secret_sql)


def ensure_ducklake_bucket_exists(
    storage_config: DuckLakeStorageConfig | None = None,
    config: dict[str, str] | None = None,
    *,
    team_id: int | None = None,
    organization_id: str | None = None,
) -> None:
    """Ensure the DuckLake bucket exists (local dev only).

    This is a no-op in production environments. In local dev, it creates the
    bucket if it doesn't exist.

    Args:
        storage_config: Storage config to use. If None, creates one from runtime.
        config: DuckLake config dict. If None, resolved from team_id or get_config().
        team_id: Optional team ID to look up team-specific configuration.
        organization_id: Optional organization ID. When provided, skips
            the Team→Organization DB lookup.
    """
    if storage_config is None:
        storage_config = DuckLakeStorageConfig.from_runtime(team_id=team_id, organization_id=organization_id)

    if not storage_config.is_local:
        return

    if config is None:
        if organization_id is not None:
            config = get_org_config(organization_id)
        elif team_id is not None:
            if is_dev_mode():
                config = get_config()
            else:
                config = get_org_config(_get_org_id_for_team(team_id))
        else:
            config = get_config()

    from products.data_warehouse.backend.facade.api import ensure_bucket_exists

    settings = _get_django_settings()
    raw_endpoint = getattr(settings, "OBJECT_STORAGE_ENDPOINT", "") if settings else ""

    ensure_bucket_exists(
        f"s3://{config['DUCKLAKE_BUCKET'].rstrip('/')}",
        storage_config.access_key,
        storage_config.secret_key,
        raw_endpoint,
    )


def get_deltalake_storage_options(
    storage_config: DuckLakeStorageConfig | None = None,
    *,
    team_id: int | None = None,
    organization_id: str | None = None,
) -> dict[str, str]:
    """Get storage options for deltalake library.

    Convenience function that creates a storage config from runtime if not provided.

    Args:
        storage_config: Storage config to use. If None, creates one from runtime.
        team_id: Optional team ID to look up team-specific configuration.
        organization_id: Optional organization ID. When provided, skips
            the Team→Organization DB lookup.

    Returns:
        Dict of storage options to pass to deltalake.DeltaTable.
    """
    if storage_config is None:
        storage_config = DuckLakeStorageConfig.from_runtime(team_id=team_id, organization_id=organization_id)
    return storage_config.to_deltalake_options()


STAGING_PREFIX = "__posthog_staging"


@dataclasses.dataclass(frozen=True, kw_only=True)
class DeltaTableSnapshotWorkload:
    file_count: int
    row_count: int | None
    byte_count: int | None


@dataclasses.dataclass(frozen=True, kw_only=True)
class StagedDeltaTable:
    staging_uri: str
    workload: DeltaTableSnapshotWorkload


def compute_staging_uri(source_uri: str, catalog_bucket: str) -> str:
    """Place source key path under __posthog_staging/ in the catalog bucket."""
    key_path = urlparse(source_uri).path.lstrip("/")
    return f"s3://{catalog_bucket}/{STAGING_PREFIX}/{key_path}"


def _get_delta_snapshot(
    source_uri: str,
    *,
    storage_config: DuckLakeStorageConfig | None = None,
    team_id: int | None = None,
    organization_id: str | None = None,
) -> tuple[int, list[str], DeltaTableSnapshotWorkload]:
    import deltalake

    delta_table = deltalake.DeltaTable(
        table_uri=source_uri,
        storage_options=get_deltalake_storage_options(
            storage_config=storage_config,
            team_id=team_id,
            organization_id=organization_id,
        ),
    )
    version = delta_table.version()
    data_keys = [urlparse(uri).path.lstrip("/") for uri in delta_table.file_uris()]

    add_actions = delta_table.get_add_actions(flatten=True)
    column_names = add_actions.schema.names

    byte_count: int | None = None
    if "size_bytes" in column_names:
        sizes = add_actions.column("size_bytes").to_pylist()
        if all(size is not None for size in sizes):
            byte_count = sum(sizes)

    row_count: int | None = None
    if "num_records" in column_names:
        record_counts = add_actions.column("num_records").to_pylist()
        if all(record_count is not None for record_count in record_counts):
            row_count = sum(record_counts)

    workload = DeltaTableSnapshotWorkload(
        file_count=len(data_keys),
        row_count=row_count,
        byte_count=byte_count,
    )
    return version, data_keys, workload


def _get_delta_snapshot_files(
    source_uri: str,
    *,
    storage_config: DuckLakeStorageConfig | None = None,
    team_id: int | None = None,
    organization_id: str | None = None,
) -> tuple[int, list[str]]:
    """Pin to the current Delta table version and return its data file S3 keys."""
    import deltalake

    delta_table = deltalake.DeltaTable(
        table_uri=source_uri,
        storage_options=get_deltalake_storage_options(
            storage_config=storage_config,
            team_id=team_id,
            organization_id=organization_id,
        ),
    )
    version = delta_table.version()
    data_keys = [urlparse(uri).path.lstrip("/") for uri in delta_table.file_uris()]
    return version, data_keys


def get_delta_table_snapshot_workload(
    source_uri: str,
    *,
    storage_config: DuckLakeStorageConfig | None = None,
    team_id: int | None = None,
    organization_id: str | None = None,
) -> DeltaTableSnapshotWorkload:
    _, _, workload = _get_delta_snapshot(
        source_uri,
        storage_config=storage_config,
        team_id=team_id,
        organization_id=organization_id,
    )
    return workload


_DELTA_LOG_VERSION_RE = re.compile(r"^(\d{20})\.")


def _collect_delta_log_keys(
    s3_client,
    bucket: str,
    prefix: str,
    max_version: int,
) -> list[str]:
    """List Delta log entries up to *max_version* (inclusive).

    Scans ``{prefix}_delta_log/`` and keeps:
    - JSON commit files (``00000000000000000001.json``)
    - Checkpoint parquet files (``00000000000000000010.checkpoint.parquet``,
      multi-part variants like ``00000000000000000010.checkpoint.0000000001.0000000002.parquet``)

    ``_last_checkpoint`` is intentionally excluded — it may reference a
    checkpoint newer than *max_version*.  Delta readers handle its absence
    by scanning the log directory directly.

    Files whose 20-digit version prefix exceeds *max_version* are excluded.
    """
    log_prefix = f"{prefix}_delta_log/"
    keys: list[str] = []
    paginator = s3_client.get_paginator("list_objects_v2")
    for page in paginator.paginate(Bucket=bucket, Prefix=log_prefix):
        for obj in page.get("Contents", []):
            key = obj["Key"]
            filename = key[len(log_prefix) :]

            m = _DELTA_LOG_VERSION_RE.match(filename)
            if m and int(m.group(1)) <= max_version:
                keys.append(key)

    return keys


def stage_delta_table(
    source_uri: str,
    catalog_bucket: str,
    *,
    storage_config: DuckLakeStorageConfig | None = None,
    team_id: int | None = None,
    organization_id: str | None = None,
) -> str:
    """Copy a version-pinned Delta table snapshot to the catalog staging prefix."""
    version, data_keys = _get_delta_snapshot_files(
        source_uri,
        storage_config=storage_config,
        team_id=team_id,
        organization_id=organization_id,
    )
    return _copy_delta_snapshot_to_staging(source_uri, catalog_bucket, version, data_keys)


def stage_delta_table_with_workload(
    source_uri: str,
    catalog_bucket: str,
    *,
    storage_config: DuckLakeStorageConfig | None = None,
    team_id: int | None = None,
    organization_id: str | None = None,
) -> StagedDeltaTable:
    """Stage a version-pinned Delta table and return its transaction-log workload."""
    version, data_keys, workload = _get_delta_snapshot(
        source_uri,
        storage_config=storage_config,
        team_id=team_id,
        organization_id=organization_id,
    )
    staging_uri = _copy_delta_snapshot_to_staging(source_uri, catalog_bucket, version, data_keys)
    return StagedDeltaTable(staging_uri=staging_uri, workload=workload)


def _copy_delta_snapshot_to_staging(source_uri: str, catalog_bucket: str, version: int, data_keys: list[str]) -> str:
    from concurrent.futures import ThreadPoolExecutor

    import boto3

    parsed = urlparse(source_uri)
    source_bucket = parsed.netloc
    source_prefix = parsed.path.lstrip("/")
    if not source_prefix.endswith("/"):
        source_prefix += "/"

    s3 = boto3.client("s3")
    log_keys = _collect_delta_log_keys(s3, source_bucket, source_prefix, version)
    objects_to_copy = data_keys + log_keys

    def copy_one(key: str) -> None:
        staging_key = f"{STAGING_PREFIX}/{key}"
        s3.copy_object(
            Bucket=catalog_bucket,
            Key=staging_key,
            CopySource={"Bucket": source_bucket, "Key": key},
        )

    with ThreadPoolExecutor(max_workers=10) as executor:
        list(executor.map(copy_one, objects_to_copy))

    return compute_staging_uri(source_uri, catalog_bucket)


def cleanup_staged_files(
    staging_uri: str,
) -> None:
    """Delete staged Delta files from the staging bucket."""
    import boto3

    parsed = urlparse(staging_uri)
    bucket = parsed.netloc
    prefix = parsed.path.lstrip("/")
    if not prefix.endswith("/"):
        prefix += "/"

    s3 = boto3.client("s3")

    paginator = s3.get_paginator("list_objects_v2")
    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
        objects: list[dict[str, str]] = [{"Key": obj["Key"]} for obj in page.get("Contents", [])]
        if objects:
            s3.delete_objects(Bucket=bucket, Delete={"Objects": objects})  # type: ignore[typeddict-item]


def setup_duckgres_session(
    conn: psycopg.Connection,
    extensions: tuple[str, ...] = ("ducklake", "httpfs", "delta"),
) -> None:
    """Install and load required extensions on a duckgres connection.

    Callers should request only what they use: extensions bundled in the duckgres
    worker image (httpfs, ducklake) make INSTALL a local no-op, but anything else
    triggers a CDN download that egress-restricted workers silently drop.
    """
    for ext in extensions:
        conn.execute(f"INSTALL {ext}")
        conn.execute(f"LOAD {ext}")


def create_staging_read_secret(conn: psycopg.Connection, catalog_bucket: str) -> None:
    """Pin HTTPS for delta-kernel reads of the staging tree on this session.

    On cache-proxy-enabled duckgres clusters the org's ambient S3 secret carries
    USE_SSL false so httpfs traffic flows through the logging proxy — but
    delta_scan's delta-kernel object store ignores DuckDB's proxy and dials the
    plain-HTTP S3 endpoint directly, which worker egress silently drops (10
    retries, ~58s, then failure — on every attempt). This longer-SCOPE session
    secret wins prefix resolution for the staging tree only and forces direct
    HTTPS (allowed egress) using the worker's own ambient credentials; the
    region also resolves from the worker's chain. Session-scoped on purpose:
    duckgres wipes non-persistent secrets at the next session create.
    """
    conn.execute(
        "CREATE OR REPLACE SECRET posthog_staging_delta_https ("
        "TYPE S3, PROVIDER credential_chain, USE_SSL true, "
        f"SCOPE 's3://{catalog_bucket}/{STAGING_PREFIX}')"
    )


def connect_to_duckgres(
    server: DuckgresServer,
    *,
    application_name: str = "posthog",
    options: str | None = None,
) -> psycopg.Connection:
    """Open a psycopg connection to a duckgres server.

    ``application_name`` is a caller-identifying slug echoed into duckgres's
    analytics events, so callers should pass one instead of relying on the
    ``"posthog"`` default.

    ``options`` is forwarded as the libpq startup options for caller-specific
    Duckgres settings.
    """
    return psycopg.connect(
        host=server.host,
        port=server.port,
        dbname=server.database,
        user=server.username,
        password=server.password,
        autocommit=True,
        application_name=application_name,
        options=options,
    )


__all__ = [
    "DuckLakeStorageConfig",
    "CrossAccountDestination",
    "cleanup_staged_files",
    "compute_staging_uri",
    "configure_connection",
    "configure_cross_account_connection",
    "connect_to_duckgres",
    "create_staging_read_secret",
    "ensure_ducklake_bucket_exists",
    "get_deltalake_storage_options",
    "normalize_endpoint",
    "setup_duckgres_session",
    "stage_delta_table",
]