Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

25.3 KB
Raw Blame
"""Databricks driver for PostHog's data-warehouse import pipeline.

Everything Databricks-specific — connection lifecycle (personal access
token vs OAuth M2M service principal auth), Unity Catalog
`information_schema` discovery, and the pipeline build — lives on
`DatabricksImplementation`. The source-class `DatabricksSource` is a thin
PostHog-layer wrapper that holds an instance and validates credentials.
"""

from __future__ import annotations

import time
import collections
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any, Optional, TypeVar

import structlog
from databricks import sql as databricks_sql
from databricks.sdk.core import (
    Config as DatabricksSdkConfig,
    oauth_service_principal,
)
from structlog.types import FilteringBoundLogger

from products.warehouse_sources.sources.databricks._config import DatabricksSourceConfig
from products.warehouse_sources.sources.sdk import (
    NOT_A_DATABRICKS_HOST_MESSAGE,
    BacktickIdentifierQuoter,
    DeadlineExceededError,
    HostNotAllowedError,
    IncrementalFieldFilter,
    IncrementalFieldType,
    NonReportableError,
    ParamStyle,
    SelectQueryBuilder,
    SourceInputs,
    SourceMetadata,
    SourceResponse,
    SQLSourceImplementation,
    TableStats,
    is_databricks_workspace_host,
    log_connection_open,
    normalize_namespace,
    resolve_source_location,
    run_with_deadline,
)

__all__ = [
    "DatabricksImplementation",
    "DatabricksMetadataTimeoutError",
    "clean_databricks_host",
    "filter_databricks_incremental_fields",
    "run_schema_discovery_with_deadline",
]

_T = TypeVar("_T")

_IDENTIFIER_QUOTER = BacktickIdentifierQuoter()

# Databricks native (server-side) parameters use PEP-249 `named` paramstyle (`:name` + dict).
_QUERY_BUILDER = SelectQueryBuilder(quoter=_IDENTIFIER_QUOTER, param_style=ParamStyle.NAMED)

# Unity Catalog metadata schema present in every catalog; never user data.
DATABRICKS_SYSTEM_SCHEMA = "information_schema"

# `fetchmany_arrow` size. CloudFetch already chunks the result set into presigned cloud-storage
# files behind the cursor, so this only bounds how many rows we materialize per yielded batch.
DEFAULT_DATABRICKS_FETCH_SIZE = 10_000

# Number of times `connect` will open a fresh Databricks SQL connection before giving up. Mirrors
# the MySQL/Postgres sources' own in-process connect retry budget.
_MAX_CONNECT_ATTEMPTS = 5

# The connector's own request-retry loop (`ThriftBackend.make_request`) only retries HTTP 429/503
# responses and `GetOperationStatus` polling failures — a bare SSL error raised while opening the
# session (including fetching the initial OAuth service-principal token from `/oidc/v1/token`) is
# classified non-retryable and raised immediately on the first attempt. `[SSL:
# UNEXPECTED_EOF_WHILE_READING]` is the peer closing the TLS connection mid-handshake — an
# overloaded endpoint, a proxy/load-balancer idle cull, or a momentary network blip, all of which a
# fresh attempt recovers from. Mirrors the MySQL source's `_is_transient_connect_drop`.
_SSL_UNEXPECTED_EOF_TOKEN = "[SSL: UNEXPECTED_EOF_WHILE_READING]"


def _is_transient_connect_ssl_eof(e: BaseException) -> bool:
    """Return True if connect failed on a transient SSL peer-close mid-handshake."""
    return isinstance(e, databricks_sql.Error) and _SSL_UNEXPECTED_EOF_TOKEN in str(e)


# Schema discovery must end before the 10 minute `start_to_close_timeout` of its Temporal
# activity (`DiscoverSchemasWorkflow.DISCOVER_SCHEMAS_ACTIVITY_TIMEOUT`). The Thrift connector has
# no limit of its own on the wall-clock duration of a running statement — `_socket_timeout` only
# bounds each individual polling request, so a legitimately slow `information_schema.columns` scan
# (e.g. listing every schema in a wide catalog) can run past the activity's budget. Temporal then
# cancels the activity thread mid-poll, and the connector surfaces that as a generic
# `databricks.sql.exc.RequestError: Error during request to server. Cancelled` — a confusing error
# that hides the real cause. Run discovery under this deadline instead, so it fails with its own
# clear, retryable error well before that happens. Mirrors the MSSQL source's
# `MSSQL_SCHEMA_DISCOVERY_DEADLINE_SECONDS`.
DATABRICKS_SCHEMA_DISCOVERY_DEADLINE_SECONDS = 300

# Stable prefix, matched by tests. Kept out of `get_non_retryable_errors` — the next scheduled
# discovery pass retries from scratch, and a transient slow catalog scan can succeed then.
DATABRICKS_METADATA_TIMEOUT_ERROR = "Databricks did not answer in time"


class DatabricksMetadataTimeoutError(NonReportableError):
    """Databricks gave no answer to a schema-discovery query before the deadline."""

    def __init__(self, timeout_seconds: float) -> None:
        super().__init__(
            f"{DATABRICKS_METADATA_TIMEOUT_ERROR} while PostHog listed tables and columns "
            f"(waited {timeout_seconds:g} seconds)"
        )


def run_schema_discovery_with_deadline(operation: Callable[[], _T], *, timeout_seconds: float) -> _T:
    """Run schema-discovery `operation`, and raise `DatabricksMetadataTimeoutError` when it hangs."""
    try:
        return run_with_deadline(operation, timeout_seconds=timeout_seconds, thread_name="databricks-metadata")
    except DeadlineExceededError as e:
        raise DatabricksMetadataTimeoutError(e.timeout_seconds) from e


def _connect_with_transient_retry(**kwargs: Any) -> Any:
    """Open a Databricks SQL connection, retrying a transient SSL peer-close on connect."""
    attempt = 0
    while True:
        try:
            return databricks_sql.connect(**kwargs)
        except Exception as e:
            attempt += 1
            if attempt >= _MAX_CONNECT_ATTEMPTS or not _is_transient_connect_ssl_eof(e):
                raise
            structlog.get_logger().warning(
                "Transient Databricks connection error during connect; retrying",
                attempt=attempt,
                max_attempts=_MAX_CONNECT_ATTEMPTS,
                exc_info=e,
            )
            time.sleep(min(2 * attempt, 30))


def filter_databricks_incremental_fields(
    columns: list[tuple[str, str, bool]],
    # nosemgrep: tuple-return-prefer-dataclass -- pre-existing, file moved
) -> list[tuple[str, IncrementalFieldType, bool]]:
    results: list[tuple[str, IncrementalFieldType, bool]] = []
    for column_name, type, nullable in columns:
        type = type.lower()
        if type.startswith("timestamp"):
            # Covers TIMESTAMP and TIMESTAMP_NTZ.
            results.append((column_name, IncrementalFieldType.Timestamp, nullable))
        elif type == "date":
            results.append((column_name, IncrementalFieldType.Date, nullable))
        elif type in ("bigint", "int", "smallint", "tinyint") or type.startswith("decimal"):
            results.append((column_name, IncrementalFieldType.Numeric, nullable))

    return results


def clean_databricks_host(host: str) -> str:
    """Normalize a pasted workspace URL down to the bare hostname the connector expects."""
    host = host.strip()
    for prefix in ("https://", "http://"):
        if host.lower().startswith(prefix):
            host = host[len(prefix) :]
    return host.rstrip("/").lower()


def _split_display_name(display_name: str, default_schema: Optional[str]) -> tuple[Optional[str], str]:
    """Split a `schema.table` display name into `(schema, table)`.

    Multi-schema discovery qualifies every table as `schema.table`; a single-schema
    source keeps bare table names and falls back to the configured schema. The dotted
    form mirrors `resolve_source_location`'s self-heal so listing keys and per-row
    routing agree.
    """
    if "." in display_name:
        schema, _, table = display_name.partition(".")
        return (normalize_namespace(schema) or default_schema), table
    return default_schema, display_name


class DatabricksImplementation(SQLSourceImplementation[DatabricksSourceConfig, Any, Any]):
    """Databricks driver implementation paired with `DatabricksSource`.

    One class owns everything Databricks-specific: the connection lifecycle
    (personal access token vs OAuth M2M service principal), the Unity Catalog
    `information_schema` batch queries used during schema listing, and the
    pipeline factory (`build_pipeline`).

    Databricks does not use the optional streaming-side hooks
    (`fetch_table_stats`, `fetch_average_row_size`, `get_table_metadata`) —
    it streams Arrow batches via `cursor.fetchmany_arrow()` (CloudFetch) and
    lets the base class return `None` for partition settings, matching the
    Snowflake driver.
    """

    # ------------------------------------------------------------------
    # Connection lifecycle
    # ------------------------------------------------------------------

    @contextmanager
    def connect(self, config: DatabricksSourceConfig, *, team_id: int | None = None) -> Iterator[Any]:
        """Open a Databricks SQL warehouse connection for the duration of the context.

        Branches on `config.auth_type.selection`: a personal access token is
        passed directly, while a service principal goes through the SDK's
        OAuth M2M credentials provider (1-hour tokens, auto-refreshed).

        Uses `config.catalog` as the session catalog so all two-part
        `schema.table` references resolve inside it; a blank schema
        (multi-schema import) leaves the session schema unset.
        """
        host = clean_databricks_host(config.host)
        # The SDK and connector send requests to URLs the server returns, so the server must be one
        # Databricks runs. `validate_credentials` checks this too, but scheduled syncs never call it.
        if not is_databricks_workspace_host(host):
            raise HostNotAllowedError(NOT_A_DATABRICKS_HOST_MESSAGE)

        auth_connect_args: dict[str, Any] = {}

        if config.auth_type.selection == "service_principal":
            client_id = config.auth_type.client_id
            client_secret = config.auth_type.client_secret

            def credentials_provider() -> Callable[[], dict[str, str]]:
                return oauth_service_principal(
                    DatabricksSdkConfig(
                        host=f"https://{host}",
                        client_id=client_id,
                        client_secret=client_secret,
                        auth_type="oauth-m2m",
                    )
                )

            auth_connect_args = {"credentials_provider": credentials_provider}
        else:
            auth_connect_args = {"access_token": config.auth_type.access_token}

        log_connection_open(db_host=host, via="vendor_https")
        connection = _connect_with_transient_retry(
            server_hostname=host,
            http_path=config.http_path,
            catalog=config.catalog,
            # A blank schema (multi-schema import) must reach the connector as None, not "" —
            # `schema=""` would try `USE SCHEMA ""`, an invalid identifier, and fail at connect time.
            schema=normalize_namespace(config.schema),
            user_agent_entry="PostHog",
            enable_telemetry=False,
            **auth_connect_args,
        )
        try:
            yield connection
        finally:
            connection.close()

    # ------------------------------------------------------------------
    # Listing — batch queries run once during `get_schemas`
    # ------------------------------------------------------------------

    def get_columns(
        self,
        conn: Any,
        config: DatabricksSourceConfig,
        names: list[str] | None,
    ) -> dict[str, list[tuple[str, str, bool]]]:  # nosemgrep: tuple-return-prefer-dataclass -- pre-existing, file moved
        selected_schema = normalize_namespace(config.schema)
        qualify = selected_schema is None

        information_schema_columns = _IDENTIFIER_QUOTER.quote_qualified(
            config.catalog, DATABRICKS_SYSTEM_SCHEMA, "columns"
        )

        with conn.cursor() as cursor:
            if selected_schema is not None:
                cursor.execute(
                    "SELECT table_schema, table_name, column_name, data_type, is_nullable"
                    f" FROM {information_schema_columns}"
                    " WHERE table_schema = :schema"
                    " ORDER BY table_schema ASC, table_name ASC, ordinal_position ASC",
                    {"schema": selected_schema},
                )
            else:
                cursor.execute(
                    "SELECT table_schema, table_name, column_name, data_type, is_nullable"
                    f" FROM {information_schema_columns}"
                    " WHERE table_schema != :system_schema"
                    " ORDER BY table_schema ASC, table_name ASC, ordinal_position ASC",
                    {"system_schema": DATABRICKS_SYSTEM_SCHEMA},
                )
            result = cursor.fetchall()

        schema_list: dict[str, list[tuple[str, str, bool]]] = collections.defaultdict(list)
        for table_schema, table_name, column_name, data_type, is_nullable in result:
            display_name = f"{table_schema}.{table_name}" if qualify else table_name
            schema_list[display_name].append((column_name, data_type, is_nullable == "YES"))

        if names is not None:
            # Match qualified (`schema.table`) and bare (`table`) names — a row requested by its
            # qualified name can still map to a bare discovery key (or vice versa) mid-migration.
            available = dict(schema_list)
            filtered: dict[str, list[tuple[str, str, bool]]] = {}
            for name in names:
                if name in available:
                    filtered[name] = available[name]
                elif "." in name:
                    _schema, _, unqualified = name.partition(".")
                    if unqualified in available:
                        filtered[name] = available[unqualified]
            return filtered

        return dict(schema_list)

    def get_primary_keys(
        self,
        conn: Any,
        config: DatabricksSourceConfig,
        tables: list[str],
    ) -> dict[str, list[str] | None]:
        """Detect informational primary-key constraints for the given tables.

        One batched query over the catalog's `information_schema` joins
        `table_constraints` to `key_column_usage`, so a blank-namespace
        discovery over a wide catalog stays a single round trip.

        Unity Catalog PK constraints are informational and optional — most
        tables won't have one, and legacy `hive_metastore` has no
        `information_schema` at all. Swallow and log failures so schema
        discovery keeps working without PKs (the base falls back to an `id`
        column when present).
        """
        result: dict[str, list[str] | None] = dict.fromkeys(tables)
        if not tables:
            return result

        default_schema = normalize_namespace(config.schema)
        display_by_pair: dict[tuple[str, str], str] = {}
        for display_name in tables:
            schema, table = _split_display_name(display_name, default_schema)
            if schema is None:
                continue
            display_by_pair[(schema, table)] = display_name

        table_constraints = _IDENTIFIER_QUOTER.quote_qualified(
            config.catalog, DATABRICKS_SYSTEM_SCHEMA, "table_constraints"
        )
        key_column_usage = _IDENTIFIER_QUOTER.quote_qualified(
            config.catalog, DATABRICKS_SYSTEM_SCHEMA, "key_column_usage"
        )

        try:
            with conn.cursor() as cursor:
                cursor.execute(
                    f"""
                    SELECT kcu.table_schema, kcu.table_name, kcu.column_name, kcu.ordinal_position
                    FROM {table_constraints} AS tc
                    JOIN {key_column_usage} AS kcu
                      ON tc.constraint_catalog = kcu.constraint_catalog
                     AND tc.constraint_schema = kcu.constraint_schema
                     AND tc.constraint_name = kcu.constraint_name
                    WHERE tc.constraint_type = 'PRIMARY KEY'
                    """
                )

                keys_by_pair: dict[tuple[str, str], list[tuple[int, str]]] = collections.defaultdict(list)
                for table_schema, table_name, column_name, ordinal_position in cursor.fetchall():
                    keys_by_pair[(table_schema, table_name)].append((ordinal_position or 0, column_name))

                for pair, ordered in keys_by_pair.items():
                    display_key = display_by_pair.get(pair)
                    if display_key is None:
                        continue
                    keys = [column for _position, column in sorted(ordered, key=lambda item: item[0])]
                    if keys:
                        result[display_key] = keys
        except Exception as e:
            structlog.get_logger().warning("Failed to detect primary keys for Databricks tables", exc_info=e)

        return result

    def get_source_metadata(
        self,
        conn: Any,
        config: DatabricksSourceConfig,
        tables: list[str],
    ) -> SourceMetadata:
        """Stamp catalog/schema/table per discovered table so per-row routing can pin a namespace.

        The catalog is the connection's catalog (constant, display-only); the
        schema and unqualified table come from the `schema.table` display name,
        falling back to the configured schema for a single-schema source.
        """
        default_schema = normalize_namespace(config.schema)
        catalog_by_table: dict[str, str | None] = {}
        schema_by_table: dict[str, str | None] = {}
        table_name_by_table: dict[str, str | None] = {}
        for display_name in tables:
            schema, table = _split_display_name(display_name, default_schema)
            catalog_by_table[display_name] = config.catalog
            schema_by_table[display_name] = schema
            table_name_by_table[display_name] = table
        return SourceMetadata(
            catalog_by_table=catalog_by_table,
            schema_by_table=schema_by_table,
            table_name_by_table=table_name_by_table,
        )

    def get_incremental_filter(self) -> IncrementalFieldFilter:
        return filter_databricks_incremental_fields

    # ------------------------------------------------------------------
    # Per-cursor metadata — used during `build_pipeline`
    # ------------------------------------------------------------------

    def get_primary_keys_for_table(
        self,
        cursor: Any,
        catalog: str,
        schema: str,
        table_name: str,
    ) -> list[str] | None:
        """Return the primary-key column names for a single table, or None.

        Permission-sensitive like the schema-level `get_primary_keys` — swallow
        a failing lookup and return None so the pipeline falls back to a
        persisted or `id`-column primary key instead of crashing a merge.
        """
        table_constraints = _IDENTIFIER_QUOTER.quote_qualified(catalog, DATABRICKS_SYSTEM_SCHEMA, "table_constraints")
        key_column_usage = _IDENTIFIER_QUOTER.quote_qualified(catalog, DATABRICKS_SYSTEM_SCHEMA, "key_column_usage")

        try:
            cursor.execute(
                f"""
                SELECT kcu.column_name, kcu.ordinal_position
                FROM {table_constraints} AS tc
                JOIN {key_column_usage} AS kcu
                  ON tc.constraint_catalog = kcu.constraint_catalog
                 AND tc.constraint_schema = kcu.constraint_schema
                 AND tc.constraint_name = kcu.constraint_name
                WHERE tc.constraint_type = 'PRIMARY KEY'
                  AND kcu.table_schema = :schema
                  AND kcu.table_name = :table_name
                ORDER BY kcu.ordinal_position ASC
                """,
                {"schema": schema, "table_name": table_name},
            )
            keys = [row[0] for row in cursor.fetchall()]
        except Exception as e:
            structlog.get_logger().warning(
                "Failed to detect primary key for Databricks table",
                catalog=catalog,
                schema=schema,
                table_name=table_name,
                exc_info=e,
            )
            return None

        return keys if len(keys) > 0 else None

    def fetch_table_stats(
        self,
        cursor: Any,
        schema: str,
        table_name: str,
        logger: FilteringBoundLogger,
    ) -> TableStats | None:
        """Estimate the table's size and row count for partition sizing.

        `DESCRIBE DETAIL` returns a Delta table's `sizeInBytes` but no row count;
        `COUNT(*)` on a Delta table is metadata-optimized (answered from file
        statistics, no scan). Non-Delta objects (views, federated tables) make
        `DESCRIBE DETAIL` raise or return no size — `get_partition_settings`
        treats any failure here as "no stats" and skips partition sizing.

        `sizeInBytes` is the compressed on-disk size, so the derived average row
        size is an underestimate and partitions come out larger than the byte
        target — acceptable for a best-effort estimate.
        """
        table_ref = _IDENTIFIER_QUOTER.quote_qualified(schema, table_name)

        cursor.execute(f"DESCRIBE DETAIL {table_ref}")
        detail_row = cursor.fetchone()
        if detail_row is None or cursor.description is None:
            return None
        size_index = next((i for i, column in enumerate(cursor.description) if column[0] == "sizeInBytes"), -1)
        if size_index == -1:
            return None
        size_in_bytes = detail_row[size_index]
        if not size_in_bytes:
            return None

        cursor.execute(f"SELECT COUNT(*) FROM {table_ref}")
        count_row = cursor.fetchone()
        if count_row is None:
            return None
        row_count = int(count_row[0] or 0)
        if row_count == 0:
            return None

        return TableStats(table_size_bytes=int(size_in_bytes), row_count=row_count)

    # ------------------------------------------------------------------
    # Pipeline build — the `SourceResponse` for a single table
    # ------------------------------------------------------------------

    def build_pipeline(self, config: DatabricksSourceConfig, inputs: SourceInputs) -> SourceResponse:
        # Per-row routing: a multi-schema row pins its own namespace via `schema_metadata`,
        # a legacy single-schema row falls back to `config.schema`. The catalog is fixed
        # per connection. `response_name` preserves the legacy Delta subdir (`dwh_storage_key`).
        location = resolve_source_location(inputs, config_namespace=config.schema)
        table_name = location.table_name
        schema = location.schema
        if not table_name:
            raise ValueError("Table name is missing")
        if not schema:
            raise ValueError("Schema is missing")

        logger = inputs.logger
        incremental_field = inputs.incremental_field if inputs.should_use_incremental_field else None
        incremental_field_type = inputs.incremental_field_type if inputs.should_use_incremental_field else None

        with self.connect(config) as connection:
            with connection.cursor() as cursor:
                primary_keys = self.get_primary_keys_for_table(cursor, config.catalog, schema, table_name)
                # The session catalog is pinned at connect time, so two-part `schema.table`
                # references always resolve inside `config.catalog`.
                query = _QUERY_BUILDER.select_all(
                    schema=schema,
                    table_name=table_name,
                    incremental_field=incremental_field,
                    incremental_field_type=incremental_field_type,
                    incremental_last_value=inputs.db_incremental_field_last_value,
                    enabled_columns=inputs.enabled_columns,
                    primary_keys=primary_keys,
                    row_filters=inputs.row_filters,
                )
                rows_to_sync = self.get_rows_to_sync(cursor, query.sql, query.params, logger)
                # Partition sizing only pays off on incremental merges (same as MSSQL/MySQL) —
                # full refreshes rewrite the whole table regardless, so skip the extra probes.
                partition_settings = (
                    self.get_partition_settings(cursor, schema, table_name, logger)
                    if inputs.should_use_incremental_field
                    else None
                )

        def get_rows() -> Iterator[Any]:
            with self.connect(config) as streaming_connection:
                with streaming_connection.cursor() as streaming_cursor:
                    logger.debug(f"Databricks query: {query.sql}")
                    streaming_cursor.execute(query.sql, query.params)

                    while True:
                        table = streaming_cursor.fetchmany_arrow(DEFAULT_DATABRICKS_FETCH_SIZE)
                        if table.num_rows == 0:
                            break
                        yield table

        return SourceResponse(
            name=location.response_name,
            items=get_rows,
            primary_keys=primary_keys,
            partition_count=partition_settings.partition_count if partition_settings else None,
            partition_size=partition_settings.partition_size if partition_settings else None,
            rows_to_sync=rows_to_sync,
            # One statement with `ORDER BY <incremental field>`, read as one stream.
            sort_mode="asc" if incremental_field is not None else None,
        )