:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from typing import Optional, cast
from products.warehouse_sources.sources.dbt._config import DbtSourceConfig
from products.warehouse_sources.sources.dbt.dbt import (
DISCOVERY_TOKEN_ERROR,
DbtResumeConfig,
dbt_source,
get_endpoint_permissions as get_dbt_endpoint_permissions,
validate_credentials as validate_dbt_credentials,
)
from products.warehouse_sources.sources.dbt.settings import (
DBT_DEFAULT_VERSION,
DBT_ENDPOINTS,
DBT_SUPPORTED_VERSIONS,
ENDPOINTS,
INCREMENTAL_FIELDS,
)
from products.warehouse_sources.sources.sdk import (
CanonicalDescriptions,
DataWarehouseSourceCategory,
ExternalDataSourceType,
FieldType,
ReleaseStatus,
ResumableSource,
ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceFieldSelectConfig,
SourceFieldSelectConfigOption,
SourceInputs,
SourceRegistry,
SourceResponse,
SourceSchema,
detect,
)
@SourceRegistry.register
class DbtSource(ResumableSource[DbtSourceConfig, DbtResumeConfig]):
# NOTE: these signals prove the project uses dbt, not that it uses dbt
# Cloud - the source needs dbt Cloud credentials. Local-only dbt users
# are a known false positive.
codebase_detection = detect(
env=("^DBT_",),
pypi=(
"dbt-core",
"dbt-snowflake",
"dbt-postgres",
"dbt-bigquery",
"dbt-redshift",
"dbt-databricks",
"dbt-spark",
"dbt-athena",
"dbt-fabric",
"dbt-trino",
"dbt-duckdb",
"dbt-clickhouse",
),
)
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
api_docs_url = "https://docs.getdbt.com/docs/dbt-cloud-apis/overview" # coverage spans both Admin API v2 and v3
supported_versions = DBT_SUPPORTED_VERSIONS
default_version = DBT_DEFAULT_VERSION
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.DBT
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.DBT,
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="dbt",
keywords=["dbt cloud", "dbt labs", "dbt platform"],
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your dbt platform (dbt Cloud) account ID and API token to pull your dbt projects, jobs, environments, and run history into the PostHog Data warehouse.
You can create a service account token under **Account settings** → **API tokens** → **Service tokens** in dbt (Team and Enterprise plans), or use a personal access token. Read-only access to the resources you want to sync is enough.
The model, test, source, snapshot, seed and exposure tables come from the dbt Discovery API, so those need a token with metadata access on a Starter plan or above. The audit log table needs an Enterprise plan.
Your account ID is the number after `/deploy/` in your dbt URL. If your account runs on a cell-based or single-tenant deployment (for example `https://ab123.us1.dbt.com`), enter that base URL in the custom base URL field, and enter your Discovery API URL from **Account settings** → **Access URLs**.""",
iconPath="/static/services/dbt.png",
docsUrl="https://posthog.com/docs/cdp/sources/dbt",
fields=cast(
list[FieldType],
[
SourceFieldInputConfig(
name="account_id",
label="Account ID",
type=SourceFieldInputConfigType.TEXT,
required=True,
placeholder="12345",
secret=False,
),
SourceFieldInputConfig(
name="api_token",
label="API token",
type=SourceFieldInputConfigType.PASSWORD,
required=True,
placeholder="",
secret=True,
),
SourceFieldSelectConfig(
name="region",
label="Region",
required=True,
defaultValue="us",
options=[
SourceFieldSelectConfigOption(label="US (cloud.getdbt.com)", value="us"),
SourceFieldSelectConfigOption(label="EMEA (emea.dbt.com)", value="emea"),
SourceFieldSelectConfigOption(label="APAC (au.dbt.com)", value="au"),
],
),
SourceFieldInputConfig(
name="custom_base_url",
label="Custom base URL (optional)",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder="https://ab123.us1.dbt.com",
secret=False,
),
SourceFieldInputConfig(
name="discovery_api_url",
label="Discovery API URL (optional)",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder="https://ab123.metadata.us1.dbt.com/graphql",
secret=False,
),
],
),
)
@property
def connection_host_fields(self) -> list[str]:
# region, custom_base_url and discovery_api_url pick the hosts; account_id is the path the
# token is sent to. Retargeting any of them must re-require the token so a preserved
# credential can't be pointed at another host or another account authorized by that token.
return ["region", "custom_base_url", "discovery_api_url", "account_id"]
def get_canonical_descriptions(self) -> CanonicalDescriptions:
from products.warehouse_sources.sources.dbt.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
def get_non_retryable_errors(self) -> dict[str, str | None]:
# The base URL varies by region/custom deployment, so match on the stable status text only.
# These errors always originate from this source's own requests to the dbt API.
return {
"401 Client Error: Unauthorized for url:": "Your dbt API token is invalid or has been revoked. Create a new service token in your dbt account settings, then reconnect.",
"403 Client Error: Forbidden for url:": "Your dbt API token is missing the permissions needed to sync this data. Grant the required read permissions in your dbt account settings, then reconnect.",
# The Discovery API answers an auth failure with HTTP 200 and an errors body, so there
# is no status text to match on.
DISCOVERY_TOKEN_ERROR: "Your dbt API token cannot read the Discovery API, which serves the model, test, source, snapshot, seed and exposure tables. Create a service token with metadata access, then reconnect.",
}
def get_schemas(
self,
config: DbtSourceConfig,
team_id: int,
with_counts: bool = False,
names: list[str] | None = None,
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
def _build_schema(endpoint: str) -> SourceSchema:
endpoint_config = DBT_ENDPOINTS[endpoint]
has_incremental = len(INCREMENTAL_FIELDS.get(endpoint, [])) > 0
return SourceSchema(
name=endpoint,
supports_incremental=has_incremental,
# The incremental lookback re-pulls a window of rows each run; only merge dedupes
# those on the primary key, append would materialize them as duplicates.
supports_append=False,
incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
should_sync_default=endpoint_config.should_sync_default,
description=endpoint_config.description,
)
schemas = [_build_schema(endpoint) for endpoint in ENDPOINTS]
if names is not None:
names_set = set(names)
schemas = [s for s in schemas if s.name in names_set]
return schemas
def validate_credentials(
self, config: DbtSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = None
) -> tuple[bool, str | None]:
return validate_dbt_credentials(
api_token=config.api_token,
account_id=config.account_id,
region=config.region,
custom_base_url=config.custom_base_url,
team_id=team_id,
schema_name=schema_name,
discovery_api_url=config.discovery_api_url,
)
def get_endpoint_permissions(
self, config: DbtSourceConfig, team_id: int, endpoints: list[str], api_version: str | None = None
) -> dict[str, str | None]:
return get_dbt_endpoint_permissions(
api_token=config.api_token,
account_id=config.account_id,
region=config.region,
custom_base_url=config.custom_base_url,
team_id=team_id,
endpoints=endpoints,
discovery_api_url=config.discovery_api_url,
)
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[DbtResumeConfig]:
return ResumableSourceManager[DbtResumeConfig](inputs, DbtResumeConfig)
def source_for_pipeline(
self,
config: DbtSourceConfig,
resumable_source_manager: ResumableSourceManager[DbtResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
return dbt_source(
api_token=config.api_token,
account_id=config.account_id,
region=config.region,
custom_base_url=config.custom_base_url,
endpoint=inputs.schema_name,
team_id=inputs.team_id,
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
should_use_incremental_field=inputs.should_use_incremental_field,
db_incremental_field_last_value=inputs.db_incremental_field_last_value
if inputs.should_use_incremental_field
else None,
incremental_field=inputs.incremental_field,
discovery_api_url=config.discovery_api_url,
)