Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.9 KB
Raw Blame
from typing import Optional, cast

from products.warehouse_sources.sources.openfda._config import OpenFDASourceConfig
from products.warehouse_sources.sources.openfda.openfda import (
    OpenFDAResumeConfig,
    openfda_source,
    validate_credentials as validate_openfda_credentials,
)
from products.warehouse_sources.sources.openfda.settings import ENDPOINTS, INCREMENTAL_FIELDS, OPENFDA_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    detect,
)


@SourceRegistry.register
class OpenFDASource(ResumableSource[OpenFDASourceConfig, OpenFDAResumeConfig]):
    codebase_detection = detect(env=("^(OPEN_FDA|OPENFDA)_",))

    lists_tables_without_credentials = True  # static endpoint catalog — safe for public docs
    api_docs_url = "https://open.fda.gov/apis/"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.OPENFDA

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.OPENFDA,
            category=DataWarehouseSourceCategory.ANALYTICS,
            label="openFDA",
            releaseStatus=ReleaseStatus.GA,
            caption="""Pull U.S. FDA drug, device, and food data — adverse event reports, recalls, drug labeling, drug applications and shortages, 510(k) clearances, device classifications, the UDI device registry, and the NDC directory — into the PostHog Data warehouse.

An API key is optional but recommended. Without one, openFDA limits you to 1,000 requests/day per IP; with one, 120,000 requests/day. Get a free key from the [openFDA API basics page](https://open.fda.gov/apis/authentication/).""",
            iconPath="/static/services/openfda.png",
            docsUrl="https://posthog.com/docs/cdp/sources/openfda",
            fields=cast(
                list[FieldType],
                [
                    SourceFieldInputConfig(
                        name="api_key",
                        label="API key (optional)",
                        type=SourceFieldInputConfigType.PASSWORD,
                        required=False,
                        placeholder="Your openFDA API key",
                        secret=True,
                    ),
                ],
            ),
            keywords=["fda", "openfda", "drug", "device", "food"],
        )

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        from products.warehouse_sources.sources.openfda.canonical_descriptions import CANONICAL_DESCRIPTIONS

        return CANONICAL_DESCRIPTIONS

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        return {
            # openFDA returns 401/403 only for an invalid or over-quota API key. Retrying can't satisfy
            # a credential/quota problem, so stop the sync. Match the stable status text and base host.
            "401 Client Error: Unauthorized for url: https://api.fda.gov": "Your openFDA API key is invalid. Generate a new key from the openFDA developer portal and reconnect, or remove it to use the unauthenticated tier.",
            "403 Client Error: Forbidden for url: https://api.fda.gov": "Your openFDA API key is invalid or has exceeded its daily request quota. Check the key, wait for the quota to reset, then reconnect.",
        }

    def get_schemas(
        self,
        config: OpenFDASourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        def _build_schema(endpoint: str) -> SourceSchema:
            endpoint_config = OPENFDA_ENDPOINTS[endpoint]
            has_incremental = endpoint_config.incremental_field is not None
            return SourceSchema(
                name=endpoint,
                supports_incremental=has_incremental,
                supports_append=has_incremental,
                incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
                should_sync_default=endpoint_config.should_sync_default,
                detected_primary_keys=endpoint_config.primary_keys,
            )

        schemas = [_build_schema(endpoint) for endpoint in ENDPOINTS]
        if names is not None:
            names_set = set(names)
            schemas = [s for s in schemas if s.name in names_set]
        return schemas

    def validate_credentials(
        self,
        config: OpenFDASourceConfig,
        team_id: int,
        schema_name: Optional[str] = None,
        api_version: str | None = None,
    ) -> tuple[bool, str | None]:
        if validate_openfda_credentials(config.api_key):
            return True, None

        return False, "Could not reach the openFDA API. Check your API key (if provided) and try again."

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[OpenFDAResumeConfig]:
        return ResumableSourceManager[OpenFDAResumeConfig](inputs, OpenFDAResumeConfig)

    def source_for_pipeline(
        self,
        config: OpenFDASourceConfig,
        resumable_source_manager: ResumableSourceManager[OpenFDAResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        return openfda_source(
            api_key=config.api_key,
            endpoint=inputs.schema_name,
            team_id=inputs.team_id,
            job_id=inputs.job_id,
            resumable_source_manager=resumable_source_manager,
            should_use_incremental_field=inputs.should_use_incremental_field,
            db_incremental_field_last_value=inputs.db_incremental_field_last_value
            if inputs.should_use_incremental_field
            else None,
            incremental_field=inputs.incremental_field,
        )