:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add openfda drug applications, shortages, device classification and udi tables (#123361)
Tom Owers298362b402df
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex d20ff919bc2e..850befe461a9 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -827,7 +827,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/opinion_stage.py1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 7fe7ff6b1d98..e3703660e98c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8808,10 +8808,10 @@ Today (9): `device_510k`, `device_enforcement`, `device_events`, `drug_enforcemeDiffed against: <https://api.fda.gov/download.json>-- [ ] `device/classification` — lookup resolving device product codes, classes and regulation numbers referenced by 510k, UDI and device events (high)-- [ ] `drug/drugsfda` — drug approvals and applications - the core regulatory fact table, absent entirely (high)-- [ ] `device/udi` — device identifier registry, the lookup that resolves devices named in adverse events and recalls (high)-- [ ] `drug/shortages` — current and resolved drug shortage records, a headline supply-chain signal (medium)+- [x] `device/classification` — lookup resolving device product codes, classes and regulation numbers referenced by 510k, UDI and device events (high)+- [x] `drug/drugsfda` — drug approvals and applications - the core regulatory fact table, absent entirely (high)+- [x] `device/udi` — device identifier registry, the lookup that resolves devices named in adverse events and recalls (high)+- [x] `drug/shortages` — current and resolved drug shortage records, a headline supply-chain signal (medium)- [ ] `device/recall` — recall records distinct from the enforcement reports already synced (medium)- [ ] `device/pma` — premarket approval submissions, the higher-risk counterpart to 510k (medium)- [ ] `device/registrationlisting` — establishment registrations, the manufacturer lookup behind device records (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.pyindex c2c8b80abd7a..72282e303278 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.py@@ -44,6 +44,35 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"product_type": "Product type (e.g. HUMAN PRESCRIPTION DRUG, HUMAN OTC DRUG).",},},+ "drug_applications": {+ "description": "Drugs@FDA: FDA-approved brand-name and generic drug products and their applications (NDA, ANDA, BLA). One row per application.",+ "docs_url": "https://open.fda.gov/apis/drug/drugsfda/",+ "columns": {+ "application_number": "NDA, ANDA, or BLA number of the application.",+ "sponsor_name": "Company that submitted the application to the FDA to market the drug in the U.S.",+ "submissions": "Nested list of submissions (original approval, supplements) with their type, status, and status date.",+ "products": "Nested list of the drug products covered by the application (brand name, dosage form, active ingredients, marketing status).",+ "openfda": "Nested object of harmonized identifiers (NDC, brand name, generic name, RxCUI, etc.).",+ },+ },+ "drug_shortages": {+ "description": "Current and resolved drug shortages reported to the FDA. One row per shortage record for a drug presentation; a package NDC can appear more than once.",+ "docs_url": "https://open.fda.gov/apis/drug/drugshortages/",+ "columns": {+ "package_ndc": "National Drug Code identifying the labeler, product, and package size.",+ "generic_name": "Generic name(s) of the drug product.",+ "company_name": "Company that makes the drug product.",+ "presentation": "Drug name, form, strength, and NDC number.",+ "status": "Shortage status of the presentation (e.g. Current, Resolved, To Be Discontinued).",+ "availability": "Availability information for the presentation.",+ "shortage_reason": "Reason for the shortage of the presentation.",+ "initial_posting_date": "Date the presentation was added to the FDA shortage list (MM/DD/YYYY).",+ "update_date": "Date the presentation was last updated (MM/DD/YYYY).",+ "update_type": "Type of the last update (e.g. New, Revised, Reverified).",+ "discontinued_date": "Date the presentation was discontinued (MM/DD/YYYY).",+ "therapeutic_category": "List of therapeutic categories for the drug.",+ },+ },"drug_enforcement": {"description": "Drug recall enforcement reports — recalls of drug products classified by the FDA. One row per recall.","docs_url": "https://open.fda.gov/apis/drug/enforcement/",@@ -82,6 +111,40 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"product_code": "FDA product classification code for the device.",},},+ "device_classification": {+ "description": "Medical device classification: every FDA device product code with its device class and classification regulation. One row per product code.",+ "docs_url": "https://open.fda.gov/apis/device/classification/",+ "columns": {+ "product_code": "Three-letter identifier assigned to a device category.",+ "device_name": "Name of the device category.",+ "device_class": "Risk-based device class (1, 2, or 3; U = unclassified, N = not classified, F = HDE).",+ "regulation_number": "Classification regulation in 21 CFR under which the device is classified.",+ "medical_specialty": "Two-letter code of the medical specialty panel for the classification regulation.",+ "medical_specialty_description": "Readable name of the medical specialty panel.",+ "review_panel": "510(k) review panel that would review the device.",+ "submission_type_id": "Submission type the product code is limited to (510(k), PMA, 510(k) exempt, etc.).",+ "implant_flag": "Whether the device is an implant (Y/N).",+ "life_sustain_support_flag": "Whether the device is life-sustaining or life-supporting (Y/N).",+ },+ },+ "device_udi": {+ "description": "Global Unique Device Identification Database (GUDID): device identifier records submitted by device labelers. One row per device identifier record.",+ "docs_url": "https://open.fda.gov/apis/device/udi/",+ "columns": {+ "public_device_record_key": "Unique identifier for the device identifier record.",+ "publish_date": "Date the record was published to public search (YYYY-MM-DD).",+ "public_version_date": "Date the current version of the record was published (YYYY-MM-DD).",+ "brand_name": "Proprietary, trade, or brand name of the device as used on its labeling.",+ "version_or_model_number": "Version or model shown on the device label or packaging.",+ "company_name": "Labeler company name associated with the record.",+ "device_description": "Additional information about the device.",+ "commercial_distribution_status": "Whether the device is in commercial distribution.",+ "record_status": "Status of the device identifier record.",+ "identifiers": "Nested list of device identifiers (primary DI, package DIs) and their issuing agencies.",+ "product_codes": "Nested list of FDA product codes for the device; joins to the device classification table.",+ "gmdn_terms": "Nested list of Global Medical Device Nomenclature terms describing the device.",+ },+ },"device_enforcement": {"description": "Medical device recall enforcement reports. One row per recall.","docs_url": "https://open.fda.gov/apis/device/enforcement/",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.pyindex 80ae038ea020..c66090cdb51f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py@@ -1,8 +1,8 @@"""openFDA (U.S. Food and Drug Administration) REST API transport.openFDA (https://api.fda.gov) is the FDA's free public API over drug, device, and food regulatory-datasets (adverse events, recalls/enforcement reports, drug labeling, 510(k) clearances, the NDC-directory). Every dataset is its own endpoint with its own schema and date field; responses wrap the+datasets (adverse events, recalls/enforcement reports, drug labeling, Drugs@FDA applications, drug+shortages, 510(k) clearances, device classification, the UDI registry, the NDC directory). Every dataset is its own endpoint with its own schema and date field; responses wrap therecords in `{"meta": ..., "results": [...]}`.Pagination is a `search_after` cursor exposed via the HTTP `Link: rel="next"` header (the `skip`diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.pyindex 9017da32bc41..8eca45997d22 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py@@ -1,15 +1,17 @@-from dataclasses import dataclassfrom typing import Optional+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@frozenclass OpenFDAEndpointConfig:name: str# Path under https://api.fda.gov, e.g. "/drug/enforcement.json".path: str- primary_keys: list[str]+ # None when the dataset has no unique record key; such endpoints must stay full refresh only.+ primary_keys: Optional[list[str]]# Server-side date field used both for incremental filtering (search=<field>:[low TO high]) and for# ascending sort. None means the endpoint has no reliable date cursor, so it ships full refresh only.incremental_field: Optional[str] = None@@ -33,9 +35,9 @@ class OpenFDAEndpointConfig:# openFDA exposes one endpoint per dataset, each with its own schema and date field. We surface the# datasets a user is most likely to want across the drug, device, and food domains. Adverse-event and-# enforcement (recall) endpoints carry a genuine server-side date filter, so they sync incrementally;-# the NDC directory and drug labeling endpoints have no reliable row-level date cursor, so they ship-# full refresh only.+# enforcement (recall) endpoints and the UDI registry carry a genuine server-side date filter, so they+# sync incrementally; the reference datasets (NDC directory, drug labeling, Drugs@FDA, device+# classification, drug shortages) have no reliable row-level date cursor, so they ship full refresh only.OPENFDA_ENDPOINTS: dict[str, OpenFDAEndpointConfig] = {"drug_events": OpenFDAEndpointConfig(name="drug_events",@@ -62,6 +64,25 @@ OPENFDA_ENDPOINTS: dict[str, OpenFDAEndpointConfig] = {incremental_field=None,partition_key=None,),+ "drug_applications": OpenFDAEndpointConfig(+ name="drug_applications",+ path="/drug/drugsfda.json",+ primary_keys=["application_number"],+ # Submission dates live inside the nested `submissions` list, so there's no top-level date+ # to filter on — full refresh only.+ incremental_field=None,+ partition_key=None,+ ),+ "drug_shortages": OpenFDAEndpointConfig(+ name="drug_shortages",+ path="/drug/shortages.json",+ # One NDC can carry several shortage records, and the dataset contains exact duplicate rows,+ # so no field combination is unique.+ primary_keys=None,+ # Dates arrive as MM/DD/YYYY, which the YYYYMMDD search filter can't match — full refresh only.+ incremental_field=None,+ partition_key=None,+ ),"drug_enforcement": OpenFDAEndpointConfig(name="drug_enforcement",path="/drug/enforcement.json",@@ -83,6 +104,25 @@ OPENFDA_ENDPOINTS: dict[str, OpenFDAEndpointConfig] = {incremental_field="decision_date",partition_key="decision_date",),+ "device_classification": OpenFDAEndpointConfig(+ name="device_classification",+ path="/device/classification.json",+ primary_keys=["product_code"],+ # A small reference list with no per-record date — full refresh only.+ incremental_field=None,+ partition_key=None,+ ),+ "device_udi": OpenFDAEndpointConfig(+ name="device_udi",+ path="/device/udi.json",+ primary_keys=["public_device_record_key"],+ # `public_version_date` moves forward whenever a record is republished, so it picks up edits;+ # `publish_date` is fixed at first publication, so it partitions.+ incremental_field="public_version_date",+ partition_key="publish_date",+ # Millions of rows — opt-in so a new source doesn't spend the unauthenticated daily quota on it.+ should_sync_default=False,+ ),"device_enforcement": OpenFDAEndpointConfig(name="device_enforcement",path="/device/enforcement.json",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.pyindex e060ecab15bf..1e1b0b783572 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.py@@ -50,7 +50,7 @@ class OpenFDASource(ResumableSource[OpenFDASourceConfig, OpenFDAResumeConfig]):category=DataWarehouseSourceCategory.ANALYTICS,label="openFDA",releaseStatus=ReleaseStatus.BETA,- caption="""Pull U.S. FDA drug, device, and food data — adverse event reports, recalls, drug labeling, 510(k) clearances, and the NDC directory — into the PostHog Data warehouse.+ caption="""Pull U.S. FDA drug, device, and food data — adverse event reports, recalls, drug labeling, drug applications and shortages, 510(k) clearances, device classifications, the UDI device registry, and the NDC directory — into the PostHog Data warehouse.An API key is optional but recommended. Without one, openFDA limits you to 1,000 requests/day per IP; with one, 120,000 requests/day. Get a free key from the [openFDA API basics page](https://open.fda.gov/apis/authentication/).""",iconPath="/static/services/openfda.png",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.pyindex 95714ee86fca..f9a87973e1eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.py@@ -144,10 +144,13 @@ class TestOpenfdaSource:("drug_enforcement", ["recall_number"], "report_date"),("drug_ndc", ["product_id"], None),("device_510k", ["k_number"], "decision_date"),+ # Incremental on `public_version_date` but partitioned on the fixed `publish_date`.+ ("device_udi", ["public_device_record_key"], "publish_date"),+ ("drug_shortages", None, None),])def test_source_response_carries_endpoint_config(- self, endpoint: str, primary_keys: list[str], partition_key: str | None+ self, endpoint: str, primary_keys: list[str] | None, partition_key: str | None) -> None:result = _driver().run(endpoint, [_page([])])assert result.raised is Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.pyindex c9672156ed16..0a9c699e3ee0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.py@@ -45,10 +45,13 @@ class TestGetSchemas:# sync silently re-scans the whole dataset.("drug_labels", False, ["id"]),("drug_ndc", False, ["product_id"]),+ ("device_udi", True, ["public_device_record_key"]),+ # No unique key, so merge-based incremental or append would duplicate rows.+ ("drug_shortages", False, None),])def test_incremental_support_matches_endpoint(- self, endpoint: str, expect_incremental: bool, primary_keys: list[str]+ self, endpoint: str, expect_incremental: bool, primary_keys: list[str] | None) -> None:schema = {s.name: s for s in OpenFDASource().get_schemas(_config(), team_id=1)}[endpoint]assert schema.supports_incremental is expect_incremental